diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml index efbf80bc5f..4dbfd3ecb5 100644 --- a/.github/workflows/helm-e2e.yaml +++ b/.github/workflows/helm-e2e.yaml @@ -25,7 +25,6 @@ jobs: env: VERSION: helm-e2e E2E_ATENET_DATAPLANE: agentgateway - E2E_INGRESS_AUTH_MODE: deprecated-insecure E2E_CREDENTIAL_PROVIDER: "1" # Retain external-volume coverage after the preview gate is introduced. ATE_PREVIEW: ALL @@ -75,12 +74,12 @@ jobs: run: helm upgrade --install substrate-crds charts/substrate-crds - name: Bootstrap mTLS authorities run: | - hack/install-ate-kind.sh --create-podcertificate-controller-cas - hack/install-ate-kind.sh --create-jwt-authority-pool-secret - hack/install-ate-kind.sh --create-actor-id-ca-pool-secret - hack/install-ate-kind.sh --create-actor-id-ca-certs-secret - hack/install-ate-kind.sh --create-api-authentication-config - hack/install-ate-kind.sh --create-egress-mitm-ca-pool-secret + hack/install-ate-kind.sh create podcertificate-controller-cas + hack/install-ate-kind.sh create jwt-authority-pool + hack/install-ate-kind.sh create actor-id-ca-pool + hack/install-ate-kind.sh create actor-id-ca-certs + hack/install-ate-kind.sh create api-authentication-config + hack/install-ate-kind.sh create egress-mitm-ca-pool - name: Install certificate controller run: | helm upgrade --install substrate-podcert charts/substrate-podcert \ @@ -109,16 +108,16 @@ jobs: sudo modprobe nfs || true sudo modprobe nfsd || true - name: Install CSI NFS driver - run: hack/install-ate-kind.sh --setup-csi=nfs + run: hack/install-ate-kind.sh setup csi nfs - name: Deploy micro-VM counter demo # The deploy creates the substrate ActorTemplate and waits for its golden # snapshot internally; the ActorTemplate CRD (and its Ready condition) # no longer exists to wait on. run: hack/run-microvm-demo-kind.sh --skip-control-plane - name: Deploy gVisor counter demo - run: hack/install-ate-kind.sh --deploy-demo-counter + run: hack/install-ate-kind.sh deploy demo counter - name: Deploy egress demo - run: hack/install-ate-kind.sh --deploy-demo-egress + run: hack/install-ate-kind.sh deploy demo egress - name: Trust local E2E TLS origins run: hack/setup-e2e-egress-tls-kind.sh - name: Run E2E tests (gVisor) diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 0323c01a0c..81a95b20b4 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -48,7 +48,7 @@ jobs: with: fetch-depth: 0 - name: Setup Go - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: 'go.mod' # Unconditional: a released migration becoming permanent is exactly what @@ -173,11 +173,6 @@ jobs: # --preview, so tests of a disabled gate skip themselves (see # internal/e2e/preview.go). ATE_PREVIEW: ${{ matrix.preview == 'on' && 'ALL' || '' }} - # Job-wide for the same reason, and E2E_INGRESS_AUTH_MODE has to match - # it so the suites dial the router's mTLS listeners. The agentgateway - # dataplane does not support static-mtls. - ATE_INGRESS_AUTH_MODE: ${{ matrix.dataplane == 'agentgateway' && 'deprecated-insecure' || 'static-mtls' }} - E2E_INGRESS_AUTH_MODE: ${{ matrix.dataplane == 'agentgateway' && 'deprecated-insecure' || 'static-mtls' }} # The suites' router client (internal/e2e/ingressauth.go). Only e2e # installs allowlist it, so the suites can reach actors through the # router's mTLS listeners. @@ -198,7 +193,7 @@ jobs: - name: Checkout uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - name: Setup Go - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: 'go.mod' # Use the e2e cache below instead. setup-go's cache key is shared with @@ -257,28 +252,55 @@ jobs: # gateway. ATE_PREVIEW above selects the preview features. env: ATE_ATENET_DATAPLANE: ${{ matrix.dataplane }} - run: hack/install-ate-kind.sh --deploy-ate-system --rollout-timeout=300s + NO_DEV_ENV: "true" + KO_DOCKER_REPO: "localhost:5001" + ATE_INSTALL_KIND: "true" + BUCKET_NAME: "ate-snapshot" + KIND_CLUSTER_NAME: "kind" + KUBECTL_CONTEXT: "kind-kind" + run: go run ./cmd/ate-setup deploy ate-system --rollout-timeout=300s - name: Enable NFS # Load NFS kernel modules so in-cluster NFS server and CSI driver can run. run: | sudo modprobe nfs || true sudo modprobe nfsd || true - name: Install CSI NFS driver - run: hack/install-ate-kind.sh --setup-csi=nfs + env: + NO_DEV_ENV: "true" + KO_DOCKER_REPO: "localhost:5001" + ATE_INSTALL_KIND: "true" + BUCKET_NAME: "ate-snapshot" + KIND_CLUSTER_NAME: "kind" + KUBECTL_CONTEXT: "kind-kind" + run: go run ./cmd/ate-setup setup csi nfs - name: Deploy micro-VM counter demo # Stages the (cached) assets into the cluster's rustfs and deploys the # counter-microvm demo onto the control plane installed above. run: hack/run-microvm-demo-kind.sh --skip-control-plane - name: Deploy gVisor counter demo - run: hack/install-ate-kind.sh --deploy-demo-counter + env: + NO_DEV_ENV: "true" + KO_DOCKER_REPO: "localhost:5001" + ATE_INSTALL_KIND: "true" + BUCKET_NAME: "ate-snapshot" + KIND_CLUSTER_NAME: "kind" + KUBECTL_CONTEXT: "kind-kind" + run: go run ./cmd/ate-setup deploy demo counter - name: Deploy egress demos # TestActorEgress in the networking suite builds its Actor from the egress # ActorTemplate for the class under test, so both fixtures have to exist # before their respective lane runs. The demo's actors trust the CA the # gateway mints its leaves from. + env: + NO_DEV_ENV: "true" + KO_DOCKER_REPO: "localhost:5001" + ATE_INSTALL_KIND: "true" + BUCKET_NAME: "ate-snapshot" + KIND_CLUSTER_NAME: "kind" + KUBECTL_CONTEXT: "kind-kind" run: | - hack/install-ate-kind.sh --deploy-demo-egress - hack/install-ate-kind.sh --deploy-demo-egress-microvm + go run ./cmd/ate-setup deploy demo egress + go run ./cmd/ate-setup deploy demo egress-microvm - name: Trust local TLS origins in egress gateway run: hack/setup-e2e-egress-tls-kind.sh # scale the idle sandbox class's demo pools to zero so each lane has the node diff --git a/.golangci-kal.yaml b/.golangci-kal.yaml index 8ae78cb43e..cc773f3336 100644 --- a/.golangci-kal.yaml +++ b/.golangci-kal.yaml @@ -83,7 +83,7 @@ linters: - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' text: '^(optionalfields|requiredfields): .*\bSandboxConfigSpec\.SandboxClass\b' - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' - text: '^(nomaps|optionalfields): .*\bSandboxVersionConfig\.(Assets|PauseImage)\b' + text: '^(nomaps|optionalfields): .*\bSandboxVersionConfig\.Assets\b' - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' text: '^(nonpointerstructs|requiredfields): .*\bSandboxConfig\.Spec\b' - path: 'pkg/api/v1alpha1/csidriverconfig_types\.go' diff --git a/AGENTS.md b/AGENTS.md index 1784a630ed..e0d6250cca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,6 +45,7 @@ - Keep atelet and the worker runtimes opaque to each other. The only knowledge they may share is `internal/proto/ateworkerpb` and the paths in `internal/nodepath`. Using the same library code is fine; depending on how the other side is built is not. We want atelet to roll ahead of workers eventually, so don't widen that interface casually. Until v1.0.0 nothing enforces this; see Backward Compatibility. - Actor features work on both gVisor and micro-VM, with e2e coverage for both, unless the issue or PR description scopes the change to one runtime. - Don't assume kind, GKE, amd64 or a newer kernel than the repo already needs. Dev scripts must work on arm64 macOS. Don't change host-wide state. +- Put shell code in standalone `.sh` scripts so shellcheck can lint them. Place each script near what uses it, such as `hack/` for dev scripts or a directory under `.github/` for a workflow. Don't inline it in YAML, such as a GitHub workflow `run:` block, in JSON, or in another language's source, such as Go or Python. Have those call the script. Get a reviewer's agreement before making an exception. - Read the `CI` environment variable only inside a named fail-closed predicate, never inline. See Testing Instructions 5. - Before changing an API, read `docs/api-style-guide.md` and `docs/api-validation.md`. - Pin image references you add or update by digest, and look the digest up yourself: `go run github.com/google/go-containerregistry/cmd/crane@latest digest `. @@ -152,7 +153,7 @@ See the [metric registry](docs/observability.md#the-metric-registry) section of Agent Substrate is built for zero-trust, multi-tenant execution of untrusted actors. Preserve and extend these invariants when modifying the codebase (see `docs/security.md` and `docs/threat-model.md` for the full architecture): -- **Workload & Storage Isolation**: Actors run strictly inside `gVisor` (`cmd/ateworker-gvisor`) or `Cloud Hypervisor` microVM (`cmd/ateworker-microvm`) sandboxes; plain Linux containers are not supported. `atelet` scrubs actor directories across lifecycle transitions (`resetActorDirs`), and snapshots are stamped with `SandboxRuntime` metadata and transferred via `cmd/objectstore-plugin`. +- **Workload & Storage Isolation**: Actors run strictly inside `gVisor` (`cmd/ateworker-gvisor`) or `Cloud Hypervisor` microVM (`cmd/ateworker-microvm`) sandboxes; plain Linux containers are not supported. `atelet` scrubs actor directories across lifecycle transitions (`resetActorDirs`), and snapshots are stamped with `ActorSandboxRuntime` metadata and transferred via `cmd/objectstore-plugin`. - **Component & Actor Identity (mTLS, JWTs, and JWKs)**: Inter-component communication uses mTLS backed by `cmd/podcertcontroller` (`podid`, `servicedns`, and `postgres` signers). Actor JWT signing (`internal/localjwtauthority`), verification (`internal/oidcjwt`), and key discovery (served via `cmd/ate-idp-server`) are built on the standard library's crypto packages. Extend those packages rather than adding `go-jose` or another JOSE library, so the code that decides whether a token is valid stays small and supports only the algorithms Substrate uses. -- **Control-Plane Authentication & Authorization**: `ate-api-server` checks every Control RPC against an embedded OpenFGA model (`cmd/ateapi/internal/authz/model.fga`), on by default. Every served unary RPC needs an entry in `defaultRPCPermissions` (`registry.go`): with enforcement on, an RPC without one is denied, and a test fails the build. Access is granted per global scope and atespace through AccessPolicy (`kubectl ate access-policy`); the install's bootstrap owner is the shared `ate-client` ServiceAccount that `kubectl ate` uses. See `docs/authorization.md` for roles and known gaps. Callers are authenticated via `SPIFFE` mTLS or configured OIDC `JWTProvider`s (`cmd/ateapi/internal/apiauthn`). -- **Network Isolation, Ingress, & Egress**: `manifests/ate-install/ate-system-networkpolicies.yaml` enforces default-deny ingress across `ate-system`, and `cmd/atecontroller` reconciles per-`WorkerPool` `NetworkPolicy` objects. Actors run in private `/30` point-to-point veth network namespaces (`internal/ateworker/actornet`) with link-local blocking, policy-enforced DNS filtering (`internal/atunnel/dns`), and zero ambient credentials. `atenet-router --ingress-auth-mode=static-mtls` requires clients to present a `SPIFFE` certificate signed by `--ingress-client-ca-file` whose ID is in `--ingress-allowed-spiffe-ids` (`manifests/ate-install/atenet-router.yaml`). Outbound traffic traverses `atunnel` to `atenet-egress`, which enforces per-actor `EgressPolicy` and injects credentials out-of-band via `cmd/credential-provider/kubernetes-secrets` (`k8s-credential-provider`) or actor JWT minting. +- **Control-Plane Authentication & Authorization**: `ate-api-server` checks every Control RPC against an embedded OpenFGA model (`cmd/ateapi/internal/authz/model.fga`), always enforced. Every served unary RPC needs an entry in `defaultRPCPermissions` (`registry.go`): an RPC without one is denied, and a test fails the build. Access is granted per global scope and atespace through AccessPolicy (`kubectl ate access-policy`); the install's bootstrap owner is the shared `ate-client` ServiceAccount that `kubectl ate` uses. See `docs/authorization.md` for roles and known gaps. Callers are authenticated via `SPIFFE` mTLS or configured OIDC `JWTProvider`s (`cmd/ateapi/internal/apiauthn`). +- **Network Isolation, Ingress, & Egress**: `manifests/ate-install/ate-system-networkpolicies.yaml` enforces default-deny ingress across `ate-system`, and `cmd/atecontroller` reconciles per-`WorkerPool` `NetworkPolicy` objects. Actors run in private `/30` point-to-point veth network namespaces (`internal/ateworker/actornet`) with link-local blocking, policy-enforced DNS filtering (`internal/atunnel/dns`), and zero ambient credentials. Outbound traffic traverses `atunnel` to `atenet-egress`, which enforces per-actor `EgressPolicy` and injects credentials out-of-band via `cmd/credential-provider/kubernetes-secrets` (`k8s-credential-provider`) or actor JWT minting. diff --git a/README.md b/README.md index 53823c2104..2508d717bd 100644 --- a/README.md +++ b/README.md @@ -119,10 +119,10 @@ To quickly set up the complete environment: hack/create-kind-cluster.sh # install ate, PostgreSQL, rustfs -hack/install-ate-kind.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' +hack/install-ate-kind.sh deploy ate-system --credential-provider='{"name":"k8s.io"}' # install counter demo -hack/install-ate-kind.sh --deploy-demo-counter +hack/install-ate-kind.sh deploy demo-counter # install kubectl-ate go install ./cmd/kubectl-ate @@ -168,8 +168,8 @@ curl -X POST \ go run ./tools/setup-gcp bootstrap ``` - On a fresh project this step also creates the atelet Workload Identity IAM - grants that snapshots depend on — see + On a fresh project this step also creates the Workload Identity IAM grants + that ate-api-server and atelet depend on. See [what `create iam` actually grants](tools/setup-gcp/README.md#what-create-iam-actually-grants) to audit them or apply them manually. If you bring your own cluster instead, note the required Kubernetes beta APIs can only be enabled **at cluster @@ -177,7 +177,7 @@ curl -X POST \ 4. Deploy the Agent Substrate system to your cluster: ```bash - ./hack/install-ate.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' + go run ./cmd/ate-setup deploy ate-system --credential-provider='{"name":"k8s.io"}' ``` `--credential-provider` is required. `{"name":"k8s.io"}` also deploys the @@ -193,7 +193,7 @@ curl -X POST \ 5. You can then deploy the sample applications. See [demos/counter/README.md](demos/counter/README.md) or [demos/sandbox/README.md](demos/sandbox/README.md) for detailed walkthroughs. ```bash - ./hack/install-ate.sh --deploy-demo-counter + go run ./cmd/ate-setup deploy demo demo-counter ``` #### Custom Setup and Deployment @@ -207,13 +207,14 @@ go run ./tools/setup-gcp create bucket To run the PostgreSQL store backend on Cloud SQL — with IAM database authentication and no passwords — see [tools/setup-gcp/cloud-sql.md](tools/setup-gcp/cloud-sql.md). -Similarly, you can deploy or cleanup specific Agent Substrate components using the installation script. See `./hack/install-ate.sh --help` for all options. +Similarly, you can deploy or cleanup specific Agent Substrate components using +the installation script. See `go run ./cmd/ate-setup` for all options. ```bash # Re-deploy only ate-apiserver of the ATE system -./hack/install-ate.sh --deploy-ate-apiserver +go run ./cmd/ate-setup deploy apiserver # Delete everything (core system and all demos) -./hack/install-ate.sh --delete-all +go run ./cmd/ate-setup delete all ``` #### Tearing down resources (GCP) @@ -243,7 +244,6 @@ We provide several sample applications demonstrating Agent Substrate's capabilit 3. **[Claude Code Multiplex](demos/claude-code-multiplex/README.md)**: Demonstrates oversubscribing physical hardware by multiplexing multiple Claude Code agents onto a limited pool of workers. 4. **[Multi-Template](demos/multi-template/README.md)**: Two `ActorTemplate`s running different binaries share one `WorkerPool`, even though the templates live in different atespaces. 5. **[Request Parking](demos/parking/README.md)**: An oversubscribed pool where the router holds inbound requests until a worker frees up, instead of returning `503`. -6. **[Autoscaled WorkerPool](demos/autoscaled-workerpool/README.md)**: Scales a `WorkerPool` on its assigned-worker count with an HPA fed by prometheus-adapter. ### Documentation & Guides * [Architecture](docs/architecture.md): How the control plane, node supervisor, and networking stack fit together. @@ -258,6 +258,7 @@ We provide several sample applications demonstrating Agent Substrate's capabilit * [Request Parking](docs/request-parking.md): How the router parks requests through transient worker-pool saturation. * [Sandbox Asset Cache](docs/sandbox-asset-cache.md): How a node caches the sandbox binaries actors run on, decides which runtimes it needs, and evicts to a budget. * [Upgrade Runbook](docs/upgrade.md): Upgrade a running install to a newer supported release, and roll back if needed. +* [Node Storage](docs/node-storage.md): How to give snapshot I/O its own disk. * [Security Guide](docs/security.md): Security overview, security principles, and threat modeling. * [Roadmap](docs/roadmap.md): Current limitations and what is planned next. * [Benchmarking Guide](benchmarking/README.md): Locust-based load tests, monitoring stack, and the orchestrated benchmark harness. diff --git a/benchmarking/README.md b/benchmarking/README.md index 58a6679a76..14be6d41c3 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -23,14 +23,16 @@ scenario ladder, read [observability.md](observability.md). > [!IMPORTANT] > The ate-api-server checks every call against > [authorization](../docs/authorization.md), and Locust calls it as the -> `benchmarking/default` ServiceAccount. Install Substrate with that identity -> as a bootstrap owner, alongside the default `ate-client` owner, or every -> benchmark call is denied: +> `benchmarking/default` ServiceAccount. The Envoy ingress router also requires +> an allowlist of client SPIFFE IDs at install time. Install Substrate with +> `benchmarking/default` in both settings alongside the default `ate-client` +> owner: > ```bash > ATE_API_AUTHZ_BOOTSTRAP_OWNERS=system:serviceaccount:ate-system:ate-client,spiffe://cluster.local/ns/benchmarking/sa/default \ -> ./hack/install-ate.sh --deploy-ate-system +> ATE_INGRESS_ALLOWED_SPIFFE_IDS=spiffe://cluster.local/ns/benchmarking/sa/default \ +> ./hack/install-ate.sh deploy ate-system > ``` -> The nightly automation sets this itself; see +> The nightly automation sets both itself; see > [automation/README.md](automation/README.md). Note that deploying the benchmarks does not run them. You must visit Locust's @@ -62,8 +64,8 @@ The same operations are also reachable from the top-level installer for convenience: ```bash -./hack/install-ate.sh --deploy-benchmarks -./hack/install-ate.sh --delete-benchmarks +go run ./cmd/ate-setup deploy benchmarks +go run ./cmd/ate-setup delete benchmarks ``` `--skip-build` and `--worker-memory` are only available when invoking @@ -74,8 +76,6 @@ convenience: `ate-setup` reads these settings in addition to those in [docs/operator-install.md](../docs/operator-install.md#settings), through the same flag, environment variable, and configuration document channels. -`hack/install-ate.sh` forwards them as `--benchmark-worker-count`, -`--benchmark-sandbox-class`, and `--benchmark-actor-memory`. | Key | Environment variable | Flag | Default | Applies to | |--------------------------|-------------------------------|----------------------------|----------|------------------------------------------| @@ -154,7 +154,7 @@ and state restoration latency when a durable directory is attached to the actor. * `glutton-storage`: Attaches an external CSI volume instead of a durable directory. `deploy_locust.sh` and `workloads/deploy.sh` deploy it only when `--storage-class-name` names an existing StorageClass. To run it on - NFS, install with `hack/install-ate.sh --setup-csi=nfs` and pass + NFS, install with `hack/install-ate.sh setup csi nfs` and pass `--storage-class-name csi-nfs-sc`, the class the e2e tests also use. #### DurDir Reported Metrics @@ -373,9 +373,10 @@ deployment fails loudly instead of showing up as OOM-flaky steps. gap gets ±20% jitter so sessions don't move in lockstep. * `--resume-mode implicit|explicit` — implicit (default) lets the parked first request wake the actor; explicit issues ResumeActor before traffic. -* `--lifecycle-mode suspend|pause|none` — durable suspend (default), - node-local pause between steps, or `none` to keep actors resident in memory - between steps without hibernating. +* `--lifecycle-mode suspend|pause|none` — synchronous suspend (`suspend`, + default, uploads the snapshot before returning), asynchronous suspend + (`pause`, uploads the snapshot in the background), or `none` to keep actors + resident in memory between steps without hibernating. * `--agentsession-snapshot-steps` — comma-separated 1-based step numbers after which the actor is hibernated (for example, `3,10,20,30,40,50,61`); between all other steps the actor stays resident. @@ -541,26 +542,26 @@ count users × `--actors-per-user`, not `--total-actors`. ### ResumePaused Benchmark The ResumePaused benchmark drives ate-api-server's cold resume path at a set -rate. A cold resume here is a ResumeActor of a paused actor: ateapi takes the +rate. A cold resume here is a ResumeActor of a suspended actor: ateapi takes the actor lease, binds a worker, and restores the actor from its snapshot. A ResumeActor of a running actor does none of that. Each user owns one actor from the `glutton` template. On its first iteration the user creates the actor -and makes it cold with a resume and a pause; every later iteration resumes the -actor and pauses it again, so every measured resume is cold. +and makes it cold with a resume and a suspend; every later iteration resumes the +actor and suspends it again, so every measured resume is cold. The warm-up runs in the step that added the user, unpaced, and is recorded under its own names. It sends no requests to the actors themselves, so it runs on the fake data plane, where it measures what a cold resume costs ate-api-server and Postgres -without the router or the restore in the way. The pauses run at the resume -rate, so a capacity it finds is one of resume and pause cycles, not of resumes +without the router or the restore in the way. The suspends run at the resume +rate, so a capacity it finds is one of resume and suspend cycles, not of resumes alone. Actors are named `rp-` and deleted when boomer exits. If the atespace cannot be ensured, boomer exits at startup. An actor that fails to warm is deleted, and its user creates another on its next iteration. When a step lowers -the user count, the actors of the stopped users stay paused, unused, until +the user count, the actors of the stopped users stay suspended, unused, until boomer exits. #### ResumePaused Configuration Knobs @@ -568,7 +569,7 @@ boomer exits. * `--target-rps`: Cold resumes per second the whole boomer process paces to (default `0`, unpaced). A `--ladder` step ending in `@target_rps`, as in `200:3m@1000`, changes it as the step starts. Keep the users at or above the - rate times the p99 of one resume plus one pause in seconds, or the step falls + rate times the p99 of one resume plus one suspend in seconds, or the step falls short of its rate. The web UI shows the same field. @@ -590,6 +591,36 @@ warm-up until they have all joined at the spawn rate. * `CreateActor`, `ResumeActorWarmup`, `SuspendActorAsyncWarmup`: A user's warm-up of its actor, in the step that added the user. +#### Analyzing a ResumePaused Ladder Run + +`stats.csv` and the other rows of `stats.jsonl` cover a whole run, so they +blend every step of a ladder. After a ladder run the runner adds one +`stats.jsonl` row per step, with `metric` `ladder_step_001`, +`ladder_step_002`, and so on ([`locust/ladder_steps.py`](locust/ladder_steps.py)). +It reads each step's start and end from the run's log and queries the +benchmarking Prometheus over the step's steady window, from 30 s after the +step starts to 5 s before it ends (`window_start` and `window_end`). A row +holds the cold resume and pause rates, failures by operation +(`failures_`), client and server latency, the CPU and memory of +ate-api-server, Postgres and fake-atelet, and Postgres's connections, +commits and waits. + +The runner logs and skips a step with no end in the log and a step too short +to leave a window after the trim. A failed query adds no step rows at all +rather than a ladder with gaps, and leaves the run's other results as they +are. Read a step's figures against its `target_rps`: a `resume_rps` below it +means the step fell short of its rate. + +To pull the step rows out of a run's results: + +```bash +RUN=/runs//run_date=/run_ts=/run_tag= +gcloud storage cat "$RUN/stats.jsonl" | grep '"metric": "ladder_step_' +``` + +`` is the orchestrator's `--dest`, and its log has the full `RUN` path +on each `Uploaded ... -> gs://...` line. + ### Isolate Mode A real cold resume is limited by the node's restore capacity, which hides the @@ -597,7 +628,7 @@ limits of ate-api-server and Postgres. Isolate mode takes the nodes out by installing substrate with a fake data plane: ```bash -hack/install-ate.sh --deploy-ate-system --fake-data-plane +hack/install-ate.sh deploy ate-system --fake-data-plane ./benchmarking/workloads/deploy.sh --deploy --worker-count 200 \ --fake-atelet-env FAKE_ATELET_DELAY=500ms ``` @@ -636,8 +667,8 @@ Fake actors need neither. fake-atelet takes the storage settings of atelet's objectstore-plugin sidecar, whose writes it takes over, so its snapshot placeholders go to the storage the install points atelet at. The install refuses a cluster whose WorkerPools still have worker pods, which would -outlive ate-controller. A normal `hack/install-ate.sh --deploy-ate-system` -puts the real data plane back, and `--delete-ate-system` removes the fakes +outlive ate-controller. A normal `hack/install-ate.sh deploy ate-system` +puts the real data plane back, and `delete ate-system` removes the fakes either way. Never use the mode on a cluster that serves real actors: actors placed on fake Workers do not exist. @@ -764,9 +795,9 @@ actually did, independent of what the load generator reported. * `snapshots.restore_p50_s` through `restore_p99_s`, `restore_mean_s`, and the same for `checkpoint_*`: how long the atelet took to resume or suspend an actor (`ate_actor_{restore,checkpoint}_duration`, `total` phase). Every - snapshot kind counts, as in the sizes: the actor's own snapshot, a - template's first start from its golden snapshot, and a pause. The buckets - reach 60s, so a slower operation reads as 60s. + snapshot kind counts, as in the sizes: the actor's own snapshot and a + template's first start from its golden snapshot. The buckets reach 60s, so a + slower operation reads as 60s. * `snapshots.checkpoints_in_window`, `checkpoints_cumulative`: checkpoint volume over the steady-state window, and since the atelet started. * `snapshots.checkpoint_mb_s`: bytes written per second spent checkpointing, diff --git a/benchmarking/analysis/README.md b/benchmarking/analysis/README.md index 3109de03fb..b2a6735bec 100644 --- a/benchmarking/analysis/README.md +++ b/benchmarking/analysis/README.md @@ -85,7 +85,7 @@ restore between `volume_mount`, `manifest_fetch`, `sandbox_assets`, `prep` / `pause` / `snapshot` / `durable_dir` / `rootfs_upper` / `teardown` on a checkpoint and `prep` / … / `vm_restore` / `wakeup_probe` on a restore; for gVisor `pause` / `checkpoint` / `fs_checkpoint` / `resume` / `teardown` and -`prep` / `net_setup` / … / `pause_restore` / `app_restore` / +`prep` / `net_setup` / … / `sandbox_restore` / `app_restore` / `wakeup_probe` / `activate` (the full lists are in `cmd/ateworker-*/phaselog.go`). The ateworker records carry no sandbox class or kind of their own; a paired one takes both diff --git a/benchmarking/analysis/phase_report.py b/benchmarking/analysis/phase_report.py index 2d7c154f40..edbe814b8c 100755 --- a/benchmarking/analysis/phase_report.py +++ b/benchmarking/analysis/phase_report.py @@ -94,7 +94,7 @@ # ateworker restore: micro-VM (cmd/ateworker-microvm/phaselog.go) and gVisor # (cmd/ateworker-gvisor/phaselog.go), each in its own sequential order "prep", "egress_prepare", "net_setup", "bundles", "upper_join", "lowers", - "pause_rootfs", "pause_create", "pause_restore", + "sandbox_create", "sandbox_restore", "app_rootfs", "app_create", "app_restore", "tap", "vmm_launch", "vm_restore", "resume", "wakeup_probe", "activate", # atelet + ateworker checkpoint diff --git a/benchmarking/atunnel/README.md b/benchmarking/atunnel/README.md index 2bc097d1f4..d1defc54ac 100644 --- a/benchmarking/atunnel/README.md +++ b/benchmarking/atunnel/README.md @@ -91,7 +91,7 @@ atunnel's own atelet client. |---|---| | atelet | `worker` serves the two atelet calls atunnel makes, certificate minting and egress policy | | the actor's program receiving requests | an ingress actor: a small HTTP server in each sandbox | -| the actor's program making outgoing connections | the egress actor, in actor-0's sandbox | +| the actor's program making outgoing connections | an egress actor in each sandbox; EgressIdleConns splits its connections across them, the other egress scenarios use actor-0's | | the router | the generator, presenting the router's identity | | the egress gateway | `sink`, with the real gateway's Envoy HTTP/2 settings; it never dials out | | Pod certificates | files from a throwaway CA | diff --git a/benchmarking/atunnel/cmd/atunnel-bench/tunnel.go b/benchmarking/atunnel/cmd/atunnel-bench/tunnel.go index 4dd997446a..cbc4bbea24 100644 --- a/benchmarking/atunnel/cmd/atunnel-bench/tunnel.go +++ b/benchmarking/atunnel/cmd/atunnel-bench/tunnel.go @@ -29,7 +29,7 @@ import ( "go.ate.dev/substrate/benchmarking/atunnel/internal/procstats" "go.ate.dev/substrate/benchmarking/atunnel/internal/workerhost" - "go.ate.dev/substrate/internal/ateworker/tunnel" + "go.ate.dev/substrate/internal/atunnel" ) // tunnelCommand is process B of the benchmark: atunnel started and hosting @@ -40,7 +40,7 @@ import ( func tunnelCommand() *cobra.Command { var actors int var gateway, adminListen, logLevel string - var cfg *tunnel.Config + var cfg *atunnel.Config cmd := &cobra.Command{ Use: "tunnel", Short: "Run atunnel, hosting actors with the worker's own code", @@ -56,7 +56,7 @@ func tunnelCommand() *cobra.Command { return runTunnel(cmd.Context(), *cfg, actors, gateway, adminListen) }, } - cfg = tunnel.RegisterFlags(cmd.Flags()) + cfg = atunnel.RegisterFlags(cmd.Flags()) cmd.Flags().IntVar(&actors, "actors", 10, "actors to host") cmd.Flags().StringVar(&gateway, "gateway", "atunnel-sink:9443", "egress gateway every actor tunnels to, host:port") cmd.Flags().StringVar(&adminListen, "admin", ":9090", "address for /stats, /healthz, and /debug/pprof") @@ -64,7 +64,7 @@ func tunnelCommand() *cobra.Command { return cmd } -func runTunnel(ctx context.Context, cfg tunnel.Config, actors int, gateway, adminListen string) error { +func runTunnel(ctx context.Context, cfg atunnel.Config, actors int, gateway, adminListen string) error { host, err := workerhost.Start(ctx, cfg, actors, gateway) if err != nil { return err diff --git a/benchmarking/atunnel/cmd/atunnel-bench/worker.go b/benchmarking/atunnel/cmd/atunnel-bench/worker.go index 36f391f579..450024f822 100644 --- a/benchmarking/atunnel/cmd/atunnel-bench/worker.go +++ b/benchmarking/atunnel/cmd/atunnel-bench/worker.go @@ -208,11 +208,12 @@ func (s *supervisor) startSandboxProcesses(info workerhost.Info) ([]*exec.Cmd, e if err := start(sb.NetNS, "ingress-actor", "--port", strconv.Itoa(info.ActorPort), "--body-bytes", strconv.Itoa(s.bodyBytes), "--stats-socket", socket); err != nil { return cmds, err } + socket = filepath.Join(s.shared, "egress-actor-"+strconv.Itoa(i)+".sock") + if err := start(sb.NetNS, "egress-actor", "--socket", socket); err != nil { + return cmds, err + } } - if err := start(info.Sandboxes[0].NetNS, "egress-actor", "--socket", filepath.Join(s.shared, "egress-actor.sock")); err != nil { - return cmds, err - } - fmt.Fprintf(os.Stderr, "worker: %d sandboxes with ingress actors, egress actor in %s\n", len(info.Sandboxes), info.Sandboxes[0].Actor) + fmt.Fprintf(os.Stderr, "worker: %d sandboxes, each with an ingress actor and an egress actor\n", len(info.Sandboxes)) return cmds, nil } diff --git a/benchmarking/atunnel/internal/load/egress_actor.go b/benchmarking/atunnel/internal/load/egress_actor.go index 7b75d9445a..d8ddadd4d7 100644 --- a/benchmarking/atunnel/internal/load/egress_actor.go +++ b/benchmarking/atunnel/internal/load/egress_actor.go @@ -23,6 +23,8 @@ import ( "fmt" "net" "net/http" + "sync" + "sync/atomic" "time" "go.ate.dev/substrate/benchmarking/atunnel/internal/procstats" @@ -159,3 +161,65 @@ func executeOnEgressActor(ctx context.Context, socket string, opts options) (rep } return report.Load{}, fmt.Errorf("egress actor: no result") } + +// executeOnEgressActors splits an EgressIdle run evenly across the egress +// actors at sockets, one per sandbox. A sandbox has one address, so it can hold +// at most ~64k egress connections at a time: atunnel's redirect makes every one +// of them look the same but for the source port. whileHolding runs once, after +// every actor holds its connections. +func executeOnEgressActors(ctx context.Context, sockets []string, opts options) (report.Load, error) { + n := len(sockets) + var waiting atomic.Int64 + waiting.Store(int64(n)) + loads := make([]report.Load, n) + errs := make([]error, n) + var wg sync.WaitGroup + for i, socket := range sockets { + sub := opts + sub.rate = opts.rate / float64(n) + sub.connections = opts.connections / n + if i < opts.connections%n { + sub.connections++ + } + sub.whileHolding = func(ctx context.Context) { + if waiting.Add(-1) == 0 && opts.whileHolding != nil { + opts.whileHolding(ctx) + } + } + wg.Go(func() { loads[i], errs[i] = executeOnEgressActor(ctx, socket, sub) }) + } + wg.Wait() + if err := errors.Join(errs...); err != nil { + return report.Load{}, err + } + return mergeLoads(loads), nil +} + +// mergeLoads adds up loads that ran side by side. Latency percentiles are the +// worst of them. +func mergeLoads(loads []report.Load) report.Load { + var m report.Load + worst := func(a, b report.Percentiles) report.Percentiles { + return report.Percentiles{P50: max(a.P50, b.P50), P90: max(a.P90, b.P90), P99: max(a.P99, b.P99), Max: max(a.Max, b.Max), Mean: max(a.Mean, b.Mean)} + } + for _, l := range loads { + m.Offered += l.Offered + m.MeasuredSeconds = max(m.MeasuredSeconds, l.MeasuredSeconds) + m.Scheduled += l.Scheduled + m.Skipped += l.Skipped + m.Completed += l.Completed + m.Errors += l.Errors + m.Bytes += l.Bytes + m.Latency = worst(m.Latency, l.Latency) + m.SendLag = worst(m.SendLag, l.SendLag) + m.ErrorSamples = append(m.ErrorSamples, l.ErrorSamples...) + } + if len(m.ErrorSamples) > maxErrorSamples { + m.ErrorSamples = m.ErrorSamples[:maxErrorSamples] + } + if m.MeasuredSeconds > 0 { + m.Achieved = float64(m.Completed) / m.MeasuredSeconds + m.MBytesPerSec = float64(m.Bytes) / m.MeasuredSeconds / 1e6 + } + return m +} diff --git a/benchmarking/atunnel/internal/load/ladder.go b/benchmarking/atunnel/internal/load/ladder.go index 9e492353a3..edd40369e8 100644 --- a/benchmarking/atunnel/internal/load/ladder.go +++ b/benchmarking/atunnel/internal/load/ladder.go @@ -103,15 +103,15 @@ func (r *Runner) Prepare(ctx context.Context) error { return nil } -// waitForSandboxSockets waits until the egress actor and every ingress actor +// waitForSandboxSockets waits until every sandbox's ingress and egress actors // answer on their sockets in the shared directory. -func (r *Runner) waitForSandboxSockets(ctx context.Context, ingressActors int) error { +func (r *Runner) waitForSandboxSockets(ctx context.Context, sandboxes int) error { if r.SharedDir == "" { return nil } - sockets := []string{filepath.Join(r.SharedDir, "egress-actor.sock")} - for i := range ingressActors { - sockets = append(sockets, filepath.Join(r.SharedDir, "ingress-actor-"+strconv.Itoa(i)+".sock")) + var sockets []string + for i := range sandboxes { + sockets = append(sockets, filepath.Join(r.SharedDir, "ingress-actor-"+strconv.Itoa(i)+".sock"), r.egressActorSocket(i)) } for _, socket := range sockets { for { @@ -246,7 +246,7 @@ func (r *Runner) rung(ctx context.Context, s Scenario, target float64) (report.R generator := func(context.Context) (procstats.Snapshot, error) { return procstats.Read(), nil } if isEgress(opts.mode) { // Egress is generated by the egress actor in the sandbox. - generator = r.processStats(unixAdmin(filepath.Join(r.SharedDir, "egress-actor.sock"))) + generator = r.processStats(unixAdmin(r.egressActorSocket(0))) } selfWatch := startWatch(ctx, generator, sampleInterval, nil) @@ -332,11 +332,23 @@ func (r *Runner) execute(ctx context.Context, opts options) (report.Load, error) if r.SharedDir == "" { return report.Load{}, fmt.Errorf("%s needs the egress actor in the shared directory", opts.mode) } - return executeOnEgressActor(ctx, filepath.Join(r.SharedDir, "egress-actor.sock"), opts) + if opts.mode == EgressIdle && len(r.Targets.Actors) > 1 { + sockets := make([]string, len(r.Targets.Actors)) + for i := range sockets { + sockets[i] = r.egressActorSocket(i) + } + return executeOnEgressActors(ctx, sockets, opts) + } + return executeOnEgressActor(ctx, r.egressActorSocket(0), opts) } return execute(ctx, r.Targets, opts) } +// egressActorSocket is the socket of the egress actor in sandbox i. +func (r *Runner) egressActorSocket(i int) string { + return filepath.Join(r.SharedDir, "egress-actor-"+strconv.Itoa(i)+".sock") +} + func (r *Runner) processStats(a *admin) func(context.Context) (procstats.Snapshot, error) { return func(ctx context.Context) (procstats.Snapshot, error) { st, err := a.stats(ctx) diff --git a/benchmarking/atunnel/internal/load/load_test.go b/benchmarking/atunnel/internal/load/load_test.go index 9a5ecdf337..455cdc805a 100644 --- a/benchmarking/atunnel/internal/load/load_test.go +++ b/benchmarking/atunnel/internal/load/load_test.go @@ -16,6 +16,7 @@ package load import ( "context" + "fmt" "net" "net/http" "path/filepath" @@ -123,6 +124,40 @@ func TestEgressActor(t *testing.T) { } } +// TestEgressActors splits an EgressIdle run across several egress actors and +// adds up what they report. +func TestEgressActors(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + target := listen(t) + go func() { + _ = http.Serve(target, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("ok")) })) + }() + + var sockets []string + for i := range 3 { + socket := filepath.Join(t.TempDir(), fmt.Sprintf("egress-actor-%d.sock", i)) + lis, err := net.Listen("unix", socket) + if err != nil { + t.Fatal(err) + } + go func() { _ = serveEgressActor(ctx, lis, Targets{Egress: target.Addr().String()}) }() + sockets = append(sockets, socket) + } + + var held atomic.Int32 + l, err := executeOnEgressActors(ctx, sockets, options{ + mode: EgressIdle, rate: 300, duration: 500 * time.Millisecond, workers: 4, connections: 10, + whileHolding: func(context.Context) { held.Add(1) }, + }) + if err != nil { + t.Fatal(err) + } + if held.Load() != 1 || l.Completed != 10 || l.Errors != 0 { + t.Fatalf("EgressIdle across 3 egress actors: whileHolding ran %d times, completed %d, errors %d; want 1, 10, 0", held.Load(), l.Completed, l.Errors) + } +} + func listen(t *testing.T) net.Listener { t.Helper() lis, err := net.Listen("tcp", "127.0.0.1:0") diff --git a/benchmarking/atunnel/internal/workerhost/workerhost.go b/benchmarking/atunnel/internal/workerhost/workerhost.go index fc82d9974d..ff7bc107f7 100644 --- a/benchmarking/atunnel/internal/workerhost/workerhost.go +++ b/benchmarking/atunnel/internal/workerhost/workerhost.go @@ -15,7 +15,7 @@ //go:build linux // Package workerhost hosts atunnel the way an ateworker does, with the -// worker's own code: tunnel.Start once, then per actor the calls +// worker's own code: atunnel.Start once, then per actor the calls // cmd/ateworker-gvisor makes to run one, in its order. Only the sandbox itself // is missing: each actor's network namespaces, veth, nftables redirect, egress // listener, and DNS relay are real, but no gVisor runs in them. @@ -30,7 +30,7 @@ import ( "sync" "go.ate.dev/substrate/internal/ateworker/actornet" - "go.ate.dev/substrate/internal/ateworker/tunnel" + "go.ate.dev/substrate/internal/atunnel" "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/internal/resources" @@ -62,7 +62,7 @@ type Info struct { // Host is a running atunnel with its actors. type Host struct { - tunnel *tunnel.Tunnel + tunnel *atunnel.Tunnel mu sync.Mutex actors []*actor @@ -75,8 +75,8 @@ type actor struct { // Start starts atunnel from cfg, as a worker's main does, and hosts actors // actors whose egress tunnels to gatewayAddress. -func Start(ctx context.Context, cfg tunnel.Config, actors int, gatewayAddress string) (*Host, error) { - t, err := tunnel.Start(ctx, cfg, actornet.ActorHTTPUpstream) +func Start(ctx context.Context, cfg atunnel.Config, actors int, gatewayAddress string) (*Host, error) { + t, err := atunnel.Start(ctx, cfg, actornet.ActorHTTPUpstream) if err != nil { return nil, err } diff --git a/benchmarking/atunnel/internal/workerhost/workerhost_linux_test.go b/benchmarking/atunnel/internal/workerhost/workerhost_linux_test.go index 5cfb9caa43..1885d159f0 100644 --- a/benchmarking/atunnel/internal/workerhost/workerhost_linux_test.go +++ b/benchmarking/atunnel/internal/workerhost/workerhost_linux_test.go @@ -30,7 +30,7 @@ import ( "go.ate.dev/substrate/benchmarking/atunnel/internal/pki" "go.ate.dev/substrate/benchmarking/atunnel/internal/sink" "go.ate.dev/substrate/internal/atenet" - "go.ate.dev/substrate/internal/ateworker/tunnel" + "go.ate.dev/substrate/internal/atunnel" "go.ate.dev/substrate/internal/credbundle" "go.ate.dev/substrate/internal/egresspolicy" "go.ate.dev/substrate/internal/installdefaults" @@ -73,7 +73,7 @@ func TestHostsActorsWithTheWorkersCode(t *testing.T) { go func() { _ = gateway.Serve(ctx, gatewayLis) }() ingressAddr := freeAddress(t) - host, err := Start(ctx, tunnel.Config{ + host, err := Start(ctx, atunnel.Config{ ListenAddress: ingressAddr, ConnectListenAddress: freeAddress(t), CredentialBundle: filepath.Join(dir, pki.WorkerBundle), @@ -180,7 +180,7 @@ func listen(t *testing.T) net.Listener { } // freeAddress returns a loopback address with a port free a moment ago, for -// listeners tunnel.Start binds itself. +// listeners atunnel.Start binds itself. func freeAddress(t *testing.T) string { t.Helper() lis, err := net.Listen("tcp", "127.0.0.1:0") diff --git a/benchmarking/atunnel/runs/20261020-163000/report.md b/benchmarking/atunnel/runs/20261020-163000/report.md new file mode 100644 index 0000000000..c7880e836f --- /dev/null +++ b/benchmarking/atunnel/runs/20261020-163000/report.md @@ -0,0 +1,54 @@ +# atunnel benchmark: 100,000 idle egress connections + +Run: `20261020-163000`, EgressIdleConns only, with the idle-buffer change +(`streamCopy`) in atunnel. Raw numbers are in `results.txt`. + +## Setup + +- Docker Desktop on an Apple M-series Mac, with the VM given 10 CPUs and + 24 GB of memory. +- atunnel limited to 4 CPUs and 16 GiB without swap, far more than it used, + so neither limit shapes the result. +- `run-docker.sh --tunnel-cpus 4 --tunnel-memory 16g -- --scenarios + EgressIdleConns --rungs 25000,50000,100000` +- Connections open at 200/s and are split across the egress actors in all 10 + sandboxes. Each completes one request and is then held idle. + +## Results + +| Idle connections | Per connection, after GC | Goroutines per connection | atunnel heap, after GC | atunnel cgroup memory | atunnel CPU opening 200 conns/s | Errors | +|---|---|---|---|---|---|---| +| 25,000 | 26.9 KiB | 4.0 | 658 MiB | 1.3 GiB | 0.15 cores | 0 | +| 50,000 | 27.1 KiB | 4.0 | 1.3 GiB | 2.6 GiB | 0.14 cores | 0 | +| 100,000 | 26.8 KiB | 4.0 | 2.6 GiB | 5.2 GiB | 0.19 cores | 0 | + +- Memory per connection stays flat at about 27 KiB, so it grows linearly with + the number of connections. Before the change it was about 157 KiB, which + would be about 15 GiB of heap at 100,000. +- The cgroup memory also counts the kernel's socket buffers, about 53 KiB per + connection in total. +- The CPU column is what opening new connections costs, about 1 ms each, so + 200/s takes about 0.2 cores. Holding the idle connections costs almost none. +- The 25k and 50k rungs show SHORTFALL because some opens waited up to ~11 ms + in the generator's queue. Every connection still opened. + +## Nothing else was the limit + +Sampled every 30 seconds with `docker stats` during the run: + +| | Peak | +|---|---| +| atunnel | 0.19 of 4 cores, 5.2 of 16 GiB, no restarts | +| worker container (atunnel and the 20 actor processes) | 1.2 cores, 6.1 GiB | +| fake gateway | 0.13 cores, 1.6 GiB | +| generator container | 0.12 cores, 0.5 GiB | +| Docker VM | 8.5 of 24 GB used, swap unused | + +## One sandbox holds at most ~64k egress connections + +With every connection coming from one sandbox, the run stopped at 64,512 +connections and later ones timed out. A sandbox has one address, and atunnel's +redirect sends every connection to the same listener, so only the source port +tells the connections apart: 1024–65535 is 64,512 ports. This is a limit per +actor, not per worker. It is why the harness now runs an egress actor in every +sandbox. diff --git a/benchmarking/atunnel/runs/20261020-163000/results.txt b/benchmarking/atunnel/runs/20261020-163000/results.txt new file mode 100644 index 0000000000..20454daef2 --- /dev/null +++ b/benchmarking/atunnel/runs/20261020-163000/results.txt @@ -0,0 +1,12 @@ +=== EgressIdleConns: rungs are connection counts: open at 200/s through egress, one request each, then hold + unit: held connection; 10s per rung, first 0s not measured + offered achieved p50 p99 max errs | cores µs/unit rss-max heap-max gor cg-cores cg-mem | A cores C cores verdict + 25000 25000 5.87ms 14.54ms 39.01ms 0 | 0.15 806 1215.8MiB 663.7MiB 100345 0.15 1309.9MiB | 0.01 0.09 SHORTFALL (units waited for a worker, p99 10.4ms) + holding 25000: live heap 657.5MiB after GC, 26.9 KiB and 4.0 goroutines per connection + 50000 50000 5.44ms 16.11ms 195ms 0 | 0.14 733 2401.5MiB 1331.0MiB 200595 0.14 2589.9MiB | 0.01 0.08 SHORTFALL (units waited for a worker, p99 11.6ms) + holding 50000: live heap 1325.0MiB after GC, 27.1 KiB and 4.0 goroutines per connection + 100000 100000 5.04ms 15.17ms 235ms 0 | 0.19 975 4733.7MiB 2627.3MiB 401095 0.19 5192.6MiB | 0.01 0.11 OK + holding 100000: live heap 2619.1MiB after GC, 26.8 KiB and 4.0 goroutines per connection + cores..cg-mem are atunnel alone (process B); A and C are the traffic generator and the far end + sink counters: {"gateway_bytes_in":6825000,"gateway_connections":999,"gateway_rejected":0,"gateway_requests":175000,"gateway_tunnels":175000,"gateway_tunnels_h2":175000,"ingress_actor_connections":0,"ingress_actor_requests":0} + diff --git a/benchmarking/automation/Dockerfile b/benchmarking/automation/Dockerfile index 45330795e0..6a77157a72 100644 --- a/benchmarking/automation/Dockerfile +++ b/benchmarking/automation/Dockerfile @@ -39,7 +39,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # Go must be new enough to honor the `tool` directive in # hack/tools/*/go.mod (Go 1.24+). Pin a version known to match the # substrate project's pinned ko (see hack/tools/ko/go.mod). -ARG GO_VERSION=1.26.3 +ARG GO_VERSION=1.27.3 RUN curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" \ | tar -C /usr/local -xz ENV PATH=/usr/local/go/bin:$PATH @@ -62,7 +62,7 @@ RUN python3 -m venv /opt/automation/venv \ && /opt/automation/venv/bin/pip install --no-cache-dir -r /opt/automation/requirements.txt ENV PATH=/opt/automation/venv/bin:$PATH -COPY orchestrator.py util.py /opt/automation/ +COPY orchestrator.py util.py controlplane.py /opt/automation/ COPY testtypes/ /opt/automation/testtypes/ COPY manifests/runner-job.yaml.tmpl manifests/nighthawk-ingress-runner-job.yaml.tmpl /opt/automation/manifests/ diff --git a/benchmarking/automation/README.md b/benchmarking/automation/README.md index 3849e6ce53..28515298a4 100644 --- a/benchmarking/automation/README.md +++ b/benchmarking/automation/README.md @@ -18,14 +18,16 @@ the router capacity benchmark — see 4. `docker build && docker push` builds the runner image for each test type in use, tagged with the commit hash: `${KO_DOCKER_REPO}/locust-test:` and/or `${KO_DOCKER_REPO}/nighthawk-ingress-test:`. -5. `hack/install-ate.sh --deploy-ate-system` (with `ATE_CREDENTIAL_PROVIDER` +5. `hack/install-ate.sh deploy ate-system` (with `ATE_CREDENTIAL_PROVIDER` in `.ate-dev-env.sh`, or `--credential-provider` in the test's `ateArgs`) + `benchmarking/workloads/deploy.sh --deploy --sandbox-class ` (these build & push substrate / workload images via `ko` as part of their deploy steps — there's no separate image build step). The install sets `ATE_API_AUTHZ_BOOTSTRAP_OWNERS` to the `ate-client` and `benchmarking/benchmark-runner` ServiceAccounts so the - runner's calls pass [authorization](../../docs/authorization.md). For a `microvm` test the orchestrator also + runner's calls pass [authorization](../../docs/authorization.md), and sets + `ATE_INGRESS_ALLOWED_SPIFFE_IDS` to `benchmarking/benchmark-runner` for the + Envoy ingress router. For a `microvm` test the orchestrator also runs `hack/install-microvm-deps.sh --install` between the two, which stages kata + cloud-hypervisor + virtiofsd assets to the cluster's object store bucket and applies the cluster-wide `microvm` SandboxConfig. @@ -54,9 +56,10 @@ Each entry in `tests.yaml` may set `sandboxClass: gvisor | microvm` (default `gvisor`). This controls both `spec.sandboxClasses[0].name` on the benchmark WorkerPool and its `workerImage` (`ateworker-gvisor` vs `ateworker-microvm`). -For `microvm` tests the target cluster must have KVM-capable nodes and the -object store bucket named in its `.ate-dev-env.sh` must be writable by the -orchestrator's Workload Identity principal. +For `microvm` tests the target cluster must have KVM-capable nodes. The +orchestrator uploads the micro-VM assets, so its Workload Identity principal +must be able to write to the bucket they go to. On GKE that is +`-assets`, where `BUCKET_NAME` comes from `.ate-dev-env.sh`. ## Setup @@ -144,7 +147,7 @@ principal://iam.googleapis.com/projects//locations/global/worklo - `roles/container.admin` on the test cluster's project — required to manage cluster-scoped resources (CRDs, ClusterRoles, ClusterRoleBindings, - Namespaces) that `hack/install-ate.sh --deploy-ate-system` creates. + Namespaces) that `hack/install-ate.sh deploy ate-system` creates. `container.developer` is not enough — it intentionally omits the `container.clusterRoles.*` and `container.customResourceDefinitions.*` permissions. diff --git a/benchmarking/automation/controlplane.py b/benchmarking/automation/controlplane.py new file mode 100644 index 0000000000..1b4003913d --- /dev/null +++ b/benchmarking/automation/controlplane.py @@ -0,0 +1,275 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shapes the control plane for a capacity run, as a test entry's +`controlPlane:` block asks: an `apiServer:` section for ate-api-server +(see API_SERVER_KNOBS) and a `postgres:` section (see POSTGRES_DEFAULTS). + +A test type validates the block with validate() and calls shape() from its +pre_test. Isolation from the load is the install's: --cordon-control-plane +in ateArgs keeps ate-api-server and Postgres on their own tainted pools. +""" + +import json +import re +from typing import Any + +from util import run, run_no_check + +NAMESPACE = "ate-system" + +# Knobs of the `apiServer:` section. None has a default here: the install +# defines ate-api-server's shape, and a run overrides only the knobs it sets. +API_SERVER_KNOBS = { + # Required, the independent variable of the run. CPU request and limit + # of each replica; Go sizes GOMAXPROCS from the limit. + "cpu", + # Memory request and limit of each replica. + "memory", + "replicas", + # pgxpool max connections per replica, set through ateapi's env + # ConfigMap (see set_pool_max_conns). + "poolMaxConns", +} + +# Knobs of the `postgres:` section. +POSTGRES_DEFAULTS = { + # Load pg_stat_statements and run postgres_exporter next to Postgres, + # for Prometheus (benchmarking/monitoring.yaml) to scrape. + "stats": True, + # Postgres settings as name=value, passed to the server as -c flags in + # the restart stats makes, over postgresql.conf. + "args": [], +} + + +def config(block: dict[str, Any]) -> dict[str, Any]: + """The `controlPlane:` block with `postgres:` merged over its defaults. + `apiServer:` holds only the knobs the block sets.""" + return { + "apiServer": dict(block.get("apiServer", {})), + "postgres": {**POSTGRES_DEFAULTS, **block.get("postgres", {})}, + } + + +def validate(block: Any) -> None: + """Raise ValueError unless block is a valid `controlPlane:` block.""" + if not isinstance(block, dict): + raise ValueError("controlPlane must be a mapping") + _check_knobs("controlPlane", block, {"apiServer", "postgres"}) + if "apiServer" not in block: + raise ValueError("needs controlPlane.apiServer") + _check_knobs("controlPlane.apiServer", block["apiServer"], API_SERVER_KNOBS) + _check_knobs("controlPlane.postgres", block.get("postgres", {}), set(POSTGRES_DEFAULTS)) + cfg = config(block) + api, pg = cfg["apiServer"], cfg["postgres"] + if "cpu" not in api or not _is_quantity(api["cpu"]): + raise ValueError("needs controlPlane.apiServer.cpu, as in 2 or 1500m") + if "memory" in api and not _is_quantity(api["memory"]): + raise ValueError("invalid controlPlane.apiServer.memory") + if "replicas" in api and not _is_positive_int(api["replicas"]): + raise ValueError("controlPlane.apiServer.replicas must be an int >= 1") + if "poolMaxConns" in api and not _is_positive_int(api["poolMaxConns"]): + raise ValueError("controlPlane.apiServer.poolMaxConns must be an int >= 1") + if not isinstance(pg["stats"], bool): + raise ValueError("controlPlane.postgres.stats must be a bool") + args = pg["args"] + if not isinstance(args, list) or not all( + isinstance(a, str) and POSTGRES_ARG_PATTERN.fullmatch(a) for a in args + ): + raise ValueError( + "controlPlane.postgres.args must be a list of name=value settings, " + "as in commit_delay=100" + ) + if args and not pg["stats"]: + raise ValueError( + "controlPlane.postgres.args needs stats: true, whose Postgres restart applies them" + ) + + +def _check_knobs(path: str, section: Any, allowed: set[str]) -> None: + if not isinstance(section, dict): + raise ValueError(f"{path} must be a mapping") + unknown = set(section) - allowed + if unknown: + raise ValueError( + f"unknown {path} knob(s) {sorted(unknown)}; allowed: {sorted(allowed)}" + ) + + +def _is_positive_int(v: Any) -> bool: + return isinstance(v, int) and not isinstance(v, bool) and v >= 1 + + +def _is_quantity(v: Any) -> bool: + """A Kubernetes quantity as tests.yaml spells it: 2, 0.5, "1500m", "4Gi".""" + if isinstance(v, bool): + return False + if isinstance(v, (int, float)): + return v > 0 + return isinstance(v, str) and bool(QUANTITY_PATTERN.fullmatch(v)) + + +# A Postgres setting as postgres.args spells it: a lowercase name, an equals +# sign, and a value with no whitespace, which the -c flag takes as is. +POSTGRES_ARG_PATTERN = re.compile(r"[a-z_]+=\S+") + + +# A non-negative Kubernetes quantity: a decimal number with a binary-SI +# suffix (Ki to Ei), a decimal-SI suffix (m to E), or a decimal exponent. +QUANTITY_PATTERN = re.compile( + r"([0-9]+(\.[0-9]*)?|\.[0-9]+)([KMGTPE]i|[mkMGTPE]|[eE][+-]?[0-9]+)?" +) + + +# TODO: guard against drift in the manifests these patches assume, so a +# change fails the run instead of quietly skewing its numbers. Tests that +# read manifests/ate-install and check the ate-api-server container, the +# Postgres base args postgres_args restates, the socket volume, and the env +# ConfigMap name and key; and a check after shape() that the running pods +# carry the requested resources and Postgres the requested settings. +def shape(cfg: dict[str, Any]) -> None: + """Shape the control plane for the run: Postgres statistics, the pool + size, then ate-api-server's resources, whose rollout picks up the pool + size. No unpatch: every test redeploys substrate.""" + api, pg = cfg["apiServer"], cfg["postgres"] + if pg["stats"]: + enable_postgres_stats(pg["args"]) + if "poolMaxConns" in api: + set_pool_max_conns(api["poolMaxConns"]) + resize_api_server(api) + # The placement of the run, for comparing runs: a step compares across + # runs only when ateapi and Postgres sat on the same nodes. + run_no_check( + ["kubectl", "-n", NAMESPACE, "get", "pods", "-o", "wide", + "-l", "app in (ate-api-server,postgres)"] + ) + + +POSTGRES_EXPORTER_IMAGE = "quay.io/prometheuscommunity/postgres-exporter:v0.17.1" + + +def postgres_args(settings: list[str]) -> list[str]: + """The postgres container's args: the base manifest's config_file, which + a strategic merge would otherwise drop, pg_stat_statements, then each of + settings as a -c flag, which overrides postgresql.conf.""" + args = [ + "-c", "config_file=/etc/postgresql/postgresql.conf", + "-c", "shared_preload_libraries=pg_stat_statements", + ] + for setting in settings: + args += ["-c", setting] + return args + + +def enable_postgres_stats(settings: list[str]) -> None: + """Load pg_stat_statements, apply settings, and add a postgres_exporter + sidecar, which reaches the server over the pod's unix socket (trusted by + pg_hba). One patch, thus one Postgres restart, before ateapi rolls.""" + patch = { + "spec": { + "template": { + "spec": { + "containers": [ + { + "name": "postgres", + # Strategic merge replaces args whole. + "args": postgres_args(settings), + }, + { + "name": "postgres-exporter", + "image": POSTGRES_EXPORTER_IMAGE, + "args": ["--collector.stat_statements"], + "env": [ + { + "name": "DATA_SOURCE_NAME", + "value": "host=/var/run/postgresql user=postgres dbname=atepg sslmode=disable", + } + ], + # ate-system admits scrapes only to named ports + # such as metrics. + "ports": [{"name": "metrics", "containerPort": 9187}], + "volumeMounts": [ + {"name": "socket", "mountPath": "/var/run/postgresql"} + ], + "resources": {"requests": {"cpu": "100m", "memory": "128Mi"}}, + }, + ] + } + } + } + } + _patch_and_wait("statefulset", "postgres", patch) + run( + ["kubectl", "-n", NAMESPACE, "exec", "postgres-0", "-c", "postgres", "--", + "psql", "-U", "postgres", "-d", "atepg", "-v", "ON_ERROR_STOP=1", + "-c", "CREATE EXTENSION IF NOT EXISTS pg_stat_statements"] + ) + + +# ate-setup's ConfigMap of ateapi environment variables. ateapi reads +# ATE_API_POSTGRES_POOL_MAX_CONNS from it at start, over any pool_max_conns in +# the DSN. +ENV_CONFIGMAP = "ate-api-server-envvars" +POOL_MAX_CONNS_KEY = "ATE_API_POSTGRES_POOL_MAX_CONNS" + + +def pool_max_conns_patch(n: int) -> dict[str, Any]: + return {"data": {POOL_MAX_CONNS_KEY: str(n)}} + + +def set_pool_max_conns(n: int) -> None: + """Set the read/write pool size ateapi reads at start; the + resize_api_server rollout that follows picks it up. The owner and watch + pools stay at 2 and 3 connections per replica.""" + run( + ["kubectl", "-n", NAMESPACE, "patch", "configmap", ENV_CONFIGMAP, + "--type", "merge", "-p", json.dumps(pool_max_conns_patch(n))] + ) + + +def api_server_patch(api: dict[str, Any]) -> dict[str, Any]: + """A strategic merge patch that sets only the knobs api holds, so the + rest of the Deployment keeps the install's values.""" + resources = {"cpu": str(api["cpu"])} + if "memory" in api: + resources["memory"] = api["memory"] + pod_spec = { + "containers": [ + { + "name": "ate-api-server", + "resources": {"requests": resources, "limits": dict(resources)}, + } + ] + } + spec: dict[str, Any] = {"template": {"spec": pod_spec}} + if "replicas" in api: + spec["replicas"] = api["replicas"] + return {"spec": spec} + + +def resize_api_server(api: dict[str, Any]) -> None: + _patch_and_wait("deployment", "ate-api-server", api_server_patch(api)) + + +def _patch_and_wait(kind: str, name: str, patch: dict[str, Any]) -> None: + run( + ["kubectl", "-n", NAMESPACE, "patch", kind, name, + "--type=strategic", "-p", json.dumps(patch)] + ) + run( + ["kubectl", "-n", NAMESPACE, "rollout", "status", f"{kind}/{name}", + "--timeout=600s"] + ) + diff --git a/benchmarking/automation/manifests/runner-job.yaml.tmpl b/benchmarking/automation/manifests/runner-job.yaml.tmpl index 5303dcb2fb..9b6f939894 100644 --- a/benchmarking/automation/manifests/runner-job.yaml.tmpl +++ b/benchmarking/automation/manifests/runner-job.yaml.tmpl @@ -175,6 +175,11 @@ spec: env: - name: OTEL_EXPORTER_OTLP_ENDPOINT value: http://opentelemetry-collector.gke-managed-otel.svc.cluster.local:4317 + # boomer's /metrics; the boomer-worker job of + # benchmarking/monitoring.yaml scrapes it. + ports: + - name: boomer-metrics + containerPort: 8001 volumeMounts: - name: servicedns-ca mountPath: /run/servicedns-ca diff --git a/benchmarking/automation/orchestrator.py b/benchmarking/automation/orchestrator.py index 97af6142dc..76f3272e28 100644 --- a/benchmarking/automation/orchestrator.py +++ b/benchmarking/automation/orchestrator.py @@ -390,9 +390,9 @@ def validate_and_normalize_tests(tests: list[dict[str, Any]]) -> None: def substrate_args(test: dict[str, Any]) -> list[str]: - """Return the hack/install-ate.sh --deploy-ate-system arguments for a + """Return the hack/install-ate.sh deploy ate-system arguments for a tests.yaml entry: its ateArgs, then the fake data plane flag its isolate - block asks for. The teardown needs no matching flag: --delete-ate-system + block asks for. The teardown needs no matching flag: delete ate-system removes the fake data plane whether or not it was installed.""" return [*test.get("ateArgs", []), *isolate_install_args(test)] @@ -401,7 +401,8 @@ def substrate_args(test: dict[str, Any]) -> list[str]: # pod certificate as BENCHMARK_RUNNER_PRINCIPAL. Each test reinstalls # Substrate, so making the runner a bootstrap owner on every install grants it # global owner for exactly the life of that install. kubectl-ate's principal -# stays an owner for the workload and teardown steps. +# stays an owner for the workload and teardown steps. The Envoy ingress router +# also requires an allowlist of client SPIFFE IDs on every install. BENCHMARK_RUNNER_PRINCIPAL = "spiffe://cluster.local/ns/benchmarking/sa/benchmark-runner" CLIENT_PRINCIPAL = "system:serviceaccount:ate-system:ate-client" @@ -410,12 +411,13 @@ def deploy_substrate(ate_args: Iterable[str] = ()) -> None: env = { **os.environ, "ATE_API_AUTHZ_BOOTSTRAP_OWNERS": f"{CLIENT_PRINCIPAL},{BENCHMARK_RUNNER_PRINCIPAL}", + "ATE_INGRESS_ALLOWED_SPIFFE_IDS": BENCHMARK_RUNNER_PRINCIPAL, } - run(["hack/install-ate.sh", "--deploy-ate-system", *(str(a) for a in ate_args)], env=env) + run(["hack/install-ate.sh", "deploy", "ate-system", *(str(a) for a in ate_args)], env=env) def teardown_substrate() -> None: - run_no_check(["hack/install-ate.sh", "--delete-ate-system"]) + run_no_check(["hack/install-ate.sh", "delete", "ate-system"]) def install_microvm_deps() -> None: diff --git a/benchmarking/automation/test_controlplane.py b/benchmarking/automation/test_controlplane.py new file mode 100644 index 0000000000..b243676257 --- /dev/null +++ b/benchmarking/automation/test_controlplane.py @@ -0,0 +1,204 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for controlplane.py: +python3 benchmarking/automation/test_controlplane.py""" + +import os +import unittest +from unittest import mock + +import yaml + +import controlplane + + +def stub(test, **fakes): + for name, fake in fakes.items(): + patcher = mock.patch.object(controlplane, name, fake) + patcher.start() + test.addCleanup(patcher.stop) + + +API = {"apiServer": {"cpu": 2}} + + +class ConfigTest(unittest.TestCase): + def test_defaults(self): + cfg = controlplane.config(API) + # The install defines ate-api-server's shape; only what the block + # sets is overridden. + self.assertEqual(cfg["apiServer"], {"cpu": 2}) + self.assertEqual(cfg["postgres"], {"stats": True, "args": []}) + + +class ValidateTest(unittest.TestCase): + def test_valid_knobs(self): + controlplane.validate(API) + controlplane.validate( + {"apiServer": {"cpu": "1500m", "memory": "8Gi", "replicas": 2, "poolMaxConns": 16}, + "postgres": {"stats": False}}) + controlplane.validate( + {**API, "postgres": {"args": ["commit_delay=100", "max_wal_size=16GB"]}}) + + def test_invalid(self): + def api(**knobs): + return {"apiServer": {"cpu": 2, **knobs}} + + def pg(**knobs): + return {**API, "postgres": knobs} + + bad = [ + {}, + "2", + {"apiServer": "2"}, + {"apiServer": {}}, + {**API, "ateapi": {}}, + {**API, "postgres": "on"}, + api(cpu=0), + api(cpu="two"), + api(cpu=True), + api(cpus=2), + api(replicas=0), + api(replicas=True), + api(poolMaxConns="64"), + api(memory="4 GiB"), + pg(stats="yes"), + pg(statz=True), + pg(args="commit_delay=100"), + pg(args=["commit_delay"]), + pg(args=["commit_delay = 100"]), + pg(args=["-c commit_delay=100"]), + pg(args=[100]), + # The settings ride on the restart stats makes. + pg(stats=False, args=["commit_delay=100"]), + ] + for block in bad: + with self.assertRaises(ValueError, msg=repr(block)): + controlplane.validate(block) + + def test_errors_name_the_knob(self): + with self.assertRaisesRegex(ValueError, "controlPlane.apiServer.cpu"): + controlplane.validate({"apiServer": {}}) + with self.assertRaisesRegex(ValueError, r"unknown controlPlane.postgres knob\(s\) \['statz'\]"): + controlplane.validate({**API, "postgres": {"statz": True}}) + + +class ApiServerPatchTest(unittest.TestCase): + def patch(self, **knobs): + return controlplane.api_server_patch( + controlplane.config({"apiServer": {"cpu": 2, **knobs}})["apiServer"]) + + def test_sets_the_knobs_given(self): + patch = self.patch(memory="4Gi", replicas=1) + self.assertEqual(patch["spec"]["replicas"], 1) + pod = patch["spec"]["template"]["spec"] + resources = pod["containers"][0]["resources"] + self.assertEqual(resources["requests"], {"cpu": "2", "memory": "4Gi"}) + self.assertEqual(resources["limits"], resources["requests"]) + # Placement is the install's. + self.assertNotIn("affinity", pod) + + def test_unset_knobs_keep_the_install(self): + patch = self.patch() + self.assertNotIn("replicas", patch["spec"]) + resources = patch["spec"]["template"]["spec"]["containers"][0]["resources"] + self.assertEqual(resources, {"requests": {"cpu": "2"}, "limits": {"cpu": "2"}}) + + +class ShapeTest(unittest.TestCase): + # Postgres restarts once, then ate-api-server rolls once and reads the + # pool size. + def test_order(self): + calls = [] + stub(self, **{ + name: lambda *_, name=name: calls.append(name) + for name in ("enable_postgres_stats", "set_pool_max_conns", "resize_api_server", "run_no_check") + }) + controlplane.shape(controlplane.config({"apiServer": {"cpu": 2, "poolMaxConns": 40}})) + self.assertEqual( + [c for c in calls if c != "run_no_check"], + ["enable_postgres_stats", "set_pool_max_conns", "resize_api_server"], + ) + + def test_unset_pool_size_keeps_the_install(self): + calls = [] + stub(self, + enable_postgres_stats=lambda *_: None, + set_pool_max_conns=lambda n: calls.append(n), + resize_api_server=lambda *_: None, + run_no_check=lambda *_: None) + controlplane.shape(controlplane.config(API)) + self.assertEqual(calls, []) + + def test_passes_the_settings(self): + got = [] + stub(self, + enable_postgres_stats=lambda settings: got.append(settings), + resize_api_server=lambda *_: None, + run_no_check=lambda *_: None) + controlplane.shape(controlplane.config({**API, "postgres": {"args": ["commit_delay=100"]}})) + self.assertEqual(got, [["commit_delay=100"]]) + + +class PostgresExporterTest(unittest.TestCase): + """The exporter's port is one ate-system's observability NetworkPolicy + admits; any other name leaves Prometheus timing out on the scrape.""" + + POLICIES = os.path.join( + os.path.dirname(__file__), "..", "..", "manifests", "ate-install", "ate-system-networkpolicies.yaml" + ) + + def admitted_ports(self): + for doc in yaml.safe_load_all(open(self.POLICIES)): + if doc and doc["metadata"]["name"] == "ate-system-observability": + return {p["port"] for rule in doc["spec"]["ingress"] for p in rule["ports"]} + self.fail("no ate-system-observability NetworkPolicy") + + def test_exporter_port_is_admitted(self): + patches = [] + stub(self, + _patch_and_wait=lambda kind, obj, patch: patches.append(patch), + run=lambda *_, **__: None) + controlplane.enable_postgres_stats([]) + containers = patches[0]["spec"]["template"]["spec"]["containers"] + exporter = next(c for c in containers if c["name"] == "postgres-exporter") + admitted = self.admitted_ports() + for port in exporter["ports"]: + self.assertIn(port["name"], admitted) + + +class PostgresArgsTest(unittest.TestCase): + def test_base_args_first_then_settings(self): + self.assertEqual( + controlplane.postgres_args(["commit_delay=100", "wal_buffers=64MB"]), + [ + "-c", "config_file=/etc/postgresql/postgresql.conf", + "-c", "shared_preload_libraries=pg_stat_statements", + "-c", "commit_delay=100", + "-c", "wal_buffers=64MB", + ], + ) + + +class PoolMaxConnsTest(unittest.TestCase): + def test_patch(self): + self.assertEqual( + controlplane.pool_max_conns_patch(40), + {"data": {"ATE_API_POSTGRES_POOL_MAX_CONNS": "40"}}, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarking/automation/test_locust.py b/benchmarking/automation/test_locust.py new file mode 100644 index 0000000000..0b3613ac15 --- /dev/null +++ b/benchmarking/automation/test_locust.py @@ -0,0 +1,127 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for testtypes/locust.py: +python3 benchmarking/automation/test_locust.py""" + +import os +import unittest +from unittest import mock + +import yaml + +import controlplane +import orchestrator +from testtypes import locust + + +def entry(**fields): + return { + "name": "t", + "file": "/app/tests/resumepaused.py,/app/shapes/ladder_shape.py", + "duration": "21m", + "users": 200, + **fields, + } + + +CONTROL_PLANE = {"apiServer": {"cpu": 2}} + + +class ValidateTest(unittest.TestCase): + def test_without_control_plane(self): + locust.validate(entry()) + self.assertIsNone(locust.control_plane_config(entry())) + + def test_with_control_plane(self): + test = entry(runnerCpu="8", runnerMemory="2Gi", controlPlane=CONTROL_PLANE) + locust.validate(test) + self.assertEqual(locust.control_plane_config(test), controlplane.config(CONTROL_PLANE)) + + # controlplane checks the block; its error names the test. + def test_invalid_control_plane_names_the_test(self): + with self.assertRaisesRegex(ValueError, "locust test 't': needs controlPlane.apiServer.cpu"): + locust.validate(entry(controlPlane={"apiServer": {}})) + + def test_missing_required(self): + test = entry() + del test["users"] + with self.assertRaises(ValueError): + locust.validate(test) + + +class PreTestTest(unittest.TestCase): + def shaped(self, test): + calls = [] + patcher = mock.patch.object(controlplane, "shape", calls.append) + patcher.start() + self.addCleanup(patcher.stop) + locust.pre_test(test) + return calls + + def test_without_control_plane_leaves_the_cluster(self): + self.assertEqual(self.shaped(entry()), []) + + def test_shapes_the_control_plane(self): + self.assertEqual(self.shaped(entry(controlPlane=CONTROL_PLANE)), [controlplane.config(CONTROL_PLANE)]) + + +class JobTest(unittest.TestCase): + def render(self, test): + subs = { + "JOB_NAME": "j", "NAME": "t", "TAG": "c", "IMAGE": "i", "DEST": "d", + **locust.job_subs(test), + } + tmpl = locust.job_tmpl(os.path.join(os.path.dirname(__file__), "manifests")) + text = orchestrator.render_template(tmpl, subs, test.get("flags", [])) + self.assertNotIn("${", text) + jobs = [d for d in yaml.safe_load_all(text) if d and d["kind"] == "Job"] + self.assertEqual(len(jobs), 1) + return jobs[0]["spec"]["template"]["spec"] + + def test_defaults_render(self): + test = entry() + pod = self.render(test) + self.assertNotIn("affinity", pod) + self.assertEqual( + pod["containers"][0]["resources"], + {"requests": {"cpu": "500m", "memory": "512Mi"}}, + ) + # monitoring.yaml's boomer-worker job scrapes this port. + self.assertIn( + {"name": "boomer-metrics", "containerPort": 8001}, + pod["containers"][0]["ports"], + ) + + def test_sized_runner_renders(self): + test = entry(runnerCpu="8", runnerMemory="2Gi", controlPlane=CONTROL_PLANE, + flags=["--actors", "200"]) + pod = self.render(test) + self.assertEqual( + pod["containers"][0]["resources"]["requests"], + {"cpu": "8", "memory": "2Gi"}, + ) + self.assertEqual(pod["containers"][0]["args"][-2:], ["--actors", "200"]) + + +class TestsYamlTest(unittest.TestCase): + def test_entries_validate(self): + path = os.path.join(os.path.dirname(__file__), "tests.yaml") + with open(path) as f: + tests = yaml.safe_load(f)["tests"] + orchestrator.validate_and_normalize_tests(tests) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarking/automation/test_orchestrator.py b/benchmarking/automation/test_orchestrator.py index aef06afed9..77833d43ba 100644 --- a/benchmarking/automation/test_orchestrator.py +++ b/benchmarking/automation/test_orchestrator.py @@ -62,13 +62,15 @@ def test_bare_isolate_key(self): class DeploySubstrateTest(unittest.TestCase): # The runner Job authenticates with its pod certificate, so ate-api-server - # denies it unless the install makes it an owner. + # denies it unless the install makes it an owner, and the Envoy ingress + # router refuses to deploy without an allowed SPIFFE ID. @mock.patch("orchestrator.run") def test_runner_and_client_are_bootstrap_owners(self, run): orchestrator.deploy_substrate(["--credential-provider={}"]) cmd = run.call_args.args[0] - self.assertEqual(cmd, ["hack/install-ate.sh", "--deploy-ate-system", "--credential-provider={}"]) - owners = run.call_args.kwargs["env"]["ATE_API_AUTHZ_BOOTSTRAP_OWNERS"].split(",") + self.assertEqual(cmd, ["hack/install-ate.sh", "deploy", "ate-system", "--credential-provider={}"]) + env = run.call_args.kwargs["env"] + owners = env["ATE_API_AUTHZ_BOOTSTRAP_OWNERS"].split(",") self.assertEqual( owners, [ @@ -76,6 +78,10 @@ def test_runner_and_client_are_bootstrap_owners(self, run): "spiffe://cluster.local/ns/benchmarking/sa/benchmark-runner", ], ) + self.assertEqual( + env["ATE_INGRESS_ALLOWED_SPIFFE_IDS"], + "spiffe://cluster.local/ns/benchmarking/sa/benchmark-runner", + ) @mock.patch("orchestrator.run") def test_keeps_the_rest_of_the_environment(self, run): @@ -83,6 +89,11 @@ def test_keeps_the_rest_of_the_environment(self, run): orchestrator.deploy_substrate() self.assertEqual(run.call_args.kwargs["env"]["KO_DOCKER_REPO"], "example") + @mock.patch("orchestrator.run_no_check") + def test_teardown_substrate(self, run_no_check): + orchestrator.teardown_substrate() + run_no_check.assert_called_once_with(["hack/install-ate.sh", "delete", "ate-system"]) + class DeployWorkloadsTest(unittest.TestCase): @mock.patch("orchestrator.run") diff --git a/benchmarking/automation/tests.yaml b/benchmarking/automation/tests.yaml index 6bc63d6495..945fdda3d1 100644 --- a/benchmarking/automation/tests.yaml +++ b/benchmarking/automation/tests.yaml @@ -52,14 +52,14 @@ # large workerCount fleets whose image pulls or scheduling # outrun the default. # ateArgs Optional. List of strings appended verbatim to -# `hack/install-ate.sh --deploy-ate-system` when this +# `hack/install-ate.sh deploy ate-system` when this # test's substrate is deployed (e.g. ["--atenet-dataplane=agentgateway"]). # The deploy requires --credential-provider; set it here # as one YAML string, '--credential-provider={"name":"k8s.io"}' # (no shell quoting: the args are passed without a shell), # or as ATE_CREDENTIAL_PROVIDER in .ate-dev-env.sh. # isolate Optional. Isolate mode: installs substrate with the fake -# data plane (hack/install-ate.sh --fake-data-plane), which +# data plane (hack/install-ate.sh deploy ate-system --fake-data-plane), which # serves the WorkerPool without worker pods: fake-atelet # takes atelet's place on every atelet node and answers each # lifecycle call after a delay, and fake-workersync honors @@ -112,6 +112,25 @@ # seconds, or the step falls short of its rate. A resumepaused iteration is a # resume plus a pause, so size it by the p99 of both together. # +# An optional `controlPlane:` block shapes the control plane for a capacity +# run; pre_test applies it after the deploy. Its `apiServer:` section, which +# is required, sizes ate-api-server: +# cpu Required. CPU of each replica, set as both request and +# limit (e.g. 2 or "1500m"). +# memory Optional. Memory request and limit of each replica. +# replicas Optional. +# poolMaxConns Optional. pgxpool max connections per replica, set +# through ateapi's env ConfigMap. +# An optional knob left unset keeps the installed value. +# Its optional `postgres:` section: +# stats Optional (default true). Load pg_stat_statements and add +# a postgres_exporter sidecar, scraped by monitoring.yaml. +# args Optional. Postgres settings as name=value (e.g. +# commit_delay=100), applied in the restart stats makes. +# The block leaves placement to the install. A capacity run sets +# --cordon-control-plane in ateArgs, which keeps ate-api-server and Postgres +# off the nodes that run the load, on the tainted pools that flag expects. +# # ---- type: nighthawk-ingress -------------------------------------------------------- # # Runs the Nighthawk adaptive router-capacity benchmark. `duration` is the @@ -662,3 +681,68 @@ tests: nighthawk-ingress: envoyCpu: 16 tailLatencySloMs: 25 + # ate-api-server and Postgres cold resume ceiling on the fake data plane. + # Each user owns one actor: its first iteration creates the actor and + # makes it cold, in the step that adds the user, and every later one + # resumes it and pauses it again. fake-atelet answers each call, restore + # and checkpoint alike, after 150 ms on every atelet node, so a cycle + # spends 300 ms in the data plane, after minting the actor's certificate + # on each restore as ateom does. The 200 fake Workers hold 1000 actors + # each, and at 10m CPU an actor books next to nothing of a Worker's CPU, + # which a fake Worker reports as its node's allocatable, so placement + # never binds on any node size. resumepaused sends the actors no + # requests, which the fake data plane needs. + - name: ateapi_cold_resume_pause_fake_2cpu + type: locust + description: "ate-api-server cold resume on the fake data plane, ate-api-server at 2 CPUs: rate ladder 100-1500 cold resumes/s" + targetCluster: dev + file: /app/tests/resumepaused.py,/app/shapes/ladder_shape.py + duration: 23m + users: 1000 + workerCount: 200 + actorCPU: "10m" + runnerCpu: "8" + runnerMemory: "2Gi" + isolate: + delay: 150ms + ateArgs: + - "--cordon-control-plane" + controlPlane: + apiServer: + replicas: 1 + cpu: 2 + memory: 4Gi + poolMaxConns: 64 + flags: + - "--ladder-spawn-rate" + - "1000" + - "--ladder" + - "1000:3m@100,1000:3m@200,1000:3m@400,1000:3m@600,1000:3m@800,1000:3m@1000,1000:3m@1500" + # Smoke for the cold resume setup above: the same fake data plane and + # ate-api-server shape at a load far below its ceiling, to check a change + # end to end in minutes. 50 users, each a cycle of about 300 ms, can make + # some 150 cold resumes/s, so both steps reach their rate. + - name: ateapi_cold_resume_pause_fake_smoke + type: locust + description: "ate-api-server cold resume smoke on the fake data plane, ate-api-server at 2 CPUs: 50 actors, 20 then 50 cold resumes/s" + targetCluster: dev + file: /app/tests/resumepaused.py,/app/shapes/ladder_shape.py + duration: 6m + users: 50 + workerCount: 2 + actorCPU: "10m" + isolate: + delay: 150ms + ateArgs: + - "--cordon-control-plane" + controlPlane: + apiServer: + replicas: 1 + cpu: 2 + memory: 4Gi + poolMaxConns: 64 + flags: + - "--ladder-spawn-rate" + - "50" + - "--ladder" + - "50:2m@20,50:2m@50" diff --git a/benchmarking/automation/testtypes/__init__.py b/benchmarking/automation/testtypes/__init__.py index 2bd024b20b..5890ef6046 100644 --- a/benchmarking/automation/testtypes/__init__.py +++ b/benchmarking/automation/testtypes/__init__.py @@ -22,8 +22,7 @@ build_image(commit) builds + pushes the runner image; called lazily, once per (target cluster, type). pre_test(test) after deploy_substrate, before deploy_workloads — - the type's chance to shape the cluster (nighthawk-ingress - pins the router; locust needs nothing). + the type's chance to shape the cluster. job_tmpl(manifests_dir) path of the type's own Job template. job_subs(test) the type's ${...} substitutions for it. The test diff --git a/benchmarking/automation/testtypes/locust.py b/benchmarking/automation/testtypes/locust.py index fd68897207..bf2aaa2540 100644 --- a/benchmarking/automation/testtypes/locust.py +++ b/benchmarking/automation/testtypes/locust.py @@ -16,21 +16,39 @@ The runner Job wraps benchmarking/locust/runner.py, which drives locust (and boomer for glutton tests) and uploads its own results. + +An optional `controlPlane:` block shapes ate-api-server and Postgres for a +capacity run (see controlplane.py); without it pre_test leaves the cluster +as installed. """ import os from typing import Any +import controlplane from util import build_and_push TEST_TYPE = "locust" +def control_plane_config(test: dict[str, Any]) -> dict[str, Any] | None: + """The test's `controlPlane:` block merged over its defaults, or None.""" + if "controlPlane" not in test: + return None + return controlplane.config(test["controlPlane"]) + + def validate(test: dict[str, Any]) -> None: name = test.get("name") for field in ("file", "duration", "users"): if field not in test: raise ValueError(f"locust test {name!r} missing {field!r}") + if "controlPlane" not in test: + return + try: + controlplane.validate(test["controlPlane"]) + except ValueError as e: + raise ValueError(f"locust test {name!r}: {e}") from None def build_image(commit: str) -> str: @@ -43,7 +61,11 @@ def build_image(commit: str) -> str: def pre_test(test: dict[str, Any]) -> None: - """Nothing to shape on the cluster before a locust run.""" + """With a `controlPlane:` block, shape the control plane for the run.""" + cfg = control_plane_config(test) + if cfg is None: + return + controlplane.shape(cfg) def job_tmpl(manifests_dir: str) -> str: diff --git a/benchmarking/locust/Dockerfile b/benchmarking/locust/Dockerfile index 67791e75fc..c2ecb40d8c 100644 --- a/benchmarking/locust/Dockerfile +++ b/benchmarking/locust/Dockerfile @@ -35,7 +35,7 @@ RUN PYTHON=python3 OUT_DIR=/out benchmarking/locust/codegen/generate.sh # Stage 3: Go build of the boomer-worker worker. Lives in the same image so # runner.py can spawn it as a subprocess during headless runs of glutton. # CGO_ENABLED=0 keeps us on boomer's pure-Go ZMQ (gomq), no libzmq needed. -FROM golang:1.26-bookworm AS goboomer +FROM golang:1.27.3-bookworm@sha256:5c3e9b43cfd3bb95c7937462e29d87ada57b481a96fa23ad0eb61778d7a00110 AS goboomer ENV GOTOOLCHAIN=auto WORKDIR /src COPY go.mod go.sum ./ @@ -59,6 +59,7 @@ COPY benchmarking/locust/runner.py /app/runner.py COPY benchmarking/locust/cluster_facts.py /app/cluster_facts.py COPY benchmarking/locust/server_telemetry.py /app/server_telemetry.py COPY benchmarking/locust/phase_breakdown.py /app/phase_breakdown.py +COPY benchmarking/locust/ladder_steps.py /app/ladder_steps.py COPY benchmarking/analysis/phase_report.py /app/phase_report.py ENV PYTHONPATH=/app:/app/deps diff --git a/benchmarking/locust/common/lifecycle_mode.py b/benchmarking/locust/common/lifecycle_mode.py index 644a50df40..77e27de69a 100644 --- a/benchmarking/locust/common/lifecycle_mode.py +++ b/benchmarking/locust/common/lifecycle_mode.py @@ -26,7 +26,7 @@ def add_lifecycle_mode_arguments(parser: LocustArgumentParser) -> None: type=str, default="suspend", choices=["suspend", "pause", "none"], - help="Lifecycle mode: 'suspend' uses SuspendActor (persists snapshot to storage); " - "'pause' uses an asynchronous SuspendActor (keeps snapshot on local node); " + help="Lifecycle mode: 'suspend' uses synchronous SuspendActor (uploads snapshot before returning); " + "'pause' uses asynchronous SuspendActor (uploads snapshot in background); " "'none' keeps actors resident in memory between steps (default: suspend)", ) diff --git a/benchmarking/locust/ladder_steps.py b/benchmarking/locust/ladder_steps.py new file mode 100644 index 0000000000..654a0cd9c3 --- /dev/null +++ b/benchmarking/locust/ladder_steps.py @@ -0,0 +1,253 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Per-step rows for stats.jsonl from a ladder run (shapes/ladder_shape.py). + +stats.jsonl's other rows cover the whole run, so they blend every step of a +ladder. This reads each step's start and end from the runner's log and +queries Prometheus over the step's steady window: from SETTLE seconds after +the step starts, as boomer reads a new target rate within 10 s, to TAIL +seconds before it ends. Each step becomes one `ladder_step_NNN` row. + +Like the server telemetry, this is additive: runner.py logs a failure here +and keeps the locust measurements. +""" + +import json +import re +import urllib.parse +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable, TextIO + +# Seconds trimmed from each end of a step. +SETTLE = 30 +TAIL = 5 + +STEP_RE = re.compile( + r"\[locust\] \[(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d),\d+\].*" + r"Ladder step (\d+)/(\d+): users=(\d+).*target_rps=([\d.]+|unchanged)" +) +DONE_RE = re.compile(r"\[locust\] \[(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d),\d+\].*Ladder complete") + + +def _ts(s: str) -> int: + return int(datetime.strptime(s, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc).timestamp()) + + +def steps(lines: Iterable[str]) -> list[dict[str, Any]]: + """The ladder's steps from runner log lines, each with its start and end + as Unix seconds. A step ends where the next starts; the last ends at + "Ladder complete", or is None if the log has no such line. A step logged + with target_rps=unchanged keeps the previous step's rate, None before + any step sets one.""" + out: list[dict[str, Any]] = [] + done = None + for line in lines: + if m := STEP_RE.search(line): + rps = out[-1]["target_rps"] if out else None + if m[5] != "unchanged": + rps = float(m[5]) + out.append({"step": int(m[2]), "users": int(m[4]), + "target_rps": rps, "start": _ts(m[1])}) + elif m := DONE_RE.search(line): + done = _ts(m[1]) + for i, s in enumerate(out): + s["end"] = out[i + 1]["start"] if i + 1 < len(out) else done + return out + + +# Seconds a query may take, so an unreachable Prometheus fails the summary +# instead of hanging the runner. +QUERY_TIMEOUT_S = 30 + + +def _get_json(url: str) -> Any: + with urllib.request.urlopen(url, timeout=QUERY_TIMEOUT_S) as r: + return json.load(r) + + +# Not server_telemetry.py's helper, which turns a failed query into an empty +# result: a step's row must fail instead of carrying a gap. +def query(prom: str, q: str, t: int) -> list[dict[str, Any]]: + url = f"{prom.rstrip('/')}/api/v1/query?" + urllib.parse.urlencode({"query": q, "time": t}) + data = _get_json(url) + if data.get("status") != "success": + raise RuntimeError(f"{q}: {data}") + return data["data"]["result"] + + +def scalar(prom: str, q: str, t: int) -> float | None: + """The query's first value, or None if it has none or it is NaN.""" + r = query(prom, q, t) + if not r: + return None + v = float(r[0]["value"][1]) + return None if v != v else v + + +def by_label(prom: str, q: str, t: int, label: str) -> dict[str, float]: + return {r["metric"].get(label, ""): float(r["value"][1]) for r in query(prom, q, t)} + + +# TODO: separate the test's metrics from the harness. summarize() and +# memory() hardcode the cold resume test's queries (its request names, +# containers and database), so each new ladder test means another copy. +# The test should declare the metrics it reports, leaving this module the +# step windows and the queries' execution. +def summarize(prom: str, s: dict[str, Any]) -> dict[str, Any]: + """One step's throughput, latency, CPU and Postgres figures over its + steady window.""" + a, b = s["start"] + SETTLE, s["end"] - TAIL + secs = b - a + d = f"{secs}s" + req = 'locust_requests_total{name="%s",status="%s"}' + hist = 'histogram_quantile(%s, sum by (le) (increase(%s_bucket{name="%s"%s}[%s])))' + mean = ('sum(increase(%s_sum{name="%s"%s}[%s]))' + '/sum(increase(%s_count{name="%s"%s}[%s]))') + cpu = 'sum(increase(container_cpu_usage_seconds_total{namespace="ate-system",container="%s"}[%s]))/%d' + thr = ('sum(increase(container_cpu_cfs_throttled_periods_total{namespace="ate-system",container="ate-api-server"}[%s]))' + '/sum(increase(container_cpu_cfs_periods_total{namespace="ate-system",container="ate-api-server"}[%s]))') + pg_conns = 'avg_over_time(sum(pg_stat_activity_count{datname="atepg"%s})[%s:15s])' + client, server = "locust_request_duration_milliseconds", "locust_server_duration_milliseconds" + ok = ',status="success"' + + def q(expr: str) -> float | None: + return scalar(prom, expr, b) + + def latency(metric: str, name: str, extra: str, quantile: float) -> float | None: + return q(hist % (quantile, metric, name, extra, d)) + + def mean_ms(metric: str, name: str, extra: str) -> float | None: + return q(mean % (metric, name, extra, d, metric, name, extra, d)) + + per_replica = by_label( + prom, + f'sum by (pod) (increase(container_cpu_usage_seconds_total{{namespace="ate-system",container="ate-api-server"}}[{d}]))/{secs}', + b, "pod") + out = { + "step": s["step"], + "users": s["users"], + "target_rps": s["target_rps"], + "window": [a, b], + "resume_rps": q(f"sum(increase({req % ('ResumeActorCold', 'success')}[{d}]))/{secs}"), + "pause_rps": q(f"sum(increase({req % ('SuspendActorAsync', 'success')}[{d}]))/{secs}"), + "failures": by_label(prom, f'sum by (name) (increase(locust_requests_total{{status="failure"}}[{d}]))', b, "name"), + "resume_client_p50_ms": latency(client, "ResumeActorCold", ok, 0.5), + "resume_client_p99_ms": latency(client, "ResumeActorCold", ok, 0.99), + "resume_server_p50_ms": latency(server, "ResumeActorCold", "", 0.5), + "resume_server_p90_ms": latency(server, "ResumeActorCold", "", 0.9), + "resume_server_p95_ms": latency(server, "ResumeActorCold", "", 0.95), + "resume_server_p99_ms": latency(server, "ResumeActorCold", "", 0.99), + "pause_client_p50_ms": latency(client, "SuspendActorAsync", ok, 0.5), + "pause_client_p99_ms": latency(client, "SuspendActorAsync", ok, 0.99), + "resume_client_mean_ms": mean_ms(client, "ResumeActorCold", ok), + "resume_server_mean_ms": mean_ms(server, "ResumeActorCold", ""), + "pause_client_mean_ms": mean_ms(client, "SuspendActorAsync", ok), + "pause_server_mean_ms": mean_ms(server, "SuspendActorAsync", ""), + "ateapi_cores": q(cpu % ("ate-api-server", d, secs)), + # Each replica's cores, busiest first. + "ateapi_cores_per_replica": sorted(per_replica.values(), reverse=True), + "ateapi_throttled_ratio": q(thr % (d, d)), + "postgres_cores": q(cpu % ("postgres", d, secs)), + # On the fake data plane this is fake-atelet, which runs in atelet's + # DaemonSet under atelet's container name. + "atelet_cores": q(cpu % ("atelet", d, secs)), + "pg_active_conns": q(pg_conns % (',state="active"', d)), + "pg_conns": q(pg_conns % ("", d)), + "pg_commits_per_s": q(f'sum(increase(pg_stat_database_xact_commit{{datname="atepg"}}[{d}]))/{secs}'), + "pg_lock_wait_conns": q(pg_conns % (',state="active",wait_event_type="Lock"', d)), + "pg_walwrite_wait_conns": q(pg_conns % (',state="active",wait_event="WALWrite"', d)), + } + out.update(memory(prom, d, secs, b)) + return out + + +def memory(prom: str, d: str, secs: int, b: int) -> dict[str, Any]: + """Memory over the step's window: peak working set and RSS in MiB, the + share of time stalled on memory (PSI), and Postgres's buffer cache hits.""" + ws = 'container_memory_working_set_bytes{namespace="ate-system",container="%s"}' + rss = 'container_memory_rss{namespace="ate-system",container="%s"}' + psi = 'rate(container_pressure_memory_waiting_seconds_total{namespace="ate-system",container="%s"}[%s])' + blks = 'sum(increase(pg_stat_database_blks_%s{datname="atepg"}[%s]))' + + def mib(q: str) -> float | None: + v = scalar(prom, q, b) + return None if v is None else v / (1024 * 1024) + + hit, read = scalar(prom, blks % ("hit", d), b), scalar(prom, blks % ("read", d), b) + return { + # The largest replica's peak, and all replicas' mean together. + "ateapi_mem_ws_max_mib": mib(f'max(max_over_time({ws % "ate-api-server"}[{d}]))'), + "ateapi_mem_ws_total_mib": mib(f'sum(avg_over_time({ws % "ate-api-server"}[{d}]))'), + "ateapi_mem_rss_max_mib": mib(f'max(max_over_time({rss % "ate-api-server"}[{d}]))'), + "ateapi_mem_psi": scalar(prom, f'max({psi % ("ate-api-server", d)})', b), + "postgres_mem_ws_mib": mib(f'max(max_over_time({ws % "postgres"}[{d}]))'), + "postgres_mem_rss_mib": mib(f'max(max_over_time({rss % "postgres"}[{d}]))'), + "postgres_mem_psi": scalar(prom, f'max({psi % ("postgres", d)})', b), + "pg_cache_hit_ratio": None if hit is None or read is None or hit + read == 0 else hit / (hit + read), + "pg_blks_read_per_s": None if read is None else read / secs, + } + + +def stats_record(summary: dict[str, Any], timestamp: str, tag: str, + test_name: str) -> dict[str, Any]: + """A step's summary as a stats.jsonl row, its measurements flattened to + scalars like every other row's.""" + nested = ("step", "window", "failures", "ateapi_cores_per_replica") + m = {k: v for k, v in summary.items() if k not in nested} + m["window_start"], m["window_end"] = summary["window"] + m["ateapi_cores_busiest_replica"] = max(summary["ateapi_cores_per_replica"], default=None) + for name, n in summary["failures"].items(): + m[f"failures_{name}"] = n + return { + "timestamp": timestamp, + "tag": tag, + "test_name": test_name, + "metric": f"ladder_step_{summary['step']:03d}", + "measurements": m, + } + + +def append_ladder_steps(jsonl_path: Path, log_path: Path, prom_url: str, + timestamp: str, tag: str, test_name: str, + logs: TextIO) -> int: + """Append a row per measurable ladder step in the runner log at + log_path to jsonl_path, and return how many. A run that is no ladder + has none. Every step is queried before any row is written, so a failed + query leaves no partial ladder.""" + with open(log_path, errors="replace") as f: + ladder = steps(f) + rows = [] + for s in ladder: + if s["end"] is None: + _log(logs, f"Ladder step {s['step']} has no end in the log; skipped") + continue + if s["end"] - s["start"] <= SETTLE + TAIL: + _log(logs, f"Ladder step {s['step']} lasts {s['end'] - s['start']}s, too short " + f"for a steady window after trimming {SETTLE + TAIL}s; skipped") + continue + rows.append(stats_record(summarize(prom_url, s), timestamp, tag, test_name)) + if rows: + with open(jsonl_path, "a", encoding="utf-8") as f: + for row in rows: + f.write(json.dumps(row) + "\n") + return len(rows) + + +def _log(logs: TextIO, msg: str) -> None: + print(msg, flush=True) + logs.write(msg + "\n") + logs.flush() diff --git a/benchmarking/locust/runner.py b/benchmarking/locust/runner.py index be7503321e..03bf485e89 100644 --- a/benchmarking/locust/runner.py +++ b/benchmarking/locust/runner.py @@ -54,6 +54,7 @@ from common.boomer_config import build_config_json from server_telemetry import extract_and_record_server_telemetry from phase_breakdown import append_phase_breakdown +from ladder_steps import append_ladder_steps # Path inside the locust image to the boomer-worker binary baked in by # benchmarking/locust/Dockerfile. @@ -658,6 +659,19 @@ def main() -> None: except Exception as e: tee(logs, f"Warning: Failed to harvest server telemetry: {e}") + # Per-step rows for a ladder run, from the same Prometheus. Placed + # after the server telemetry, whose wait lets the last step's samples + # be scraped. + try: + n = append_ladder_steps( + jsonl_path, logs_path, args.prometheus_url, data_ts, + args.tag, args.name, logs, + ) + if n: + tee(logs, f"Appended {n} ladder step rows to {jsonl_path}") + except Exception as e: + tee(logs, f"Warning: Failed to summarize ladder steps: {e}") + # The phase breakdown is additive too. It reads the node logs now, # while the pods still exist: the orchestrator deletes them as soon # as this process exits. The window reaches back to this run's start diff --git a/benchmarking/locust/unit_tests/test_ladder_steps.py b/benchmarking/locust/unit_tests/test_ladder_steps.py new file mode 100644 index 0000000000..89d6c3c9db --- /dev/null +++ b/benchmarking/locust/unit_tests/test_ladder_steps.py @@ -0,0 +1,223 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for ladder_steps.py. + +Run via: python3 benchmarking/locust/unit_tests/test_ladder_steps.py +""" + +import io +import json +import sys +import tempfile +import unittest +import urllib.parse +from pathlib import Path +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import ladder_steps + +LOG = """\ +[locust] [2026-10-12 00:19:07,123] host/INFO/ladder_shape: Ladder step 1/2: users=100 trace_probability=unchanged target_rps=500.0 +[boomer] {"msg":"dynconfig applied"} +[locust] [2026-10-12 00:22:07,456] host/INFO/ladder_shape: Ladder step 2/2: users=100 trace_probability=unchanged target_rps=1000.0 +[locust] [2026-10-12 00:25:07,789] host/INFO/ladder_shape: Ladder complete after 360s +""" +T0 = 1791764347 # 2026-10-12 00:19:07 UTC + + +class StepsTest(unittest.TestCase): + def test_parses_steps_and_their_ends(self): + self.assertEqual(ladder_steps.steps(LOG.splitlines()), [ + {"step": 1, "users": 100, "target_rps": 500.0, "start": T0, "end": T0 + 180}, + {"step": 2, "users": 100, "target_rps": 1000.0, "start": T0 + 180, "end": T0 + 360}, + ]) + + def test_unfinished_ladder_leaves_the_last_end_open(self): + got = ladder_steps.steps(LOG.splitlines()[:3]) + self.assertEqual([s["end"] for s in got], [T0 + 180, None]) + + # A step that changes only the users still bounds its neighbors. + def test_unchanged_rate_keeps_the_previous_one(self): + log = ( + "[locust] [2026-10-12 00:19:07,1] x: Ladder step 1/3: users=100 trace_probability=unchanged target_rps=unchanged\n" + "[locust] [2026-10-12 00:22:07,1] x: Ladder step 2/3: users=100 trace_probability=unchanged target_rps=500.0\n" + "[locust] [2026-10-12 00:25:07,1] x: Ladder step 3/3: users=200 trace_probability=unchanged target_rps=unchanged\n" + "[locust] [2026-10-12 00:28:07,1] x: Ladder complete after 540s\n" + ) + got = ladder_steps.steps(log.splitlines()) + self.assertEqual([(s["step"], s["target_rps"]) for s in got], [(1, None), (2, 500.0), (3, 500.0)]) + self.assertEqual([s["end"] for s in got], [T0 + 180, T0 + 360, T0 + 540]) + + +def prometheus(answers): + """A fake Prometheus API: the first matching answer for each query, by + substring; a query no answer matches gets an empty result.""" + def get_json(url): + q = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)["query"][0] + for needle, result in answers: + if needle in q: + return {"status": "success", "data": {"result": result}} + return {"status": "success", "data": {"result": []}} + return get_json + + +def value(v, **labels): + return [{"metric": labels, "value": [0, str(v)]}] + + +class SummarizeTest(unittest.TestCase): + STEP = {"step": 1, "users": 100, "target_rps": 500.0, "start": T0, "end": T0 + 180} + + def summarize(self, answers): + with mock.patch.object(ladder_steps, "_get_json", prometheus(answers)): + return ladder_steps.summarize("http://prom", self.STEP) + + def test_window_trims_both_ends(self): + got = self.summarize([]) + self.assertEqual(got["window"], [T0 + ladder_steps.SETTLE, T0 + 180 - ladder_steps.TAIL]) + + def test_missing_series_are_none_and_nan_is_none(self): + got = self.summarize([('status="success"', value("NaN"))]) + self.assertIsNone(got["resume_rps"]) + self.assertIsNone(got["postgres_cores"]) + self.assertIsNone(got["pg_cache_hit_ratio"]) + self.assertEqual(got["ateapi_cores_per_replica"], []) + + def test_per_replica_cores_busiest_first(self): + answers = [("sum by (pod)", [ + {"metric": {"pod": "a"}, "value": [0, "290"]}, + {"metric": {"pod": "b"}, "value": [0, "435"]}, + ])] + got = self.summarize(answers) + self.assertEqual(got["ateapi_cores_per_replica"], [435.0, 290.0]) + + def test_atelet_cores_come_from_the_atelet_container(self): + got = self.summarize([('container="atelet"', value(0.25))]) + self.assertEqual(got["atelet_cores"], 0.25) + + def test_cache_hit_ratio(self): + got = self.summarize([("blks_hit", value(990)), ("blks_read", value(10))]) + self.assertAlmostEqual(got["pg_cache_hit_ratio"], 0.99) + self.assertAlmostEqual(got["pg_blks_read_per_s"], 10 / 145) + + def test_walwrite_waiters_come_from_their_own_query(self): + got = self.summarize([('wait_event="WALWrite"', value(42)), ('wait_event_type="Lock"', value(3))]) + self.assertEqual(got["pg_walwrite_wait_conns"], 42.0) + self.assertEqual(got["pg_lock_wait_conns"], 3.0) + + def test_query_trims_the_url_and_times_out(self): + opened = [] + + def urlopen(url, timeout=None): + opened.append((url, timeout)) + return mock.MagicMock(__enter__=lambda self: self) + + with mock.patch.object(ladder_steps.urllib.request, "urlopen", urlopen), \ + mock.patch.object(ladder_steps.json, "load", lambda r: {"status": "success", "data": {"result": []}}): + ladder_steps.query("http://prom/", "up", T0) + url, timeout = opened[0] + self.assertTrue(url.startswith("http://prom/api/v1/query?"), url) + self.assertEqual(timeout, ladder_steps.QUERY_TIMEOUT_S) + + def test_failed_query_raises(self): + with mock.patch.object(ladder_steps, "_get_json", lambda url: {"status": "error"}): + with self.assertRaises(RuntimeError): + ladder_steps.summarize("http://prom", self.STEP) + + +class StatsRecordTest(unittest.TestCase): + def test_flattens_the_summary(self): + summary = {"step": 2, "users": 100, "target_rps": 500.0, "window": [10, 20], + "failures": {"ResumeActorCold": 3.0}, "ateapi_cores_per_replica": [1.5, 0.5], + "resume_rps": 499.0} + self.assertEqual(ladder_steps.stats_record(summary, "ts", "c", "t"), { + "timestamp": "ts", "tag": "c", "test_name": "t", "metric": "ladder_step_002", + "measurements": {"users": 100, "target_rps": 500.0, "resume_rps": 499.0, + "window_start": 10, "window_end": 20, + "ateapi_cores_busiest_replica": 1.5, + "failures_ResumeActorCold": 3.0}, + }) + + def test_no_replicas_reported(self): + summary = {"step": 1, "window": [10, 20], "failures": {}, "ateapi_cores_per_replica": []} + m = ladder_steps.stats_record(summary, "ts", "c", "t")["measurements"] + self.assertIsNone(m["ateapi_cores_busiest_replica"]) + + +class AppendLadderStepsTest(unittest.TestCase): + def append(self, log, get_json=None, existing=""): + with tempfile.TemporaryDirectory() as d: + log_path, jsonl_path = Path(d, "logs.txt"), Path(d, "stats.jsonl") + log_path.write_text(log) + if existing: + jsonl_path.write_text(existing) + logs = io.StringIO() + with mock.patch.object(ladder_steps, "_get_json", get_json or prometheus([])), \ + mock.patch("builtins.print"): + n = ladder_steps.append_ladder_steps( + jsonl_path, log_path, "http://prom", "ts", "c", "t", logs) + rows = jsonl_path.read_text().splitlines() if jsonl_path.exists() else None + return n, rows, logs.getvalue() + + def test_appends_a_row_per_finished_step(self): + n, rows, _ = self.append(LOG, existing='{"metric": "locust"}\n') + self.assertEqual(n, 2) + self.assertEqual(json.loads(rows[0]), {"metric": "locust"}) + self.assertEqual([json.loads(r)["metric"] for r in rows[1:]], + ["ladder_step_001", "ladder_step_002"]) + + def test_skips_a_step_too_short_to_measure(self): + short = LOG.replace("00:22:07,456", "00:19:37,456") # step 1 lasts 30s + n, rows, logged = self.append(short) + self.assertEqual(n, 1) + self.assertEqual(json.loads(rows[0])["metric"], "ladder_step_002") + self.assertIn("too short", logged) + + def test_unfinished_step_is_skipped(self): + n, _, logged = self.append("\n".join(LOG.splitlines()[:3])) + self.assertEqual(n, 1) + self.assertIn("no end", logged) + + # A run without the ladder shape leaves stats.jsonl alone, and creates + # none when the run produced none. + def test_no_ladder_writes_nothing(self): + self.assertEqual(self.append("[locust] no ladder here\n"), (0, None, "")) + + def test_failed_query_writes_no_partial_ladder(self): + calls = [] + + def get_json(url): + calls.append(url) + # Step 1 takes some 35 queries, so this fails within step 2. + if len(calls) > 40: + return {"status": "error"} + return {"status": "success", "data": {"result": []}} + + with tempfile.TemporaryDirectory() as d: + log_path, jsonl_path = Path(d, "logs.txt"), Path(d, "stats.jsonl") + log_path.write_text(LOG) + jsonl_path.write_text('{"metric": "locust"}\n') + with mock.patch.object(ladder_steps, "_get_json", get_json): + with self.assertRaises(RuntimeError): + ladder_steps.append_ladder_steps( + jsonl_path, log_path, "http://prom", "ts", "c", "t", io.StringIO()) + self.assertGreater(len(calls), 40) + self.assertEqual(jsonl_path.read_text(), '{"metric": "locust"}\n') + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarking/locust/unit_tests/test_server_telemetry.py b/benchmarking/locust/unit_tests/test_server_telemetry.py index 47ed4539d4..9e4525f21c 100644 --- a/benchmarking/locust/unit_tests/test_server_telemetry.py +++ b/benchmarking/locust/unit_tests/test_server_telemetry.py @@ -785,6 +785,29 @@ def test_telemetry_survives_a_missing_stats_csv(self, _run, _up, telemetry): self.assertEqual(telemetry.call_args.kwargs["lag_s"], 70) self.assertNotIn("worker_pod_count", telemetry.call_args.kwargs) + # Ladder rows follow the telemetry harvest, whose wait covers their + # last samples, and a failure there still uploads the run. + @mock.patch.object(runner, "upload") + @mock.patch.object(runner, "run_test", return_value=1) + def test_ladder_steps_follow_telemetry_and_may_fail(self, _run, upload): + calls = [] + telemetry = mock.Mock(side_effect=lambda **_: calls.append("telemetry")) + + def ladder(jsonl_path, log_path, prom_url, *_): + calls.append(("ladder", log_path.name, prom_url)) + raise RuntimeError("prometheus down") + + out = io.StringIO() + with mock.patch.object(runner, "extract_and_record_server_telemetry", telemetry), \ + mock.patch.object(runner, "append_ladder_steps", ladder), \ + mock.patch.object(sys, "argv", ARGV + ["--no-cluster-facts", "--allow-empty-stats", + "--prometheus-url", "http://x:9090"]), \ + contextlib.redirect_stdout(out): + runner.main() + self.assertEqual(calls, ["telemetry", ("ladder", "unit-run_logs.txt", "http://x:9090")]) + self.assertIn("Failed to summarize ladder steps: prometheus down", out.getvalue()) + self.assertTrue(upload.called) + if __name__ == "__main__": unittest.main() diff --git a/benchmarking/monitoring.yaml b/benchmarking/monitoring.yaml index 772f85050e..abb9b8b228 100644 --- a/benchmarking/monitoring.yaml +++ b/benchmarking/monitoring.yaml @@ -81,6 +81,30 @@ subjects: name: prometheus namespace: benchmarking --- +# The postgres-exporter job discovers pods in ate-system. A Role there would +# go with the namespace, which every automated test deletes and recreates. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus-benchmarking-pods +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus-benchmarking-pods +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-benchmarking-pods +subjects: +- kind: ServiceAccount + name: prometheus + namespace: benchmarking +--- apiVersion: v1 kind: ConfigMap metadata: @@ -137,7 +161,8 @@ data: target_label: pod # The locust pod annotates port 8000 for the Python locust process. The - # boomer-worker sidecar exposes its Prometheus metrics on port 8001. + # boomer-worker sidecar exposes its Prometheus metrics on port 8001, and + # so does boomer inside an automation runner Job. - job_name: 'boomer-worker' kubernetes_sd_configs: - role: pod @@ -149,7 +174,27 @@ data: - __meta_kubernetes_pod_label_app - __meta_kubernetes_pod_container_port_number action: keep - regex: locust;8001 + regex: (locust|substrate-benchmark-runner);8001 + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod + + # The postgres_exporter sidecar that an automation test with a + # `controlPlane:` block adds to Postgres (benchmarking/automation/ + # controlplane.py): connections by state, commits, cache hits, and + # pg_stat_statements. + - job_name: 'postgres-exporter' + kubernetes_sd_configs: + - role: pod + namespaces: + names: + - ate-system + relabel_configs: + - source_labels: + - __meta_kubernetes_pod_label_app + - __meta_kubernetes_pod_container_port_number + action: keep + regex: postgres;9187 - source_labels: [__meta_kubernetes_pod_name] action: replace target_label: pod diff --git a/benchmarking/nighthawk-egress/README.md b/benchmarking/nighthawk-egress/README.md index a0796bae9f..24c3fcc2ac 100644 --- a/benchmarking/nighthawk-egress/README.md +++ b/benchmarking/nighthawk-egress/README.md @@ -36,15 +36,23 @@ variable when deploying ATE. 1. **Cluster with Agent Substrate deployed**: - **Existing cluster** (with `.ate-dev-env.sh` configured): ```bash - hack/install-ate.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' + hack/install-ate.sh deploy ate-system \ + --credential-provider='{"name":"k8s.io"}' \ + --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/nighthawk/sa/default ``` - **Local `kind` cluster**: ```bash - hack/install-ate-kind.sh --deploy-ate-system --credential-provider='{"enabled":false}' + hack/install-ate-kind.sh deploy ate-system \ + --credential-provider='{"enabled":false}' \ + --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/nighthawk/sa/default ``` - To set the `atenet-egress` Envoy worker thread count (`--concurrency`), set `E2E_ATE_ENVOY_CONCURRENCY` when deploying: ```bash - E2E_ATE_ENVOY_CONCURRENCY=1 hack/install-ate-kind.sh --deploy-ate-system --credential-provider='{"enabled":false}' + E2E_ATE_ENVOY_CONCURRENCY=1 hack/install-ate-kind.sh deploy ate-system --credential-provider='{"name":"k8s.io"}' --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/nighthawk/sa/default + ``` + - Provision CA bundle which includes cluster's trust bundle, so that egress gateway can trust nighhawk's test server cert, run: + ```bash + hack/setup-e2e-egress-tls-kind.sh ``` 2. **`kubectl-ate` installed in `PATH`**: ```bash @@ -79,7 +87,7 @@ kubectl -n ate-system port-forward svc/traffic-driver-invoker 19980:19980 Run `nighthawk_client` against the port-forwarded `traffic-driver-invoker` endpoint (`grpc://127.0.0.1:19980`), which routes the gRPC request to `nighthawk_service` inside the `nighthawk/tester` actor: ```bash -docker run --rm --network=host --entrypoint /usr/local/bin/nighthawk_client envoyproxy/nighthawk-dev:latest --nighthawk-service grpc://127.0.0.1:19980 --rps 1000 --concurrency 1 --protocol http1 --duration 60 http://nighthawk-server.ate-system.svc.cluster.local:80 +docker run --rm --network=host --entrypoint /usr/local/bin/nighthawk_client envoyproxy/nighthawk-dev:latest --nighthawk-service grpc://127.0.0.1:19980 --rps 1000 --concurrency 1 --protocol http1 --duration 60 http://nighthawk-server.ate-system.svc:80 ``` See [Nighthawk CLI help](https://github.com/envoyproxy/nighthawk#using-the-nighthawk-client-cli) for all command line options. @@ -88,9 +96,9 @@ See [Nighthawk CLI help](https://github.com/envoyproxy/nighthawk#using-the-night | Egress Mode | Target URL | Port | |---|---|---| -| Plaintext HTTP | `http://nighthawk-server.ate-system.svc.cluster.local:80` | `80` | -| HTTPS (TLS MITM) | `https://nighthawk-server.ate-system.svc.cluster.local:443` | `443` | -| TLS Passthrough | `https://nighthawk-server.ate-system.svc.cluster.local:8443` | `8443` | +| Plaintext HTTP | `http://nighthawk-server.ate-system.svc:80` | `80` | +| HTTPS (TLS MITM) | `https://nighthawk-server.ate-system.svc:443` | `443` | +| TLS Passthrough | `https://nighthawk-server.ate-system.svc:8443` | `8443` | ## Cleanup diff --git a/benchmarking/nighthawk-egress/deploy.sh b/benchmarking/nighthawk-egress/deploy.sh index 713e76dc0a..f10f2d2441 100755 --- a/benchmarking/nighthawk-egress/deploy.sh +++ b/benchmarking/nighthawk-egress/deploy.sh @@ -85,9 +85,9 @@ if [[ "${action}" == "deploy" ]]; then echo "Deploying traffic-driver-invoker from ${INVOKER_MANIFEST}..." "${kubectl_cmd[@]}" apply -f "${INVOKER_MANIFEST}" echo "Waiting for deployment/nighthawk-server in ate-system to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..." - "${kubectl_cmd[@]}" -n ate-system rollout status deployment/nighthawk-server --timeout="${WAIT_TIMEOUT_SECS}s" + "${kubectl_cmd[@]}" -n nighthawk rollout status deployment/nighthawk-server --timeout="${WAIT_TIMEOUT_SECS}s" echo "Waiting for deployment/traffic-driver-invoker in ate-system to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..." - "${kubectl_cmd[@]}" -n ate-system rollout status deployment/traffic-driver-invoker --timeout="${WAIT_TIMEOUT_SECS}s" + "${kubectl_cmd[@]}" -n nighthawk rollout status deployment/traffic-driver-invoker --timeout="${WAIT_TIMEOUT_SECS}s" elif [[ "${action}" == "delete" ]]; then echo "Deleting Nighthawk server and traffic-driver-invoker..." "${kubectl_cmd[@]}" delete --ignore-not-found -f "${INVOKER_MANIFEST}" -f "${SERVER_MANIFEST}" diff --git a/benchmarking/nighthawk-egress/traffic-driver-invoker.yaml b/benchmarking/nighthawk-egress/traffic-driver-invoker.yaml index dd224690a5..4b37bc243d 100644 --- a/benchmarking/nighthawk-egress/traffic-driver-invoker.yaml +++ b/benchmarking/nighthawk-egress/traffic-driver-invoker.yaml @@ -14,7 +14,9 @@ # In-cluster Envoy proxy that listens for plaintext HTTP/1.1 and HTTP/2 # (h2c/gRPC) on :19980, injects `ate-target-actor: nighthawk/tester` on every -# request, and forwards upstream to atenet-router.ate-system.svc.cluster.local:80. +# request, and forwards upstream over mTLS to +# atenet-router.ate-system.svc.cluster.local:443 using a SPIFFE client +# certificate issued by podidentity.podcert.ate.dev/identity. # # TODO(yanavlasov): this is a temporary workaround for nighthawk's cli's inability # to add headers when invoking traffic driver gRPC service. So this Envoy adds the @@ -26,6 +28,11 @@ metadata: namespace: nighthawk data: envoy.yaml: | + # SDS requires a node id and cluster. + node: + id: traffic-driver-invoker + cluster: traffic-driver-invoker + static_resources: listeners: - name: listener_http_19980 @@ -76,6 +83,30 @@ data: use_downstream_protocol_config: http_protocol_options: {} http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + sni: atenet-router.ate-system.svc + common_tls_context: + tls_certificate_sds_secret_configs: + - name: podidentity_client_cert + sds_config: + resource_api_version: V3 + path_config_source: + path: /etc/envoy/sds-podidentity-cert.yaml + combined_validation_context: + default_validation_context: + match_typed_subject_alt_names: + - san_type: DNS + matcher: + exact: atenet-router.ate-system.svc + validation_context_sds_secret_config: + name: servicedns_validation_context + sds_config: + resource_api_version: V3 + path_config_source: + path: /etc/envoy/sds-servicedns-validation.yaml load_assignment: cluster_name: atenet_router endpoints: @@ -84,7 +115,22 @@ data: address: socket_address: address: atenet-router.ate-system.svc.cluster.local - port_value: 80 + port_value: 443 + sds-podidentity-cert.yaml: | + resources: + - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret + name: podidentity_client_cert + tls_certificate: + certificate_chain: { filename: /run/podidentity.podcert.ate.dev/credential-bundle.pem } + private_key: { filename: /run/podidentity.podcert.ate.dev/credential-bundle.pem } + watched_directory: { path: /run/podidentity.podcert.ate.dev } + sds-servicedns-validation.yaml: | + resources: + - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret + name: servicedns_validation_context + validation_context: + trusted_ca: { filename: /run/servicedns.podcert.ate.dev/trust-bundle.pem } + watched_directory: { path: /run/servicedns.podcert.ate.dev } --- apiVersion: apps/v1 kind: Deployment @@ -107,6 +153,7 @@ spec: runAsUser: 65532 runAsGroup: 65532 runAsNonRoot: true + fsGroup: 65532 containers: - name: envoy # Same envoyproxy/envoy digest as cmd/dataplane/envoy/Dockerfile. @@ -136,10 +183,32 @@ spec: - name: config mountPath: /etc/envoy readOnly: true + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: servicedns-ca + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true volumes: - name: config configMap: name: traffic-driver-invoker + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem --- apiVersion: v1 kind: Service diff --git a/benchmarking/nighthawk-ingress/README.md b/benchmarking/nighthawk-ingress/README.md index 46ad4e9fda..d5a94ac329 100644 --- a/benchmarking/nighthawk-ingress/README.md +++ b/benchmarking/nighthawk-ingress/README.md @@ -120,7 +120,9 @@ requests 2×`envoyCpu` CPUs; the runner requests `clientConcurrency`+1). ```bash # .ate-dev-env.sh at the repo root, then: -hack/install-ate.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' +hack/install-ate.sh deploy ate-system \ + --credential-provider='{"name":"k8s.io"}' \ + --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/benchmarking/sa/benchmark-runner benchmarking/workloads/deploy.sh --deploy --worker-count 50 --sandbox-class gvisor ``` @@ -147,7 +149,7 @@ Flags: `--envoy-cpu`, `--actors`, `--tail-latency-slo-ms`, `--atespace`, Notes: - The script does not redeploy substrate — if substrate code changed, - rerun `hack/install-ate.sh --deploy-ate-system` first. + rerun `hack/install-ate.sh deploy ate-system` first. - `--atespace` isolates actor *fleets* between experiments sharing a cluster; capacity runs themselves still need the router exclusively (the cpu pin is global and the run saturates it). diff --git a/benchmarking/nighthawk-ingress/run-dev.sh b/benchmarking/nighthawk-ingress/run-dev.sh index a905597909..e20dd51e1a 100755 --- a/benchmarking/nighthawk-ingress/run-dev.sh +++ b/benchmarking/nighthawk-ingress/run-dev.sh @@ -80,7 +80,7 @@ docker info >/dev/null 2>&1 || { kubectl get deployment atenet-router -n ate-system >/dev/null 2>&1 || { echo "ERROR: substrate is not deployed on ${CLUSTER_NAME}. Run:" >&2 - echo " hack/install-ate.sh --deploy-ate-system --credential-provider='{\"name\":\"k8s.io\"}'" >&2 + echo " hack/install-ate.sh deploy ate-system --credential-provider='{\"name\":\"k8s.io\"}' --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/benchmarking/sa/benchmark-runner" >&2 exit 1 } diff --git a/benchmarking/telemetry/README.md b/benchmarking/telemetry/README.md index c5aa2c431b..a5d991b037 100644 --- a/benchmarking/telemetry/README.md +++ b/benchmarking/telemetry/README.md @@ -37,7 +37,11 @@ Install the meter, then send the control plane to it: kubectl apply -f benchmarking/telemetry/meter.yaml METER=http://telemetry-meter.benchmarking.svc.cluster.local:4317 -./hack/install-ate.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' --otlp-endpoint "${METER}" + +go run ./cmd/ate-setup deploy ate-system \ + --credential-provider='{"name":"k8s.io"}' \ + --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/benchmarking/sa/default \ + --otlp-endpoint "${METER}" ``` `--otlp-endpoint` patches the `ate-otel-config` ConfigMap and restarts the @@ -223,7 +227,9 @@ upgrade. ## Remove the meter ```bash -./hack/install-ate.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' +go run ./cmd/ate-setup deploy ate-system \ + --credential-provider='{"name":"k8s.io"}' \ + --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/benchmarking/sa/default ./benchmarking/workloads/deploy.sh --deploy kubectl delete -f benchmarking/telemetry/meter.yaml ``` diff --git a/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml index 56e30286c5..3e7c7e085f 100644 --- a/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml +++ b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml @@ -91,8 +91,8 @@ spec: keyed by name. items: description: |- - SandboxVersionConfig is one version of a SandboxConfig: the exact pause image - and asset set a sandbox boots with. + SandboxVersionConfig is one version of a SandboxConfig: the asset set a + sandbox boots with. properties: assets: additionalProperties: @@ -142,26 +142,6 @@ spec: minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string - pauseImage: - description: |- - pauseImage is the container image used as the root sandbox container. - It holds the sandbox's namespaces and runs no workload code, so it is an - implementation detail of the sandbox rather than something actor authors - choose. Required for gvisor; not allowed for microvm, which runs no pause - container. It is captured in the snapshot manifest alongside the sandbox - binaries, so a restore always re-creates the sandbox from the same image - the snapshot was taken with. - - Typically, set it to [1] for on-gcp, and [2] for off-gcp - - - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da - - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 - maxLength: 512 - minLength: 1 - type: string - x-kubernetes-validations: - - message: All images must include a digest - rule: self.contains('@') state: default: Enabled description: |- @@ -190,11 +170,6 @@ spec: - message: the version named by defaultVersion must not be Disabled rule: '!self.versions.exists(v, v.name == self.defaultVersion && has(v.state) && v.state == ''Disabled'')' - - message: pauseImage is required on every version for gvisor and not - allowed for other sandbox classes - rule: 'self.sandboxClass == ''gvisor'' ? self.versions.all(v, has(v.pauseImage) - && size(v.pauseImage) > 0) : self.versions.all(v, !has(v.pauseImage) - || size(v.pauseImage) == 0)' required: - spec type: object diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 25d09ca30b..fcb0f6ac84 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -71,7 +71,6 @@ See `values.yaml` for the full set; the important keys: | `objectStorePlugin.extraEnv` | `[]` | Object storage credentials/settings shared by both object-store plugins | | `objectStorePlugin.control.resources` / `objectStorePlugin.node.resources` | see `values.yaml` | Resources for the API and node object-store sidecars | | `ateApi.logLevel` | `info` | API server log level in its mounted config; one of `debug`, `info`, `warn`, or `error` | -| `ateApi.authorization.enabled` | `true` | Enforce API authorization | | `ateApi.authorization.bootstrapOwners` | `[]` | Global owners; empty defaults to the release's shared `ate-client` ServiceAccount | | `ateApi.extraArgs` | `[]` | Additional command-line arguments appended to the ateapi defaults | | `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces, metrics, the actor lifecycle events and the router access log | @@ -92,8 +91,9 @@ memory values. Worker images are now named `ateworker-gvisor` and `ateworker-microvm`; update custom WorkerPools and image mirrors accordingly. Upgrade `substrate-crds` before the application chart. Custom `SandboxConfig` -resources now put assets and `pauseImage` under `spec.versions[]`, with a named -`spec.defaultVersion`. WorkerPools select the runtime through +resources now put assets under `spec.versions[]`, with a named +`spec.defaultVersion`. Remove `pauseImage` from custom configs; gVisor no longer +uses a pause container. WorkerPools select the runtime through `spec.sandboxClasses[]`; remove `sandboxConfig` from actor templates. WorkerPools now resolve `spec.sandboxClasses[].configRef` or the class default @@ -178,7 +178,8 @@ The API server now requires `--config-file`. The chart mounts its generated `ate-api-config` ConfigMap and rolls the API pods when `ateApi.logLevel` changes. Replace `--log-level` in `ateApi.extraArgs` with `ateApi.logLevel`. -API authorization is enabled by default, matching `ate-setup`. The release's +API authorization is always enforced, matching `ate-setup`. Remove +`ateApi.authorization.enabled` from custom values. The release's shared `ate-client` ServiceAccount is a bootstrap global owner so demos and CLI access work. For shared installs, set `ateApi.authorization.bootstrapOwners` to your operators' own principals and grant tenant access with `kubectl ate @@ -195,6 +196,8 @@ The pinned agentgateway image includes the updated `Actor.status`, actor-state, and `MintActorJwtRequest` bindings. If you override `images.agentgateway`, use these bindings too; older images cannot resolve actor routes or mint actor JWTs against this API. +The image also preserves the requested actor port across HTTP/1 and HTTP/2 +CONNECT tunnels. Every checkpoint requires an object-storage URI, including a pause that keeps its snapshot local. The chart sets the cluster-wide snapshot location diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index 50b782710f..49d08bd0b4 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -25,6 +25,9 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["get", "watch", "list"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] - apiGroups: ["ate.dev"] resources: ["workerpools", "csidriverconfigs"] verbs: ["get", "watch", "list"] @@ -140,7 +143,6 @@ spec: - "--grpc-listen-addr=0.0.0.0:443" - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" - --config-file=/run/apiconfig.ate.dev/apiconfig.textproto - - {{ printf "--experimental-enable-authz=%t" .Values.ateApi.authorization.enabled | quote }} - {{ printf "--authz-bootstrap-owners=%s" (join "," (.Values.ateApi.authorization.bootstrapOwners | default (list (printf "system:serviceaccount:%s:%s" .Release.Namespace (include "substrate.fullname" (list "ate-client" .)))))) | quote }} - "--authentication-config=/etc/ateapi/authentication/authentication.yaml" - "--postgres-read-write-connection-string=@env" diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml index 3c5d15bd20..42d0ed60e3 100644 --- a/charts/substrate/templates/atelet.yaml +++ b/charts/substrate/templates/atelet.yaml @@ -148,6 +148,8 @@ spec: - --root=/var/lib/ate - --asset-staging-dir=/var/lib/objectstore-plugin/asset-staging - --asset-staging-capacity=8Gi + - '--exchange-address={{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:8445' + - '--exchange-server-name={{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc' startupProbe: exec: command: @@ -164,6 +166,10 @@ spec: resources: {{ toYaml .Values.objectStorePlugin.node.resources | indent 10 }} env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace {{- include "substrate.storage.env" . | nindent 8 }} {{- with include "substrate.otel.env" . }} {{- . | trim | nindent 8 }} @@ -171,10 +177,14 @@ spec: volumeMounts: - name: ate-base mountPath: /var/lib/ate + mountPropagation: HostToContainer - name: objectstore-plugin mountPath: /run/objectstore-plugin - name: asset-staging mountPath: /var/lib/objectstore-plugin/asset-staging + - name: servicedns-ca + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true containers: - name: atelet image: {{ include "substrate.componentImage" (list "atelet" .) }} @@ -254,6 +264,7 @@ spec: mountPath: /run/objectstore-plugin - name: ate-base mountPath: /var/lib/ate + mountPropagation: HostToContainer - name: podidentity mountPath: /run/podidentity.podcert.ate.dev readOnly: true diff --git a/charts/substrate/templates/sandboxconfig-gvisor.yaml b/charts/substrate/templates/sandboxconfig-gvisor.yaml index 1f41aab78d..888f7f68ce 100644 --- a/charts/substrate/templates/sandboxconfig-gvisor.yaml +++ b/charts/substrate/templates/sandboxconfig-gvisor.yaml @@ -21,6 +21,10 @@ limitations under the License. # it) matching the worker node's architecture and extracts it locally. To pin a # different release, add a version below and point defaultVersion at it, or # create another SandboxConfig and reference it from the WorkerPool. +# +# A bump adds a version and points defaultVersion at it. Keep the +# previous version unless it has a known vulnerability: an actor restores its +# memory only on the version its snapshot was taken with. apiVersion: ate.dev/v1alpha1 kind: SandboxConfig metadata: @@ -34,12 +38,10 @@ spec: # gVisor nightly 2026-10-17, the first to include google/gvisor@b79256080dec # (save and restore multiple filesystem checkpoint bundles). ateworker-gvisor # uses it to save container rootfs overlays (rootfs/) and durable-dir volumes - # (fs/) as separate bundles. + # (fs/) as separate bundles, and to run the sandbox with no root container + # (google/gvisor#15551). - name: v4 state: Enabled - # The root sandbox container's image. On GCP, prefer the in-project mirror - # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da. - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" assets: amd64: gvisor: diff --git a/charts/substrate/tests/control_plane_test.yaml b/charts/substrate/tests/control_plane_test.yaml index b984949e8a..a803cdd54e 100644 --- a/charts/substrate/tests/control_plane_test.yaml +++ b/charts/substrate/tests/control_plane_test.yaml @@ -189,9 +189,6 @@ tests: - contains: path: spec.template.spec.containers[0].args content: --config-file=/run/apiconfig.ate.dev/apiconfig.textproto - - contains: - path: spec.template.spec.containers[0].args - content: --experimental-enable-authz=true - contains: path: spec.template.spec.containers[0].args content: --authz-bootstrap-owners=system:serviceaccount:custom-system:custom-ate-client @@ -207,19 +204,15 @@ tests: name: api-config configMap: name: custom-ate-api-config -- it: configures operator ownership even with runtime enforcement disabled +- it: configures operator ownership template: ate-api-server.yaml set: - ateApi.authorization.enabled: false ateApi.authorization.bootstrapOwners: - system:serviceaccount:operators:admin documentSelector: path: kind value: Deployment asserts: - - contains: - path: spec.template.spec.containers[0].args - content: --experimental-enable-authz=false - contains: path: spec.template.spec.containers[0].args content: --authz-bootstrap-owners=system:serviceaccount:operators:admin @@ -274,7 +267,7 @@ tests: value: ClusterRole asserts: - equal: - path: rules[1] + path: rules[2] value: apiGroups: [ate.dev] resources: [workerpools, csidriverconfigs] diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index 1e9295454f..5091635ba5 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -113,7 +113,6 @@ ateApi: snapshotLocation: "" logLevel: info authorization: - enabled: true # Empty defaults to the release's shared ate-client principal. Use explicit # operator principals for shared installs; see docs/authorization.md. bootstrapOwners: [] @@ -233,5 +232,5 @@ image: images: rustfs: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 awsCli: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 - agentgateway: ghcr.io/kagent-dev/substrate/agentgateway:b736d06f88eb@sha256:39898817ed3e94b78570d20862fec06f32142fe8dee8691070128e74b5290407 + agentgateway: ghcr.io/kagent-dev/substrate/agentgateway:cd65181fd52e@sha256:ebebf5a7fa2a54fb524ccce1f6f74ee6e7ef22d661f22e212717fd0a68dadb33 busybox: busybox:1.36 diff --git a/cmd/ate-setup/cli-diff.md b/cmd/ate-setup/cli-diff.md index da775db851..5f538cd2a9 100644 --- a/cmd/ate-setup/cli-diff.md +++ b/cmd/ate-setup/cli-diff.md @@ -1,171 +1,4 @@ # `hack/install-ate.sh` after the move to `ate-setup` -`hack/install-ate.sh` no longer installs anything. It is a translation shim: it -parses the flags and environment variables the installer has always accepted -and runs `go run ./cmd/ate-setup ` for each action, in command-line -order. [`commands.md`](commands.md) has the flag-by-flag mapping; -[`differences.md`](differences.md) explains how the Go implementation differs -from the shell one it replaced. - -This document is for the person whose command line or CI job already worked. -Everything not listed here behaves as it did. What is listed here either could -not be reproduced through the shim, or was reproduced closely enough to keep -scripts working but not closely enough to keep a sharp-eyed reader from -noticing. - -## What the shim preserves - -Deliberately, and covered by `shim_test.go`: - -- Every flag, with the same spelling, the same `=value` and separated-value - forms, and the same "value flags may appear anywhere on the line" pre-scan. -- Several actions per invocation, executed in the order they were written. -- Every environment variable, including `SETUP_CSI`, `STORAGE_CLASS`, - `NO_DEV_ENV`, `ATE_INSTALL_KIND`, `KUBECTL_CONTEXT`, `PROJECT_ID`, and the - `ATE_API_POSTGRES_*` set. `ate-setup` reads them directly, so they need no - translation. -- `--help` / `-h` anywhere on the line, no arguments at all (usage, exit 1), - and an unrecognized flag (`Error: unknown option: …`, usage, exit 1) — - including after earlier actions on the same line have already run. -- `--deploy-ate-system --setup-csi=nfs` still sets the CSI driver up twice, - once inside the system deploy and once as its own action, as it always has. - -## Differences you can observe - -### One process per action - -Each action is a separate `go run ./cmd/ate-setup`. The build is cached after -the first, but a line asking for six actions pays six process starts, and each -one re-resolves the configuration — including, on GKE with `PROJECT_ID` set and -no `KUBECTL_CONTEXT`, a `gcloud container clusters get-credentials` per action -rather than one for the whole line. - -`go` is therefore required to run the installer. In exchange, `kubectl`, `jq`, -`openssl`, `sed`, and `base64` are not; see the external-binaries table -in [`differences.md`](differences.md#external-binaries) for what is still -needed. - -### Value flags are validated only when an action uses them - -The shell pre-scan validated `--atenet-dataplane`, `--podcert-workers-per-signer`, -`--rollout-timeout`, and `--benchmark-sandbox-class` on every run, whether or -not anything used them. Validation now happens inside `ate-setup`, so: - -- A line with no action at all — `./hack/install-ate.sh --rollout-timeout=zzz` - — exits 0 without complaining. It used to exit 1. -- `--benchmark-sandbox-class=zzz` is rejected only by `--deploy-benchmarks` / - `--delete-benchmarks`. Paired with any other action it is ignored, where it - used to fail the run up front. - -The four flags that apply to every action (`--atenet-dataplane`, -`--podcert-workers-per-signer`, `--rollout-timeout`, `--otlp-endpoint`) are -passed to every `ate-setup` invocation, so as long as the line has one action -they are still rejected before that action touches the cluster. - -### `--deploy-demo-autoscaled-workerpool` fails later, off Kind - -The demo registered itself only under `ATE_INSTALL_KIND=true`, so on GKE the -flag was rejected as an unknown option, before anything ran, and `--help` did -not list it. The cobra command tree is built before flags are parsed, so the -subcommand now always exists: `--help` always lists the demo, and off Kind it -fails when it runs, with `demo-autoscaled-workerpool is only supported for Kind -installations; re-run with --kind`. Earlier actions on the same line will -already have completed. `--delete-all` still skips it off Kind. - -### The demo list lives in two places - -`ATE_DEMOS` in the shim is a literal list, so that `--help` answers without a -Go build and an unknown demo is rejected here rather than reaching `ate-setup` -as an unknown subcommand. A demo added to `internal/demos/all` and not to that -list is invisible to the shim; `TestShimDemoListMatchesTheRegistry` fails when -the two drift. - -### Two step log lines changed - -`log.Step` names are otherwise unchanged, but two lost a parenthesized -qualifier that CI log scrapers may be matching on: - -| before | now | -|---|---| -| `demo-counter_deploy (with_external_volume=true)` | `demo-counter_deploy` | -| `setup_csi (nfs)` | `setup_csi` | - -### `--setup-csi` on its own does more - -The standalone action used to ensure the CRDs and then install the driver. -`ate-setup setup csi` also ensures the `ate-system` namespace, the -podcertificate CAs, and a ready podcertificate-controller with its trust -bundles first. The hostpath driver's ghostunnel sidecar projects a -podCertificate volume and cannot roll out without them, which is why -`deploy_ate_system` had always called CSI setup from that point in its -sequence; the standalone path just did not. - -### `.ate-dev-env.sh` is layered, not sourced over you - -The script sourced the file into its own shell, so an `export FOO=...` in the -file won over a `FOO` already exported in yours. `ate-setup` still runs the -file through bash — it can contain arbitrary shell — but treats the result as -the lowest-precedence layer: process environment first, then flags. If you -have been relying on the file to override your shell, unset the variable in -your shell instead. - -Sourcing is also skipped for Kind installs. `hack/install-ate-kind.sh` exports -`NO_DEV_ENV`, so that path is unchanged, but running the shim yourself with -`ATE_INSTALL_KIND=true` no longer reads the file. - -### `--rollout-timeout` reaches further - -It used to govern only the 60s workload rollout waits. It now also governs the -two waits fixed at 120s (podcertificate-controller and the CSI drivers) — -but only when it is passed, so the 60s default still cannot shorten those slow -bootstrap paths. - -It also has to be positive. The value reached `kubectl rollout status ---timeout=`, where `0` means "wait forever"; `ate-setup` has no unbounded wait, -so `--rollout-timeout=0` is rejected at startup rather than read as its -opposite. - -### Applies are server-side - -`kubectl apply -f -` was a client-side apply. `ate-setup` uses server-side -apply with field manager `ate-setup` and `force: true`. Objects previously -installed by the shell script show both managers in `managedFields` until the -next apply reconciles them. `last-applied-configuration` stops growing on the -generated CRDs. - -### The first install after this change rolls `ate-api-server` once - -Both installers stamp `ate.dev/env-hash` on the pod template so a changed -environment starts a rollout, but they compute the digest differently. The -value is opaque, so the only consequence is one extra rollout the first time -`ate-setup` installs over a shell-installed cluster. - -### `--create-*-ca-pool-secret` is idempotent - -Against a cluster that already has the pool, these used to fail with -`AlreadyExists`. They now log `already exists; keeping it` and succeed. They -still refuse to overwrite: regenerating would rotate the root out from under -every certificate already issued from it. - -### `--setup-csi=hostpath` and `=both` off Kind are hard errors - -They used to warn and continue. Only the hostpath plugin is patched for the -single-node Kind layout; `nfs` is accepted anywhere, as before. - -### The per-demo scripts are gone - -`hack/install-demo-*.sh` and `hack/experimental-additional-egress-extproc.sh` -have been deleted — each demo is a Go package under -`cmd/ate-setup/internal/demos/` now. Nothing that goes through -`hack/install-ate.sh` notices, but anything sourcing those files directly, or -calling a `demo-…_deploy` shell function, has to move to -`go run ./cmd/ate-setup deploy demo `. - -## Not replicated, and not worth replicating - -Two behaviors of the old script were bugs that the shim does not reproduce: - -- `--delete-atenet` worked but was missing from `--help`. It is listed now. -- `--deploy-demo-jupyter` was printed twice in `--help`, and the - `demo-counter-microvm` flags were printed inside `demo-counter`'s block. - Each demo now gets exactly one block. +`hack/install-ate.sh` no longer installs anything. It merely directly invokes +`go run ./cmd/ate-setup`. New logic should not be added to it. \ No newline at end of file diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index a32674c2a2..54a80ea7f1 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -205,7 +205,6 @@ See | `deploy demo sandbox` | `--deploy-demo-sandbox` | An on-demand sandbox actor driven by the sandbox client | | `deploy demo multi-template` | `--deploy-demo-multi-template` | Two ActorTemplates sharing one WorkerPool | | `deploy demo parking` | `--deploy-demo-parking` | Actor parking and unparking on a small WorkerPool | -| `deploy demo autoscaled-workerpool` | `--deploy-demo-autoscaled-workerpool` | A WorkerPool scaled by an HPA over custom metrics (Kind only) | | `deploy demo claude-code-multiplex` | `--deploy-demo-claude-code-multiplex` | Several Claude Code agents multiplexed onto one WorkerPool (requires `ANTHROPIC_API_KEY`, `BUCKET_NAME`, `KO_DOCKER_REPO`) | Each has a matching `delete demo NAME` / `--delete-demo-NAME`. Demo flags bind diff --git a/cmd/ate-setup/differences.md b/cmd/ate-setup/differences.md index 20f60de36c..e54eb99e31 100644 --- a/cmd/ate-setup/differences.md +++ b/cmd/ate-setup/differences.md @@ -121,9 +121,9 @@ The rewritten reference then keeps that tag and adds the digest it names, as tagged release. Resolving the digest is one HEAD request per image, cached for the install, so `--image-repo` needs read access to the registry from wherever `ate-setup` runs rather than only from the cluster. It is not optional: an -ActorTemplate's container image, an image volume's reference, and a -SandboxConfig's `pauseImage` each carry the CEL rule `self.contains('@')`, so an -unpinned reference is rejected by admission and every demo fails to deploy. +ActorTemplate's container image and an image volume's reference each carry the +CEL rule `self.contains('@')`, so an unpinned reference is rejected by admission +and every demo fails to deploy. Pinning also gives a pre-built install the property a `ko` install had for free, that a tag moving afterwards cannot change what is running. @@ -232,14 +232,6 @@ demo is its own package registering itself from `init`, and `internal/demos/all` is the single list of what gets linked in — with a test asserting no demo package is missing from it. -**Kind-only demos are visible but refuse.** `demo-autoscaled-workerpool` -registered itself only when `ATE_INSTALL_KIND=true`, so on GKE -`--deploy-demo-autoscaled-workerpool` was rejected as an *unknown option*. The -cobra tree is built before flags are parsed, so `--kind` is not known yet; the -subcommand always exists and fails at run time with `demo-autoscaled-workerpool -is only supported for Kind installations; re-run with --kind`. `delete all` -still skips it on a non-Kind install, via `demos.For`. - **Template expansion is not sed.** The demo templates were expanded by a stack of `sed -e` expressions doing two things: substitute `${PLACEHOLDER}`, or delete the whole line (`/${NAME}/d`) when the placeholder is not in play. diff --git a/cmd/ate-setup/internal/cmd/delete.go b/cmd/ate-setup/internal/cmd/delete.go index 7bb9df466e..dab32dd0e6 100644 --- a/cmd/ate-setup/internal/cmd/delete.go +++ b/cmd/ate-setup/internal/cmd/delete.go @@ -48,7 +48,7 @@ var deleteAllCmd = &cobra.Command{ Short: "Delete every demo and then the core system", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { - return env.DeleteAll(cmd.Context(), demos.Deleters(env.Cfg)) + return env.DeleteAll(cmd.Context(), demos.Deleters()) }, } diff --git a/cmd/ate-setup/internal/cmd/deploy.go b/cmd/ate-setup/internal/cmd/deploy.go index 45b35a3014..76d3f12ff0 100644 --- a/cmd/ate-setup/internal/cmd/deploy.go +++ b/cmd/ate-setup/internal/cmd/deploy.go @@ -60,7 +60,7 @@ ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING or the ATE_API_POSTGRES_CLOUDSQL_* select an external database. Shape the install with the global --atenet-dataplane, --cluster-size, ---cordon-control-plane, --fake-data-plane, and --ingress-auth-mode flags.`, +--cordon-control-plane, and --fake-data-plane flags.`, // Args runs before the root command loads the configuration, so only the // flag is readable here. Checking it keeps an unusable value from costing // a credential fetch, which is the common case; a value arriving from the diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index e497c6ebbf..50df2171ec 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -50,11 +50,6 @@ const ( // the podcertificate controller's API rate limits to match. ClusterSizeSize0 = "size0" ClusterSizeSize10 = "size10" - - // Ingress router authentication modes; see atenet router's - // --ingress-auth-mode. - IngressAuthDeprecatedInsecure = "deprecated-insecure" - IngressAuthStaticMTLS = "static-mtls" ) // DefaultRolloutTimeout is the default wait timeout for workload rollouts. @@ -122,9 +117,6 @@ type Config struct { // follows neither form. ExpectedJWTIssuer string - // APIExperimentalEnableAuthz turns on ate-api-server's runtime - // authorization (ATE_API_EXPERIMENTAL_ENABLE_AUTHZ). - APIExperimentalEnableAuthz bool // APIAuthzBootstrapOwners is the comma-separated principal list granted // owner on global:root (ATE_API_AUTHZ_BOOTSTRAP_OWNERS). APIAuthzBootstrapOwners string @@ -218,11 +210,6 @@ type Config struct { // (ATE_INSTALL_FAKE_DATA_PLANE), for control-plane benchmarks. FakeDataPlane bool - // IngressAuthMode is how the ingress router authenticates its clients - // (ATE_INGRESS_AUTH_MODE): IngressAuthDeprecatedInsecure or - // IngressAuthStaticMTLS. - IngressAuthMode string - // IngressAllowedSPIFFEIDs are the SPIFFE IDs the ingress router lets // reach actors over its TLS listeners (ATE_INGRESS_ALLOWED_SPIFFE_IDS). // See RequireIngressAllowedSPIFFEIDs. @@ -497,10 +484,6 @@ func (c *Config) ScriptEnv() []string { if c.FakeDataPlane { merged["ATE_INSTALL_FAKE_DATA_PLANE"] = "true" } - delete(merged, "ATE_INGRESS_AUTH_MODE") - if c.IngressAuthMode != "" && c.IngressAuthMode != IngressAuthDeprecatedInsecure { - merged["ATE_INGRESS_AUTH_MODE"] = c.IngressAuthMode - } delete(merged, "ATE_INGRESS_ALLOWED_SPIFFE_IDS") if len(c.IngressAllowedSPIFFEIDs) > 0 { merged["ATE_INGRESS_ALLOWED_SPIFFE_IDS"] = strings.Join(c.IngressAllowedSPIFFEIDs, ",") diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index 1e887b4f45..d39d53ab7c 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -48,7 +48,6 @@ func loadEnv(t *testing.T) { "ANTHROPIC_API_KEY", "ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE", "ATE_API_AUTHZ_BOOTSTRAP_OWNERS", - "ATE_API_EXPERIMENTAL_ENABLE_AUTHZ", "ATE_API_LOG_LEVEL", "ATE_API_POSTGRES_CLOUDSQL_GSA", "ATE_API_POSTGRES_CLOUDSQL_IAM_AUTH", @@ -65,7 +64,6 @@ func loadEnv(t *testing.T) { "ATE_IMAGE_REPO", "ATE_IMAGE_TAG", "ATE_INGRESS_ALLOWED_SPIFFE_IDS", - "ATE_INGRESS_AUTH_MODE", "ATE_INSTALL_CLUSTER_SIZE", "ATE_INSTALL_CORDON_CONTROL_PLANE", "ATE_INSTALL_FAKE_DATA_PLANE", @@ -133,9 +131,6 @@ func TestLoadDefaults(t *testing.T) { if cfg.CordonControlPlane { t.Error("CordonControlPlane = true, want false") } - if cfg.IngressAuthMode != IngressAuthDeprecatedInsecure { - t.Errorf("IngressAuthMode = %q, want %q", cfg.IngressAuthMode, IngressAuthDeprecatedInsecure) - } } func TestLoadClusterSize(t *testing.T) { @@ -251,39 +246,6 @@ func TestLoadFakeDataPlane(t *testing.T) { }) } } -func TestLoadIngressAuthMode(t *testing.T) { - for _, tc := range []struct { - name string - opts Options - env string - want string - wantExport bool - }{ - {name: "flag static-mtls", opts: Options{IngressAuthMode: IngressAuthStaticMTLS}, want: IngressAuthStaticMTLS, wantExport: true}, - {name: "environment static-mtls", env: IngressAuthStaticMTLS, want: IngressAuthStaticMTLS, wantExport: true}, - {name: "environment deprecated-insecure", env: IngressAuthDeprecatedInsecure, want: IngressAuthDeprecatedInsecure}, - {name: "flag outranks environment", opts: Options{IngressAuthMode: IngressAuthDeprecatedInsecure}, env: IngressAuthStaticMTLS, want: IngressAuthDeprecatedInsecure}, - } { - t.Run(tc.name, func(t *testing.T) { - loadEnv(t) - if tc.env != "" { - t.Setenv("ATE_INGRESS_AUTH_MODE", tc.env) - } - cfg, err := Load(tc.opts) - if err != nil { - t.Fatalf("Load() error = %v", err) - } - if cfg.IngressAuthMode != tc.want { - t.Errorf("IngressAuthMode = %q, want %q", cfg.IngressAuthMode, tc.want) - } - got, exported := scriptEnvMap(t, cfg)["ATE_INGRESS_AUTH_MODE"] - if exported != tc.wantExport || (exported && got != tc.want) { - t.Errorf("ScriptEnv() ATE_INGRESS_AUTH_MODE = %q (exported %v), want %q (exported %v)", got, exported, tc.want, tc.wantExport) - - } - }) - } -} func TestLoadIngressAllowedSPIFFEIDs(t *testing.T) { const a, b = "spiffe://cluster.local/ns/a/sa/a", "spiffe://cluster.local/ns/b/sa/b" @@ -333,7 +295,6 @@ func TestRequireIngressAllowedSPIFFEIDs(t *testing.T) { wantErr bool }{ {name: "envoy without IDs", cfg: Config{Router: RouterEnvoy}, wantErr: true}, - {name: "envoy static-mtls without IDs", cfg: Config{Router: RouterEnvoy, IngressAuthMode: IngressAuthStaticMTLS}, wantErr: true}, {name: "envoy with IDs", cfg: Config{Router: RouterEnvoy, IngressAllowedSPIFFEIDs: []string{"spiffe://cluster.local/ns/a/sa/a"}}}, {name: "agentgateway without IDs", cfg: Config{Router: RouterAgentgateway}}, } { @@ -680,36 +641,27 @@ func TestLoadAPIAuthz(t *testing.T) { owners = "alice@example.com,bob@example.com" ) for _, tc := range []struct { - name string - env map[string]string - wantEnabled bool - wantOwners string - wantErr bool + name string + env map[string]string + wantOwners string + wantErr bool }{ { - name: "on by default, owned by the client ServiceAccount", - wantEnabled: true, - wantOwners: clientOwner, - }, - { - name: "default owner follows the namespace", - env: map[string]string{"ATE_NAMESPACE": "custom"}, - wantEnabled: true, - wantOwners: "system:serviceaccount:custom:ate-client", + name: "owned by the client ServiceAccount by default", + wantOwners: clientOwner, }, { - name: "explicit owners replace the default", - env: map[string]string{"ATE_API_AUTHZ_BOOTSTRAP_OWNERS": owners}, - wantEnabled: true, - wantOwners: owners, + name: "default owner follows the namespace", + env: map[string]string{"ATE_NAMESPACE": "custom"}, + wantOwners: "system:serviceaccount:custom:ate-client", }, { - name: "turned off", - env: map[string]string{"ATE_API_EXPERIMENTAL_ENABLE_AUTHZ": "false"}, - wantOwners: clientOwner, + name: "explicit owners replace the default", + env: map[string]string{"ATE_API_AUTHZ_BOOTSTRAP_OWNERS": owners}, + wantOwners: owners, }, { - name: "enabled with only separators", + name: "only separators", env: map[string]string{"ATE_API_AUTHZ_BOOTSTRAP_OWNERS": " , "}, wantErr: true, }, @@ -729,8 +681,8 @@ func TestLoadAPIAuthz(t *testing.T) { if err != nil { t.Fatalf("Load() error = %v", err) } - if cfg.APIExperimentalEnableAuthz != tc.wantEnabled || cfg.APIAuthzBootstrapOwners != tc.wantOwners { - t.Errorf("authz = %v, %q, want %v, %q", cfg.APIExperimentalEnableAuthz, cfg.APIAuthzBootstrapOwners, tc.wantEnabled, tc.wantOwners) + if cfg.APIAuthzBootstrapOwners != tc.wantOwners { + t.Errorf("APIAuthzBootstrapOwners = %q, want %q", cfg.APIAuthzBootstrapOwners, tc.wantOwners) } }) } @@ -1006,9 +958,6 @@ func TestLoadRejectsInvalidValues(t *testing.T) { {"kubernetes address without a port", Options{CredentialProvider: `{"name":"k8s.io","address":"secrets.ate-system.svc"}`}}, {"other provider without an address", Options{CredentialProvider: `{"name":"vault.example.com"}`}}, {"other provider address without a port", Options{CredentialProvider: `{"name":"vault.example.com","address":"vault.ate-system.svc"}`}}, - {"ingress auth mode", Options{IngressAuthMode: "mtls"}}, - {"old insecure ingress auth mode", Options{IngressAuthMode: "insecure"}}, - {"static-mtls agentgateway", Options{Router: RouterAgentgateway, IngressAuthMode: IngressAuthStaticMTLS}}, } { t.Run(tc.name, func(t *testing.T) { if _, err := Load(tc.opts); err == nil { diff --git a/cmd/ate-setup/internal/config/coverage_test.go b/cmd/ate-setup/internal/config/coverage_test.go index 23ad45729f..8f23e55ea5 100644 --- a/cmd/ate-setup/internal/config/coverage_test.go +++ b/cmd/ate-setup/internal/config/coverage_test.go @@ -90,8 +90,6 @@ func valueFor(key string) string { return RouterEnvoy case "clusterSize": return ClusterSizeSize0 - case "ingressAuthMode": - return IngressAuthStaticMTLS case "ingressAllowedSPIFFEIDs": return "spiffe://cluster.local/ns/a/sa/b" case "ateapi.postgres.cloudsql.ipType": diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index 0861321817..1b31c4fac1 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -212,8 +212,8 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro } // kubectl-ate, ate-setup's demo steps and the e2e clients fall back to // tokens minted from the client ServiceAccount, so making it the default - // bootstrap owner keeps them working with authorization on. The deploy - // warns whenever it is a bootstrap owner. + // bootstrap owner keeps them working. The deploy warns whenever it is a + // bootstrap owner. bootstrapOwners := r.String("ateapi.authz.bootstrapOwners") if bootstrapOwners == "" { bootstrapOwners = installdefaults.ClientPrincipal(r.String("namespace")) @@ -230,8 +230,7 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro ClusterLocation: r.String("gcp.clusterLocation"), ExpectedJWTIssuer: r.String("ateapi.expectedJWTIssuer"), - APIExperimentalEnableAuthz: r.Bool("ateapi.authz.enabled"), - APIAuthzBootstrapOwners: bootstrapOwners, + APIAuthzBootstrapOwners: bootstrapOwners, BucketName: r.String("storage.bucketName"), SnapshotLocation: r.String("storage.snapshotLocation"), @@ -275,7 +274,6 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro ClusterSize: r.String("clusterSize"), CordonControlPlane: r.Bool("cordonControlPlane"), FakeDataPlane: r.Bool("fakeDataPlane.enabled"), - IngressAuthMode: r.String("ingressAuthMode"), IngressAllowedSPIFFEIDs: splitList(r.String("ingressAllowedSPIFFEIDs")), AdditionalEgressExtprocService: r.String("atenet.egress.additionalExtprocService"), CredentialProviderJSON: r.String("atenet.egress.credentialProvider"), @@ -412,18 +410,6 @@ func validateResolved(cfg *Config, r *Resolved) error { return &InvalidError{Value: size, Want: ClusterSizeSize0 + " or " + ClusterSizeSize10} } - authMode, _ := r.Value("ingressAuthMode") - switch cfg.IngressAuthMode { - case IngressAuthDeprecatedInsecure: - case IngressAuthStaticMTLS: - if cfg.Router != RouterEnvoy { - return &ConflictError{A: authMode, B: dataplane, - Why: IngressAuthStaticMTLS + " requires dataplane " + RouterEnvoy} - } - default: - return &InvalidError{Value: authMode, Want: IngressAuthDeprecatedInsecure + " or " + IngressAuthStaticMTLS} - } - // The router rejects a malformed ID at startup; catch it before the // rollout does. for _, id := range cfg.IngressAllowedSPIFFEIDs { @@ -478,12 +464,11 @@ func validateResolved(cfg *Config, r *Resolved) error { return fmt.Errorf("%w (from %s)", err, provider.From.Describe(provider.Setting)) } } - // ate-api-server exits on this combination, so catch it before deploying. - if cfg.APIExperimentalEnableAuthz && strings.Trim(cfg.APIAuthzBootstrapOwners, " ,") == "" { - enabled, _ := r.Value("ateapi.authz.enabled") + // ate-api-server exits without a bootstrap owner, so catch it before + // deploying. + if strings.Trim(cfg.APIAuthzBootstrapOwners, " ,") == "" { owners, _ := r.Value("ateapi.authz.bootstrapOwners") - return &ConflictError{A: enabled, B: owners, - Why: "enforced authorization needs at least one bootstrap owner"} + return &InvalidError{Value: owners, Want: "a list of at least one bootstrap owner"} } return nil } diff --git a/cmd/ate-setup/internal/config/options_test.go b/cmd/ate-setup/internal/config/options_test.go index 4aac4df5ea..3e8fba4ffc 100644 --- a/cmd/ate-setup/internal/config/options_test.go +++ b/cmd/ate-setup/internal/config/options_test.go @@ -40,7 +40,6 @@ type Options struct { ClusterSize string CordonControlPlane bool FakeDataPlane bool - IngressAuthMode string AdditionalEgressExtprocService string CredentialProvider string OtlpEndpoint string @@ -89,7 +88,6 @@ func optionsFlagSet(opts Options) (*pflag.FlagSet, error) { {"atenet-dataplane", opts.Router}, {"rollout-timeout", opts.RolloutTimeout}, {"cluster-size", opts.ClusterSize}, - {"ingress-auth-mode", opts.IngressAuthMode}, {"experimental-additional-egress-extproc-service", opts.AdditionalEgressExtprocService}, {"credential-provider", opts.CredentialProvider}, {"otlp-endpoint", opts.OtlpEndpoint}, diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index f7d2cfd1ce..f3e0117c7a 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -177,15 +177,10 @@ var Registry = []Setting{ Kind: KindString, Usage: "Service account token issuer ate-api-server trusts, overriding derivation and discovery", }, - { - Key: "ateapi.authz.enabled", Env: "ATE_API_EXPERIMENTAL_ENABLE_AUTHZ", - Flag: "ateapi-experimental-enable-authz", Kind: KindBool, Default: "true", - Usage: "Enforce ate-api-server's runtime authorization", - }, { Key: "ateapi.authz.bootstrapOwners", Env: "ATE_API_AUTHZ_BOOTSTRAP_OWNERS", Flag: "ateapi-authz-bootstrap-owners", Kind: KindString, - Usage: "Comma-separated principals that always hold owner on global:root when authorization is enforced. " + + Usage: "Comma-separated principals that always hold owner on global:root. " + "Unset means kubectl-ate's ServiceAccount, system:serviceaccount::ate-client", }, { @@ -300,22 +295,14 @@ var Registry = []Setting{ "atelet, and fake-workersync in place of ate-controller. Actors do not exist; never " + "use it on a cluster that serves real actors", }, - { - Key: "ingressAuthMode", Env: "ATE_INGRESS_AUTH_MODE", Flag: "ingress-auth-mode", Kind: KindString, - Default: IngressAuthDeprecatedInsecure, - Usage: "How the ingress router authenticates clients: deprecated-insecure or static-mtls. Both check a " + - "client certificate on the router's TLS ports against the podidentity CA and --ingress-allowed-spiffe-ids; " + - "deprecated-insecure also lets through TLS clients without one and serves plaintext ports, while " + - "static-mtls requires the certificate and --atenet-dataplane=envoy", - }, { // Comma-separated, as the router's own flag is. The registry has no // list kind, so the value is carried as written and split where it is // used. Key: "ingressAllowedSPIFFEIDs", Env: "ATE_INGRESS_ALLOWED_SPIFFE_IDS", Flag: "ingress-allowed-spiffe-ids", Kind: KindString, - Usage: "SPIFFE IDs (exact match, comma-separated) of the clients the ingress router lets reach actors over " + - "its TLS ports; any listed client may reach any actor. Required to deploy the router with " + + Usage: "SPIFFE IDs (exact match, comma-separated) of the clients the ingress router lets reach actors; " + + "any listed client may reach any actor. Required to deploy the router with " + "--atenet-dataplane=envoy. The podidentity signer issues spiffe://cluster.local/ns//sa/", }, { diff --git a/cmd/ate-setup/internal/demos/all/all.go b/cmd/ate-setup/internal/demos/all/all.go index 49fec9da2c..11d7ba220a 100644 --- a/cmd/ate-setup/internal/demos/all/all.go +++ b/cmd/ate-setup/internal/demos/all/all.go @@ -21,7 +21,6 @@ package all import ( - _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/autoscaledworkerpool" _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/claudemultiplex" _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/counter" _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/countermicrovm" diff --git a/cmd/ate-setup/internal/demos/autoscaledworkerpool/autoscaledworkerpool.go b/cmd/ate-setup/internal/demos/autoscaledworkerpool/autoscaledworkerpool.go deleted file mode 100644 index 4b246f6b16..0000000000 --- a/cmd/ate-setup/internal/demos/autoscaledworkerpool/autoscaledworkerpool.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package autoscaledworkerpool installs the counter workload plus the -// custom-metrics stack that drives a HorizontalPodAutoscaler over its -// WorkerPool. -// -// The demo is Kind only: it ships its own prometheus-adapter and a Kind -// specific HPA, so it is not offered at all on a GKE install. -package autoscaledworkerpool - -import ( - "context" - - "go.ate.dev/substrate/cmd/ate-setup/internal/demos" - "go.ate.dev/substrate/cmd/ate-setup/internal/kube" - "go.ate.dev/substrate/cmd/ate-setup/internal/log" - "go.ate.dev/substrate/cmd/ate-setup/internal/steps" - "go.ate.dev/substrate/internal/resources" -) - -// namespace holds the demo workload, its prometheus-adapter, and the HPA; -// it doubles as the atespace holding the demo's ActorTemplate. -const namespace = "ate-demo-autoscaled-workerpool" - -// Add-ons that only make sense on Kind. -const ( - prometheusAdapterManifest = "demos/autoscaled-workerpool/prometheus-adapter.yaml" - hpaKindManifest = "demos/autoscaled-workerpool/hpa-kind.yaml" -) - -type demo struct { - demos.Substrate -} - -func init() { - demos.Register(&demo{Substrate: demos.Substrate{ - DemoName: "demo-autoscaled-workerpool", - Short: "A WorkerPool scaled by an HPA over custom metrics (Kind only)", - WorkerPoolManifest: "demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl", - Deployments: []steps.TemplateRef{{Atespace: namespace, Name: "counter"}}, - Templates: []demos.SubstrateTemplate{{ - Manifest: "demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl", - Ref: resources.ActorTemplateRef{Atespace: namespace, Name: "counter"}, - }}, - }}) -} - -func (d *demo) KindOnly() bool { return true } - -func (d *demo) Deploy(ctx context.Context, e *steps.Env) error { - if err := e.RequireKind(d.DemoName); err != nil { - return err - } - if err := d.Substrate.Deploy(ctx, e); err != nil { - return err - } - - log.Step("Deploying prometheus-adapter and HPA for kind...") - if err := e.Kube.ApplyPath(ctx, e.Cfg.Path(prometheusAdapterManifest)); err != nil { - return err - } - if err := e.Kube.RolloutStatus(ctx, kube.KindDeployment, namespace, - "prometheus-adapter", e.Cfg.WaitTimeout(steps.BootstrapTimeout)); err != nil { - return err - } - return e.Kube.ApplyPath(ctx, e.Cfg.Path(hpaKindManifest)) -} - -func (d *demo) Delete(ctx context.Context, e *steps.Env) error { - if err := e.RequireKind(d.DemoName); err != nil { - return err - } - // The HPA goes first so it cannot scale the pool back up while the - // workload is being removed. - if err := e.Kube.DeletePath(ctx, e.Cfg.Path(hpaKindManifest)); err != nil { - return err - } - if err := e.Kube.DeletePath(ctx, e.Cfg.Path(prometheusAdapterManifest)); err != nil { - return err - } - return d.Substrate.Delete(ctx, e) -} diff --git a/cmd/ate-setup/internal/demos/demos.go b/cmd/ate-setup/internal/demos/demos.go index ff99a65f92..44a14c23ea 100644 --- a/cmd/ate-setup/internal/demos/demos.go +++ b/cmd/ate-setup/internal/demos/demos.go @@ -54,11 +54,6 @@ type Demo interface { Delete(ctx context.Context, e *steps.Env) error } -// kindOnlyDemo is implemented by demos that cannot run on GKE. -type kindOnlyDemo interface { - KindOnly() bool -} - // registry holds every demo, keyed by name. var registry = map[string]Demo{} @@ -97,30 +92,13 @@ func All() []Demo { return list } -// For returns the demos that apply to cfg, sorted by name. -// -// Kind-only demos are filtered out for non-Kind installs, mirroring how the -// autoscaled-workerpool demo only registered itself with the shell installer -// when ATE_INSTALL_KIND was true. DeleteAll iterates this, so a GKE teardown -// does not try to delete resources that could never have been installed. -func For(cfg *config.Config) []Demo { - var list []Demo - for _, d := range All() { - if ko, ok := d.(kindOnlyDemo); ok && ko.KindOnly() && !cfg.Kind { - continue - } - list = append(list, d) - } - return list -} - -// Deleters returns the demos that apply to cfg as the narrower interface +// Deleters returns every registered demo as the narrower interface // [steps.Env.DeleteAll] takes. // // The conversion lives here because the dependency only runs one way: the demos // import steps for Env, so steps cannot name this package's Demo. -func Deleters(cfg *config.Config) []steps.Deleter { - list := For(cfg) +func Deleters() []steps.Deleter { + list := All() deleters := make([]steps.Deleter, 0, len(list)) for _, d := range list { deleters = append(deleters, d) diff --git a/cmd/ate-setup/internal/demos/render.go b/cmd/ate-setup/internal/demos/render.go index 941d1d3dc1..a755057dfe 100644 --- a/cmd/ate-setup/internal/demos/render.go +++ b/cmd/ate-setup/internal/demos/render.go @@ -24,7 +24,7 @@ import ( const bucketNamePlaceholder = "BUCKET_NAME" // ExternalVolumePlaceholders are the optional external-volume hooks in the -// counter and autoscaled-workerpool templates. Every path except +// counter templates. Every path except // `deploy demo counter --with-external-volume` drops them, which removes the // lines entirely. var ExternalVolumePlaceholders = []string{ diff --git a/cmd/ate-setup/internal/images/prebuilt.go b/cmd/ate-setup/internal/images/prebuilt.go index 63656b29ad..873dcb01e8 100644 --- a/cmd/ate-setup/internal/images/prebuilt.go +++ b/cmd/ate-setup/internal/images/prebuilt.go @@ -79,13 +79,13 @@ func (p *Prebuilt) ResolveBytes(ctx context.Context, manifest []byte) ([]byte, e // it, such as envoy-dataplane: a release publishes that under the same // repository and tag as the rest, so it is found the same way. // -// The digest is not decoration. An ActorTemplate's container image, an image -// volume's reference, and a SandboxConfig's pauseImage each carry the CEL rule -// self.contains('@'), so a tag on its own is rejected by admission and every -// demo would fail to deploy. Pinning also restores what ko gave the control -// plane deployments for free: an install that cannot shift under a tag someone -// moves later. Keeping the tag alongside the digest is ko's own output shape, -// and leaves the reference legible in `kubectl get`. +// The digest is not decoration. An ActorTemplate's container image and an image +// volume's reference each carry the CEL rule self.contains('@'), so a tag on +// its own is rejected by admission and every demo would fail to deploy. Pinning +// also restores what ko gave the control plane deployments for free: an install +// that cannot shift under a tag someone moves later. Keeping the tag alongside +// the digest is ko's own output shape, and leaves the reference legible in +// `kubectl get`. func (p *Prebuilt) Pin(ctx context.Context, image string) (string, error) { tagged := p.src.Repo + "/" + image + ":" + p.src.Tag // A tag that already carries one is pinned as it stands. Looking it up diff --git a/cmd/ate-setup/internal/images/prebuilt_test.go b/cmd/ate-setup/internal/images/prebuilt_test.go index 1264caa09a..41a36bf181 100644 --- a/cmd/ate-setup/internal/images/prebuilt_test.go +++ b/cmd/ate-setup/internal/images/prebuilt_test.go @@ -370,10 +370,10 @@ func TestResolveEveryInstallManifest(t *testing.T) { if refs := koRefPattern.FindAllString(string(out), -1); len(refs) > 0 { t.Errorf("unresolved references survived: %v", refs) } - // Every reference has to name a digest. ActorTemplate images, - // image volume references, and a SandboxConfig's pauseImage all - // carry the CEL rule self.contains('@'), so an unpinned reference - // is not applied at all, it is rejected by admission. + // Every reference has to name a digest. ActorTemplate images and + // image volume references carry the CEL rule self.contains('@'), so + // an unpinned reference is not applied at all, it is rejected by + // admission. for _, ref := range resolvedRef.FindAllString(string(out), -1) { if !strings.Contains(ref, "@sha256:") { t.Errorf("%s is not pinned to a digest", ref) diff --git a/cmd/ate-setup/internal/steps/apienv.go b/cmd/ate-setup/internal/steps/apienv.go index 2675bdc954..c9fc76e08d 100644 --- a/cmd/ate-setup/internal/steps/apienv.go +++ b/cmd/ate-setup/internal/steps/apienv.go @@ -158,15 +158,12 @@ func (e *Env) UpdateAPIServerEnvVars(ctx context.Context) error { return e.annotateAPIServerEnvHash(ctx) } -// sharedBootstrapOwnerWarning returns a warning when enforced authorization -// makes the shared client ServiceAccount a bootstrap owner, and "" otherwise. +// sharedBootstrapOwnerWarning returns a warning when the shared client +// ServiceAccount is a bootstrap owner, and "" otherwise. // It is the default so demos, benchmarks and e2e work, but everyone who can // mint its token, including every kubectl-ate user without --token-file, is // then a global owner. -func sharedBootstrapOwnerWarning(enabled bool, bootstrapOwners, namespace string) string { - if !enabled { - return "" - } +func sharedBootstrapOwnerWarning(bootstrapOwners, namespace string) string { shared := installdefaults.ClientPrincipal(namespace) for owner := range strings.SplitSeq(bootstrapOwners, ",") { if strings.TrimSpace(owner) == shared { @@ -349,26 +346,22 @@ func (e *Env) applyWithAPIServerEnvHash(ctx context.Context, manifest []byte) er if err := stampAPIServerEnvHash(objs, hash); err != nil { return err } - if err := setAPIServerAuthzArgs(objs, e.Cfg.APIExperimentalEnableAuthz, e.Cfg.APIAuthzBootstrapOwners); err != nil { + if err := setAPIServerAuthzArgs(objs, e.Cfg.APIAuthzBootstrapOwners); err != nil { return err } - if msg := sharedBootstrapOwnerWarning(e.Cfg.APIExperimentalEnableAuthz, e.Cfg.APIAuthzBootstrapOwners, e.Namespace()); msg != "" { + if msg := sharedBootstrapOwnerWarning(e.Cfg.APIAuthzBootstrapOwners, e.Namespace()); msg != "" { log.Warnf("%s", msg) } return e.Kube.Apply(ctx, objs) } -// apiServerAuthzFlags are the ate-api-server flags setAPIServerAuthzArgs owns. -var apiServerAuthzFlags = []string{"--experimental-enable-authz", "--authz-bootstrap-owners"} +// apiServerAuthzFlag is the ate-api-server flag setAPIServerAuthzArgs owns. +const apiServerAuthzFlag = "--authz-bootstrap-owners" -// setAPIServerAuthzArgs sets ate-api-server's authorization flags on its +// setAPIServerAuthzArgs sets ate-api-server's bootstrap owners flag on its // container in objs, replacing any the manifest already carries. A changed // flag changes the pod template, so it rolls the Deployment. -// -// The bootstrap owners are passed even with enforcement off: access policy -// RPCs are checked either way, so without them nobody could set up policies -// before turning enforcement on. -func setAPIServerAuthzArgs(objs []*unstructured.Unstructured, enabled bool, bootstrapOwners string) error { +func setAPIServerAuthzArgs(objs []*unstructured.Unstructured, bootstrapOwners string) error { var dep *unstructured.Unstructured for _, obj := range objs { if obj.GetKind() == "Deployment" && obj.GetName() == "ate-api-server" { @@ -393,15 +386,10 @@ func setAPIServerAuthzArgs(objs []*unstructured.Unstructured, enabled bool, boot return fmt.Errorf("%s: %w", kube.Describe(dep), err) } args = slices.DeleteFunc(args, func(arg string) bool { - return slices.ContainsFunc(apiServerAuthzFlags, func(flag string) bool { - return arg == flag || strings.HasPrefix(arg, flag+"=") - }) + return arg == apiServerAuthzFlag || strings.HasPrefix(arg, apiServerAuthzFlag+"=") }) - if enabled { - args = append(args, "--experimental-enable-authz=true") - } if strings.Trim(bootstrapOwners, " ,") != "" { - args = append(args, "--authz-bootstrap-owners="+bootstrapOwners) + args = append(args, apiServerAuthzFlag+"="+bootstrapOwners) } if err := unstructured.SetNestedStringSlice(container, args, "args"); err != nil { return fmt.Errorf("%s: %w", kube.Describe(dep), err) diff --git a/cmd/ate-setup/internal/steps/apienv_test.go b/cmd/ate-setup/internal/steps/apienv_test.go index a78d3f9378..81e22f22bd 100644 --- a/cmd/ate-setup/internal/steps/apienv_test.go +++ b/cmd/ate-setup/internal/steps/apienv_test.go @@ -618,8 +618,8 @@ spec: } } -// Authorization reaches ate-api-server as flags on its container, so a change -// rolls the Deployment, and the envvars ConfigMap carries none of it. +// The bootstrap owners reach ate-api-server as a flag on its container, so a +// change rolls the Deployment, and the envvars ConfigMap carries none of it. func TestSetAPIServerAuthzArgs(t *testing.T) { const manifest = `apiVersion: apps/v1 kind: Deployment @@ -636,35 +636,22 @@ spec: args: - --config-file=/run/apiconfig.ate.dev/apiconfig.textproto - --authz-bootstrap-owners=stale - - --experimental-enable-authz ` for _, tc := range []struct { - name string - enabled bool - owners string - want []string + name string + owners string + want []string }{ { - name: "enforced", - enabled: true, - owners: "alice,bob", + name: "replaces the owners", + owners: "alice,bob", want: []string{ "--config-file=/run/apiconfig.ate.dev/apiconfig.textproto", - "--experimental-enable-authz=true", "--authz-bootstrap-owners=alice,bob", }, }, { - // Owners stay, so access policies can be set up before enforcement is on. - name: "off keeps the owners", - owners: "alice", - want: []string{ - "--config-file=/run/apiconfig.ate.dev/apiconfig.textproto", - "--authz-bootstrap-owners=alice", - }, - }, - { - name: "off with no owners", + name: "no owners drops the flag", want: []string{"--config-file=/run/apiconfig.ate.dev/apiconfig.textproto"}, }, } { @@ -673,7 +660,7 @@ spec: if err != nil { t.Fatalf("DecodeManifestBytes() error = %v", err) } - if err := setAPIServerAuthzArgs(objs, tc.enabled, tc.owners); err != nil { + if err := setAPIServerAuthzArgs(objs, tc.owners); err != nil { t.Fatalf("setAPIServerAuthzArgs() error = %v", err) } containers, _, _ := unstructured.NestedSlice(objs[0].Object, "spec", "template", "spec", "containers") @@ -694,18 +681,17 @@ spec: if err != nil { t.Fatalf("DecodeManifestBytes() error = %v", err) } - if err := setAPIServerAuthzArgs(objs, true, "alice"); err == nil { + if err := setAPIServerAuthzArgs(objs, "alice"); err == nil { t.Error("setAPIServerAuthzArgs() without deployment/ate-api-server error = nil, want an error") } } func TestCreateAPIServerEnvVarsOmitsAuthz(t *testing.T) { e := &Env{Cfg: &config.Config{ - PostgresReadWriteRole: config.DefaultPostgresReadWriteRole, - PostgresOwnerRole: config.DefaultPostgresOwnerRole, - APIExperimentalEnableAuthz: true, - APIAuthzBootstrapOwners: "system:serviceaccount:ate-system:ate-client", - SnapshotLocation: "gs://ate-snapshots/", + PostgresReadWriteRole: config.DefaultPostgresReadWriteRole, + PostgresOwnerRole: config.DefaultPostgresOwnerRole, + APIAuthzBootstrapOwners: "system:serviceaccount:ate-system:ate-client", + SnapshotLocation: "gs://ate-snapshots/", }, Kube: fakeKube(t, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: NamespaceAteSystem}}, apiServerEnvVarsConfigMap(map[string]string{}), @@ -718,10 +704,8 @@ func TestCreateAPIServerEnvVarsOmitsAuthz(t *testing.T) { if err != nil || cm == nil { t.Fatalf("GetConfigMap() = %v, %v", cm, err) } - for _, key := range []string{"ATE_API_EXPERIMENTAL_ENABLE_AUTHZ", "ATE_API_AUTHZ_BOOTSTRAP_OWNERS"} { - if v, ok := cm.Data[key]; ok { - t.Errorf("ConfigMap carries %s=%q; authorization is passed as flags", key, v) - } + if v, ok := cm.Data["ATE_API_AUTHZ_BOOTSTRAP_OWNERS"]; ok { + t.Errorf("ConfigMap carries ATE_API_AUTHZ_BOOTSTRAP_OWNERS=%q; the bootstrap owners are passed as a flag", v) } } @@ -730,22 +714,20 @@ func TestCreateAPIServerEnvVarsOmitsAuthz(t *testing.T) { func TestSharedBootstrapOwnerWarning(t *testing.T) { const shared = "system:serviceaccount:ate-system:ate-client" for _, tc := range []struct { - name string - enabled bool - owners string - warn bool + name string + owners string + warn bool }{ - {name: "default owner", enabled: true, owners: shared, warn: true}, - {name: "listed among others", enabled: true, owners: "alice, " + shared + " ,bob", warn: true}, - {name: "own principals only", enabled: true, owners: "alice,bob", warn: false}, - {name: "another namespace's client", enabled: true, owners: "system:serviceaccount:other:ate-client", warn: false}, - {name: "prefix is not a match", enabled: true, owners: shared + "-x", warn: false}, - {name: "authorization off", enabled: false, owners: shared, warn: false}, + {name: "default owner", owners: shared, warn: true}, + {name: "listed among others", owners: "alice, " + shared + " ,bob", warn: true}, + {name: "own principals only", owners: "alice,bob", warn: false}, + {name: "another namespace's client", owners: "system:serviceaccount:other:ate-client", warn: false}, + {name: "prefix is not a match", owners: shared + "-x", warn: false}, } { t.Run(tc.name, func(t *testing.T) { - got := sharedBootstrapOwnerWarning(tc.enabled, tc.owners, "ate-system") + got := sharedBootstrapOwnerWarning(tc.owners, "ate-system") if (got != "") != tc.warn { - t.Fatalf("sharedBootstrapOwnerWarning(%v, %q) = %q, want warning: %v", tc.enabled, tc.owners, got, tc.warn) + t.Fatalf("sharedBootstrapOwnerWarning(%q) = %q, want warning: %v", tc.owners, got, tc.warn) } if tc.warn { for _, want := range []string{shared, "--ateapi-authz-bootstrap-owners", "ATE_API_AUTHZ_BOOTSTRAP_OWNERS"} { diff --git a/cmd/ate-setup/internal/steps/csi.go b/cmd/ate-setup/internal/steps/csi.go index 1fccb87793..b05d71842f 100644 --- a/cmd/ate-setup/internal/steps/csi.go +++ b/cmd/ate-setup/internal/steps/csi.go @@ -32,6 +32,7 @@ import ( "go.ate.dev/substrate/cmd/ate-setup/internal/kube" "go.ate.dev/substrate/cmd/ate-setup/internal/kustomize" "go.ate.dev/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/internal/installdefaults" "go.ate.dev/substrate/internal/nodepath" ) @@ -40,10 +41,10 @@ import ( // they mount inside it are visible to atelet. const workerHostDir = nodepath.BasePath -// csiHostpathControllerService fronts the hostpath driver's socat sidecar, -// which bridges the TCP endpoint atelet dials to the driver's unix socket. The -// CSIDriverConfig applied later names this Service in both its -// controllerEndpoint and its TLS server name. +// csiHostpathControllerService fronts the hostpath driver's Envoy sidecar, +// which terminates mTLS on the TCP endpoint ateapi dials and forwards to the +// driver's unix socket. The CSIDriverConfig applied later names this Service +// in both its controllerEndpoint and its TLS server name. const csiHostpathControllerService = ` apiVersion: v1 kind: Service @@ -203,8 +204,8 @@ func (e *Env) setupCSIHostpath(ctx context.Context) error { log.Infof("Using Kind node: %s", node) cleanupKindWorkerDir(node) - // Before the driver, not after: atelet talks to the controller over TCP, - // the socat sidecar bridges that to the driver's unix socket, and + // Before the driver, not after: ateapi talks to the controller over TCP, + // the Envoy sidecar bridges that to the driver's unix socket, and // servicednssigner issues the sidecar's serving certificate from the // Service. A sidecar that starts before the Service exists has no // certificate to be issued against and never becomes ready. @@ -292,6 +293,184 @@ spec: return e.Kube.RolloutRestart(ctx, e.Namespace(), "atelet", time.Now()) } +// csiNFSControllerService fronts the Envoy sidecar csiNFSControllerPatch adds to the NFS controller pod. +const csiNFSControllerService = ` +apiVersion: v1 +kind: Service +metadata: + name: csi-nfs-controller + namespace: kube-system +spec: + selector: + app: csi-nfs-controller + ports: + - port: 50052 + targetPort: 10000 + name: grpc +` + +const csiNFSEnvoyConfigMapName = "csi-nfs-envoy-config" + +// csiNFSEnvoyConfigMap renders the NFS controller's Envoy config: an mTLS listener accepting only apiServerSPIFFEID, proxied to the driver's unix socket. +func csiNFSEnvoyConfigMap(apiServerSPIFFEID string) string { + return strings.ReplaceAll(csiNFSEnvoyConfigMapTemplate, "{{API_SERVER_SPIFFE_ID}}", apiServerSPIFFEID) +} + +const csiNFSEnvoyConfigMapTemplate = ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: ` + csiNFSEnvoyConfigMapName + ` + namespace: kube-system +data: + envoy.yaml: | + node: + id: csi-nfs-envoy + cluster: csi-nfs-envoy + static_resources: + listeners: + - name: csi_tls_listener + address: + socket_address: { address: 0.0.0.0, port_value: 10000 } + filter_chains: + - transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + require_client_certificate: true + common_tls_context: + alpn_protocols: + - h2 + tls_certificate_sds_secret_configs: + - name: servicedns_serving_cert + sds_config: + resource_api_version: V3 + path_config_source: + path: /etc/envoy/sds-servicedns-cert.yaml + combined_validation_context: + default_validation_context: + match_typed_subject_alt_names: + - san_type: URI + matcher: + exact: "{{API_SERVER_SPIFFE_ID}}" + validation_context_sds_secret_config: + name: podidentity_validation_context + sds_config: + resource_api_version: V3 + path_config_source: + path: /etc/envoy/sds-podidentity-validation.yaml + filters: + - name: envoy.filters.network.tcp_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + stat_prefix: csi_proxy + cluster: csi_unix_socket + clusters: + - name: csi_unix_socket + connect_timeout: 0.25s + type: STATIC + load_assignment: + cluster_name: csi_unix_socket + endpoints: + - lb_endpoints: + - endpoint: + address: + pipe: + path: /csi/csi.sock + sds-servicedns-cert.yaml: | + resources: + - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret + name: servicedns_serving_cert + tls_certificate: + certificate_chain: { filename: /run/servicedns/credential-bundle.pem } + private_key: { filename: /run/servicedns/credential-bundle.pem } + watched_directory: { path: /run/servicedns } + sds-podidentity-validation.yaml: | + resources: + - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret + name: podidentity_validation_context + validation_context: + trusted_ca: { filename: /run/podidentity-ca/trust-bundle.pem } + watched_directory: { path: /run/podidentity-ca } +` + +// csiNFSControllerPatch adds the Envoy TLS proxy to the upstream csi-nfs-controller Deployment and removes the socat sidecar earlier installs added. +const csiNFSControllerPatch = ` +spec: + template: + spec: + containers: + - $patch: delete + name: socat + - name: envoy + image: envoyproxy/envoy:v1.34-latest + command: + - /usr/local/bin/envoy + args: + - -c + - /etc/envoy/envoy.yaml + readinessProbe: + tcpSocket: + port: 10000 + periodSeconds: 5 + securityContext: + # Root because the driver creates csi.sock mode 0755; nothing else is needed. + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - mountPath: /etc/envoy + name: envoy-config + readOnly: true + - mountPath: /csi + name: socket-dir + - mountPath: /run/servicedns + name: servicedns-certs + readOnly: true + - mountPath: /run/podidentity-ca + name: podidentity-ca + readOnly: true + volumes: + - name: envoy-config + configMap: + name: ` + csiNFSEnvoyConfigMapName + ` + - name: servicedns-certs + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: podidentity-ca + projected: + sources: + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +` + +// csiNFSDriverConfig registers the NFS driver; its serverName is the one DNS name servicednssigner issues, ..svc. +const csiNFSDriverConfig = ` +apiVersion: ate.dev/v1alpha1 +kind: CSIDriverConfig +metadata: + name: nfs.csi.k8s.io +spec: + driverName: nfs.csi.k8s.io + controllerEndpoint: tcp://csi-nfs-controller.kube-system.svc.cluster.local:50052 + nodeSocketOverride: unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock + tls: + enabled: true + usePodIdentity: true + serverName: csi-nfs-controller.kube-system.svc +` + func (e *Env) setupCSINFS(ctx context.Context) error { log.Step("setup_csi_nfs") @@ -306,6 +485,15 @@ func (e *Env) setupCSINFS(ctx context.Context) error { return err } + // Before the driver: servicednssigner only issues the sidecar's certificate once a Service selects its pod. + log.Infof("Exposing the CSI NFS controller over a TLS Service...") + if err := e.applyInline(ctx, csiNFSControllerService); err != nil { + return err + } + if err := e.applyInline(ctx, csiNFSEnvoyConfigMap(installdefaults.APIServerSPIFFEID(e.Namespace()))); err != nil { + return err + } + log.Infof("Deploying the CSI NFS driver...") for _, name := range []string{ "rbac-csi-nfs.yaml", @@ -344,24 +532,8 @@ spec: return fmt.Errorf("while patching the csi-nfs-node DaemonSet: %w", err) } - log.Infof("Patching the CSI NFS controller Deployment...") - const controllerPatch = ` -spec: - template: - spec: - containers: - - name: socat - image: docker.io/alpine/socat:1.7.4.3-r0 - args: - - tcp-listen:10000,fork,reuseaddr - - unix-connect:/csi/csi.sock - securityContext: - privileged: true - volumeMounts: - - mountPath: /csi - name: socket-dir -` - controllerPatchJSON, err := yaml.YAMLToJSON([]byte(controllerPatch)) + log.Infof("Patching the CSI NFS controller Deployment with the Envoy TLS proxy...") + controllerPatchJSON, err := yaml.YAMLToJSON([]byte(csiNFSControllerPatch)) if err != nil { return fmt.Errorf("while converting csi-nfs-controller patch to JSON: %w", err) } @@ -370,24 +542,6 @@ spec: return fmt.Errorf("while patching the csi-nfs-controller Deployment: %w", err) } - log.Infof("Exposing the CSI NFS controller over a Service...") - if err := e.applyInline(ctx, ` -apiVersion: v1 -kind: Service -metadata: - name: csi-nfs-controller - namespace: kube-system -spec: - selector: - app: csi-nfs-controller - ports: - - port: 50052 - targetPort: 10000 - name: grpc -`); err != nil { - return err - } - // The driver resolves the server's DNS name at provisioning time, so the // StorageClass can be created before the NFS server has an address. log.Infof("Creating the csi-nfs-sc StorageClass...") @@ -407,16 +561,7 @@ spec: } log.Infof("Creating the nfs.csi.k8s.io CSIDriverConfig...") - if err := e.applyInline(ctx, ` -apiVersion: ate.dev/v1alpha1 -kind: CSIDriverConfig -metadata: - name: nfs.csi.k8s.io -spec: - driverName: nfs.csi.k8s.io - controllerEndpoint: tcp://csi-nfs-controller.kube-system.svc.cluster.local:50052 - nodeSocketOverride: unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock -`); err != nil { + if err := e.applyInline(ctx, csiNFSDriverConfig); err != nil { return err } diff --git a/cmd/ate-setup/internal/steps/csi_test.go b/cmd/ate-setup/internal/steps/csi_test.go index e558bfb9a2..695593f140 100644 --- a/cmd/ate-setup/internal/steps/csi_test.go +++ b/cmd/ate-setup/internal/steps/csi_test.go @@ -18,10 +18,13 @@ import ( "context" "os" "path/filepath" + "strconv" "strings" "testing" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" "go.ate.dev/substrate/cmd/ate-setup/internal/config" "go.ate.dev/substrate/cmd/ate-setup/internal/kube" @@ -103,7 +106,7 @@ func functionBody(t *testing.T, file, signature string) string { return body[:end] } -// servicednssigner issues the socat sidecar's serving certificate from the +// servicednssigner issues the Envoy sidecar's serving certificate from the // csi-hostpath-controller Service, so a driver deployed before that Service // exists has nothing to be issued against and never becomes ready. The shell // installer created the Service first and said why. @@ -122,7 +125,7 @@ func TestTheHostpathControllerServiceIsCreatedBeforeTheDriver(t *testing.T) { t.Fatal("setupCSIHostpath no longer applies the driver bundle with ApplyTolerant") case service > driver: t.Error("setupCSIHostpath deploys the hostpath driver before creating the " + - "csi-hostpath-controller Service; the socat sidecar cannot be issued a " + + "csi-hostpath-controller Service; the Envoy sidecar cannot be issued a " + "certificate until the Service exists") } } @@ -197,3 +200,91 @@ func TestCSIDriverSpellings(t *testing.T) { }) } } + +// decodeYAML unmarshals a manifest defined in csi.go. +func decodeYAML(t *testing.T, name, src string, into any) { + t.Helper() + if err := yaml.Unmarshal([]byte(src), into); err != nil { + t.Fatalf("%s does not parse: %v", name, err) + } +} + +// ateapi verifies tls.serverName, which must be the ..svc name servicednssigner issues. +func TestTheNFSDriverConfigEnablesTLSAgainstTheControllerService(t *testing.T) { + var svc corev1.Service + decodeYAML(t, "csiNFSControllerService", csiNFSControllerService, &svc) + var cfg struct { + Spec struct { + ControllerEndpoint string `json:"controllerEndpoint"` + TLS *struct { + Enabled bool `json:"enabled"` + UsePodIdentity bool `json:"usePodIdentity"` + ServerName string `json:"serverName"` + } `json:"tls"` + } `json:"spec"` + } + decodeYAML(t, "csiNFSDriverConfig", csiNFSDriverConfig, &cfg) + + if cfg.Spec.TLS == nil || !cfg.Spec.TLS.Enabled || !cfg.Spec.TLS.UsePodIdentity { + t.Fatalf("csiNFSDriverConfig tls = %+v, want enabled with pod identity", cfg.Spec.TLS) + } + if want := svc.Name + "." + svc.Namespace + ".svc"; cfg.Spec.TLS.ServerName != want { + t.Errorf("tls.serverName = %q, want %q, the name servicednssigner issues", cfg.Spec.TLS.ServerName, want) + } + if want := "tcp://" + svc.Name + "." + svc.Namespace + ".svc.cluster.local:" + strconv.Itoa(int(svc.Spec.Ports[0].Port)); cfg.Spec.ControllerEndpoint != want { + t.Errorf("controllerEndpoint = %q, want %q", cfg.Spec.ControllerEndpoint, want) + } +} + +// hack/setup-csi-nfs-kind.sh is the same install by hand and must keep the same security decisions. +func TestTheNFSShellSetupMatchesTheGoSetup(t *testing.T) { + script, err := os.ReadFile(filepath.Join(repoRoot(t), "hack", "setup-csi-nfs-kind.sh")) + if err != nil { + t.Fatalf("reading the shell setup: %v", err) + } + src := string(script) + + for _, forbidden := range []string{"socat:", "privileged: true"} { + if strings.Contains(src, forbidden) { + t.Errorf("hack/setup-csi-nfs-kind.sh still contains %q", forbidden) + } + } + for _, required := range []string{ + "require_client_certificate: true", + "serverName: csi-nfs-controller.kube-system.svc\n", + "usePodIdentity: true", + "/sa/ate-api-server", + } { + if !strings.Contains(src, required) { + t.Errorf("hack/setup-csi-nfs-kind.sh no longer contains %q", required) + } + } + if service, driver := strings.Index(src, "name: csi-nfs-controller\n namespace: kube-system"), strings.Index(src, "csi-nfs-controller.yaml"); service < 0 || driver < 0 || service > driver { + t.Error("hack/setup-csi-nfs-kind.sh deploys the NFS driver before creating the csi-nfs-controller Service") + } +} + +// The hostpath proxy needs no more than root to reach the driver's socket. +func TestTheHostpathProxyIsNotPrivileged(t *testing.T) { + objs, err := kube.LoadPath(filepath.Join(repoRoot(t), "hack", "third_party", "csi-driver-host-path", "deploy", "csi-hostpath-testing.yaml")) + if err != nil { + t.Fatalf("loading the hostpath proxy manifest: %v", err) + } + var found bool + for _, obj := range objs { + if obj.GetKind() != "StatefulSet" { + continue + } + containers, _, _ := unstructured.NestedSlice(obj.Object, "spec", "template", "spec", "containers") + for _, c := range containers { + c := c.(map[string]any) + found = true + if privileged, _, _ := unstructured.NestedBool(c, "securityContext", "privileged"); privileged { + t.Errorf("%s container %q is privileged", kube.Describe(obj), c["name"]) + } + } + } + if !found { + t.Fatal("csi-hostpath-testing.yaml no longer holds a proxy StatefulSet") + } +} diff --git a/cmd/ate-setup/internal/steps/env.go b/cmd/ate-setup/internal/steps/env.go index 63b81920ae..8cfba859a7 100644 --- a/cmd/ate-setup/internal/steps/env.go +++ b/cmd/ate-setup/internal/steps/env.go @@ -278,8 +278,7 @@ func (e *Env) EnsureAteSystemNamespace(ctx context.Context) error { return e.Kube.WaitNamespaceActive(ctx, ns, NamespaceTimeout) } -// RequireKind fails a step that only makes sense on a local Kind cluster. The -// Kind-only demos, which live outside this package, use it too. +// RequireKind fails a step that only makes sense on a local Kind cluster. func (e *Env) RequireKind(what string) error { if !e.Cfg.Kind { return fmt.Errorf("%s is only supported for Kind installations; re-run with --kind", what) diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index 396cd4d89c..8a2d6d3e36 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -48,11 +48,6 @@ const cordonControlPlaneComponent = installDir + "/components/cordon-control-pla // over the atelet and ate-controller applies under --fake-data-plane. const fakeDataPlaneComponent = installDir + "/components/fake-data-plane" -// routerStaticMTLSComponent is the kustomize component that switches the -// ingress router to --ingress-auth-mode=static-mtls, layered over every -// control plane apply under --ingress-auth-mode=static-mtls. -const routerStaticMTLSComponent = installDir + "/components/router-static-mtls" - // components lists the kustomize components every control plane apply is // composed with. Each patch targets its workloads by name, and kustomize leaves // a stream alone when nothing in it matches, so composing a manifest that @@ -62,9 +57,6 @@ func (e *Env) components() []string { if e.Cfg.FakeDataPlane { components = append(components, e.Cfg.Path(fakeDataPlaneComponent)) } - if e.Cfg.IngressAuthMode == config.IngressAuthStaticMTLS { - components = append(components, e.Cfg.Path(routerStaticMTLSComponent)) - } if e.Cfg.CordonControlPlane { components = append(components, e.Cfg.Path(cordonControlPlaneComponent)) } diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index 619022b42e..dd5202e5c9 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -767,79 +767,8 @@ func TestRenderFakeDataPlane(t *testing.T) { } } -// Under --ingress-auth-mode=static-mtls both paths that deliver the router -// -- the system bundle and the lone router redeploy -- have to carry the -// static-mtls flags, alone or alongside the cordon component. Whether the -// router accepts those flags is pinned on the router side, by -// TestRouterIngressAuthManifestsAreValid. -func TestRenderRouterStaticMTLS(t *testing.T) { - root := repoRoot(t) - for _, tc := range []struct { - name string - cfg config.Config - path func(e *Env) string - cordon bool - }{ - { - name: "base bundle", - cfg: config.Config{Router: config.RouterEnvoy}, - path: func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) }, - }, - { - name: "kind bundle", - cfg: config.Config{Router: config.RouterEnvoy, Kind: true}, - path: func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) }, - }, - { - name: "router file", - cfg: config.Config{Router: config.RouterEnvoy}, - path: func(e *Env) string { return e.Cfg.Manifest("atenet-router.yaml") }, - }, - { - name: "kind bundle with cordon", - cfg: config.Config{Router: config.RouterEnvoy, Kind: true}, - path: func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) }, - cordon: true, - }, - } { - t.Run(tc.name, func(t *testing.T) { - cfg := tc.cfg - cfg.Root = root - cfg.IngressAuthMode = config.IngressAuthStaticMTLS - cfg.CordonControlPlane = tc.cordon - e := &Env{Cfg: &cfg} - - rendered, err := e.render(tc.path(e)) - if err != nil { - t.Fatalf("render: %v", err) - } - args := routerArgs(t, rendered) - if !slices.Contains(args, "--ingress-auth-mode=static-mtls") { - t.Errorf("atenet-router args %v do not select static-mtls", args) - } - if tc.cordon { - if pinned, _ := pinnedWorkloads(t, rendered); pinned["atenet-router"] != "ate-control-plane" { - t.Errorf("composing both components dropped the cordon pinning (pinned: %v)", pinned) - } - } - t.Run("deprecated-insecure leaves the router alone", func(t *testing.T) { - e := &Env{Cfg: &config.Config{Root: root, Router: config.RouterEnvoy, IngressAuthMode: config.IngressAuthDeprecatedInsecure}} - rendered, err := e.render(e.Cfg.Manifest("atenet-router.yaml")) - if err != nil { - t.Fatalf("render: %v", err) - } - for _, arg := range routerArgs(t, rendered) { - if strings.HasPrefix(arg, "--ingress-auth-mode") { - t.Errorf("deprecated-insecure render sets %s", arg) - } - } - }) - }) - } -} - // Both paths that deliver the Envoy router pass it the install's allowlist, -// whatever the overlay and auth mode, and refuse to render without one. +// whatever the overlay, and refuse to render without one. func TestRenderRouterAllowedSPIFFEIDs(t *testing.T) { const a, b = "spiffe://cluster.local/ns/a/sa/a", "spiffe://cluster.local/ns/b/sa/b" src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"} @@ -857,8 +786,8 @@ func TestRenderRouterAllowedSPIFFEIDs(t *testing.T) { render: func(e *Env) ([]byte, error) { return e.renderSystemManifests(t.Context()) }, }, { - name: "kind bundle static-mtls", - cfg: config.Config{Kind: true, IngressAuthMode: config.IngressAuthStaticMTLS}, + name: "kind bundle", + cfg: config.Config{Kind: true}, render: func(e *Env) ([]byte, error) { return e.renderSystemManifests(t.Context()) }, }, { @@ -866,11 +795,6 @@ func TestRenderRouterAllowedSPIFFEIDs(t *testing.T) { cfg: config.Config{}, render: func(e *Env) ([]byte, error) { return e.renderAtenetRouterManifest(t.Context()) }, }, - { - name: "router file static-mtls", - cfg: config.Config{IngressAuthMode: config.IngressAuthStaticMTLS}, - render: func(e *Env) ([]byte, error) { return e.renderAtenetRouterManifest(t.Context()) }, - }, } { t.Run(tc.name, func(t *testing.T) { cfg := tc.cfg diff --git a/cmd/ate-setup/preflight_test.go b/cmd/ate-setup/preflight_test.go index f54785ee71..729907dc9e 100644 --- a/cmd/ate-setup/preflight_test.go +++ b/cmd/ate-setup/preflight_test.go @@ -31,6 +31,15 @@ import ( "go.ate.dev/substrate/cmd/ate-setup/internal/cmd" ) +func repoRoot(t *testing.T) string { + t.Helper() + out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() + if err != nil { + t.Fatalf("git rev-parse --show-toplevel: %v", err) + } + return strings.TrimSpace(string(out)) +} + // Each invocation gets its own command tree and flags, including invocations // made by the real shell shims. The test executable runs the installer without // rebuilding it or contacting a real cluster. @@ -48,7 +57,7 @@ func TestInstallerPreflightProcess(t *testing.T) { t.Fatal("missing installer arguments") } -func runPreflightInstaller(t *testing.T, kubeconfig, dockerStatus string, shim bool, args ...string) (string, int) { +func runPreflightInstaller(t *testing.T, kubeconfig, dockerStatus string, args ...string) (string, int) { t.Helper() bin := t.TempDir() executable, err := os.Executable() @@ -76,9 +85,6 @@ fi ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) defer cancel() process := exec.CommandContext(ctx, executable, append([]string{"-test.run=^TestInstallerPreflightProcess$", "--"}, args...)...) - if shim { - process = exec.CommandContext(ctx, "bash", append([]string{"hack/install-ate-kind.sh"}, args...)...) - } process.Dir = repoRoot(t) process.Env = []string{ "PATH=" + bin + string(os.PathListSeparator) + os.Getenv("PATH"), @@ -134,22 +140,15 @@ current-context: kind-kind } func TestKindInstallerMissingContext(t *testing.T) { - for _, shim := range []bool{false, true} { - t.Run(fmt.Sprintf("shim=%t", shim), func(t *testing.T) { - args := []string{"--kind", "deploy", "demo", "counter"} - if shim { - args = []string{"--deploy-demo-counter"} - } - out, code := runPreflightInstaller(t, preflightKubeconfig(t, ""), "0", shim, args...) - if code == 0 || !strings.Contains(out, "create-kind-cluster.sh") || !strings.Contains(out, "export kubeconfig") { - t.Fatalf("exit=%d, want missing-context recovery guidance\n%s", code, out) - } - }) + args := []string{"--kind", "deploy", "demo", "counter"} + out, code := runPreflightInstaller(t, preflightKubeconfig(t, ""), "0", args...) + if code == 0 || !strings.Contains(out, "create-kind-cluster.sh") || !strings.Contains(out, "export kubeconfig") { + t.Fatalf("exit=%d, want missing-context recovery guidance\n%s", code, out) } } func TestKindInstallerDockerUnavailable(t *testing.T) { - out, code := runPreflightInstaller(t, preflightKubeconfig(t, "http://127.0.0.1:1"), "1", false, + out, code := runPreflightInstaller(t, preflightKubeconfig(t, "http://127.0.0.1:1"), "1", "--kind", "deploy", "demo", "counter") if code == 0 || !strings.Contains(out, "Docker") || !strings.Contains(out, "docker info") { t.Fatalf("exit=%d, want Docker readiness guidance\n%s", code, out) @@ -159,14 +158,10 @@ func TestKindInstallerDockerUnavailable(t *testing.T) { func TestInstallerRequiresControlPlaneBeforeDemoOrBenchmark(t *testing.T) { for _, tc := range []struct { name string - shim bool args []string }{ - {"native demo", false, []string{"deploy", "demo", "counter"}}, - {"native benchmark", false, []string{"deploy", "benchmarks"}}, - {"kind demo", true, []string{"--deploy-demo-autoscaled-workerpool"}}, - {"kind benchmark", true, []string{"--deploy-benchmarks"}}, - {"demo before system", true, []string{"--deploy-demo-counter", "--deploy-ate-system"}}, + {"native demo", []string{"deploy", "demo", "counter"}}, + {"native benchmark", []string{"deploy", "benchmarks"}}, } { t.Run(tc.name, func(t *testing.T) { var mu sync.Mutex @@ -180,7 +175,7 @@ func TestInstallerRequiresControlPlaneBeforeDemoOrBenchmark(t *testing.T) { _, _ = fmt.Fprint(w, `{"kind":"Status","apiVersion":"v1","status":"Failure","reason":"NotFound","code":404,"message":"deployments.apps ate-controller not found"}`) })) defer server.Close() - out, code := runPreflightInstaller(t, preflightKubeconfig(t, server.URL), "0", tc.shim, tc.args...) + out, code := runPreflightInstaller(t, preflightKubeconfig(t, server.URL), "0", tc.args...) if code == 0 || !strings.Contains(out, "deploy ate-system") || !strings.Contains(out, "credential-provider") { t.Errorf("exit=%d, want control-plane install guidance\n%s", code, out) } @@ -195,15 +190,10 @@ func TestInstallerRequiresControlPlaneBeforeDemoOrBenchmark(t *testing.T) { } func TestKindInstallerHelpNeedsNoCluster(t *testing.T) { - for _, shim := range []bool{false, true} { - args := []string{"--kind", "deploy", "demo", "counter", "--help"} - if shim { - args = []string{"--help"} - } - out, code := runPreflightInstaller(t, preflightKubeconfig(t, ""), "1", shim, args...) - if code != 0 || !strings.Contains(out, "Usage:") { - t.Fatalf("shim=%t exit=%d, want help without prerequisites\n%s", shim, code, out) - } + args := []string{"--kind", "deploy", "demo", "counter", "--help"} + out, code := runPreflightInstaller(t, preflightKubeconfig(t, ""), "1", args...) + if code != 0 || !strings.Contains(out, "Usage:") { + t.Fatalf("exit=%d, want help without prerequisites\n%s", code, out) } } @@ -225,7 +215,7 @@ func TestInstallerControlPlaneCheckPreservesErrorsAndAllowsInstalledSystem(t *te _, _ = fmt.Fprint(w, `{"kind":"Status","apiVersion":"v1","status":"Failure","reason":"Forbidden","code":403,"message":"fixture access denied"}`) })) defer server.Close() - out, code := runPreflightInstaller(t, preflightKubeconfig(t, server.URL), "0", false, "deploy", "demo", "counter") + out, code := runPreflightInstaller(t, preflightKubeconfig(t, server.URL), "0", "deploy", "demo", "counter") want := "fixture access denied" if installed { want = "demo-counter_deploy" @@ -258,7 +248,7 @@ func TestKindInstallerUsesResolvedContextAndClusterName(t *testing.T) { if err := os.WriteFile(document, []byte("apiVersion: install.ate.dev/v1alpha1\nkind: SubstrateInstall\ncontext: selected-by-config\nkindCluster:\n name: local-dev\n"), 0o600); err != nil { t.Fatal(err) } - out, code := runPreflightInstaller(t, preflightKubeconfig(t, "http://127.0.0.1:1"), "0", false, + out, code := runPreflightInstaller(t, preflightKubeconfig(t, "http://127.0.0.1:1"), "0", "--kind", "--config", document, "deploy", "demo", "counter") for _, want := range []string{`context "selected-by-config"`, `export kubeconfig --name="local-dev"`, `KIND_CLUSTER_NAME="local-dev"`, "replaces an existing cluster"} { if code == 0 || !strings.Contains(out, want) { @@ -277,7 +267,7 @@ func TestKindInstallerMissingOrMalformedKubeconfig(t *testing.T) { } want = "while reading kubeconfig" } - out, code := runPreflightInstaller(t, path, "0", false, "--kind", "deploy", "demo", "counter") + out, code := runPreflightInstaller(t, path, "0", "--kind", "deploy", "demo", "counter") if code == 0 || !strings.Contains(out, want) { t.Errorf("malformed=%t exit=%d, want %q\n%s", malformed, code, want, out) } diff --git a/cmd/ate-setup/shim_test.go b/cmd/ate-setup/shim_test.go deleted file mode 100644 index 9c76926869..0000000000 --- a/cmd/ate-setup/shim_test.go +++ /dev/null @@ -1,605 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -// hack/install-ate.sh translates the installer's historical flags onto this -// binary's commands. These tests pin that translation: they put a stub `go` on -// PATH, run the script, and check the command lines it would have run. Nothing -// is built and no cluster is touched, so the whole file is a unit test of the -// argument mapping -- the one part of the shim that can silently break a CI job -// or a developer's muscle memory. - -import ( - "errors" - "os" - "os/exec" - "path/filepath" - "regexp" - "strings" - "testing" - - "go.ate.dev/substrate/cmd/ate-setup/internal/cmd" - "go.ate.dev/substrate/cmd/ate-setup/internal/config" - "go.ate.dev/substrate/cmd/ate-setup/internal/demos" - // Registers every bundled demo, so demos.All() is the real list. - _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/all" -) - -// stubGo is a `go` that records the arguments of every `go run ./cmd/ate-setup` -// the script makes, one invocation per line, and succeeds. -const stubGo = `#!/usr/bin/env bash -printf '%s\n' "$*" >> "${SHIM_TEST_LOG}" -` - -// runShim runs hack/install-ate.sh with args and returns the recorded -// invocations and the script's exit status. -func runShim(t *testing.T, env []string, args ...string) (invocations []string, exitCode int) { - t.Helper() - - root := repoRoot(t) - bin := t.TempDir() - if err := os.WriteFile(filepath.Join(bin, "go"), []byte(stubGo), 0o755); err != nil { - t.Fatalf("writing the stub go: %v", err) - } - log := filepath.Join(t.TempDir(), "invocations") - - script := exec.Command("bash", filepath.Join(root, "hack", "install-ate.sh")) - script.Args = append(script.Args, args...) - script.Dir = root - // A fixed environment: the script reads SETUP_CSI and STORAGE_CLASS, so a - // developer who exports either would otherwise change the expected output. - // git needs PATH, and the stub go has to come first on it. - script.Env = append([]string{ - "PATH=" + bin + ":" + os.Getenv("PATH"), - "HOME=" + os.Getenv("HOME"), - "SHIM_TEST_LOG=" + log, - }, env...) - - out, err := script.CombinedOutput() - var exitErr *exec.ExitError - switch { - case err == nil: - case errors.As(err, &exitErr): - exitCode = exitErr.ExitCode() - default: - t.Fatalf("running install-ate.sh %v: %v\n%s", args, err, out) - } - - recorded, err := os.ReadFile(log) - if err != nil && !os.IsNotExist(err) { - t.Fatalf("reading the invocation log: %v", err) - } - for _, line := range strings.Split(string(recorded), "\n") { - if line != "" { - invocations = append(invocations, line) - assertResolves(t, line) - } - } - return invocations, exitCode -} - -// goRunPrefix is what the script puts in front of every command line, since it -// invokes the installer as `go run ./cmd/ate-setup`. -const goRunPrefix = "run ./cmd/ate-setup " - -// assertResolves checks that a command line the script produced names a command -// ate-setup actually has, with flags it accepts and arguments it allows. -// -// Without this the tests below only pin the script against itself: the stub go -// exits 0 whatever it is handed, so renaming `deploy apiserver` would keep them -// green while every job that goes through the shim broke. -func assertResolves(t *testing.T, invocation string) { - t.Helper() - - argv, ok := strings.CutPrefix(invocation, goRunPrefix) - if !ok { - t.Errorf("the script ran `go %s`, want it to run the installer as `go %s...`", invocation, goRunPrefix) - return - } - args := strings.Fields(argv) - - // Find walks as deep into the tree as the arguments name a command, and - // hands back what is left for that command to parse. - target, rest, err := cmd.Root().Find(args) - if err != nil { - t.Errorf("`ate-setup %s`: %v", argv, err) - return - } - if err := target.ParseFlags(rest); err != nil { - t.Errorf("`ate-setup %s`: %v", argv, err) - return - } - if err := target.ValidateArgs(target.Flags().Args()); err != nil { - t.Errorf("`ate-setup %s`: %v", argv, err) - return - } - // A command that only groups subcommands prints help and does nothing, so - // resolving to one means the script named a subcommand that is gone. - if !target.Runnable() { - t.Errorf("`ate-setup %s` resolves to %q, which is not runnable", argv, target.CommandPath()) - } -} - -func repoRoot(t *testing.T) string { - t.Helper() - out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() - if err != nil { - t.Fatalf("git rev-parse --show-toplevel: %v", err) - } - return strings.TrimSpace(string(out)) -} - -// The script keeps its own copy of the demo list so that --help answers -// without a build and an unknown --deploy-demo-* is rejected outright. That -// copy has to track the registry: a demo added only to internal/demos would -// otherwise have no installer flag, which is exactly the regression the shim -// exists to prevent. -func TestShimDemoListMatchesTheRegistry(t *testing.T) { - script, err := os.ReadFile(filepath.Join(repoRoot(t), "hack", "install-ate.sh")) - if err != nil { - t.Fatalf("reading install-ate.sh: %v", err) - } - match := regexp.MustCompile(`(?s)\nATE_DEMOS=\((.*?)\n\)`).FindSubmatch(script) - if match == nil { - t.Fatal("no ATE_DEMOS=( ... ) array in install-ate.sh") - } - listed := map[string]bool{} - for _, name := range strings.Fields(string(match[1])) { - listed[name] = true - } - - registered := map[string]bool{} - for _, d := range demos.All() { - registered[d.Name()] = true - } - - for name := range registered { - if !listed[name] { - t.Errorf("%s is registered but has no --deploy-%s flag; add it to ATE_DEMOS", name, name) - } - } - for name := range listed { - if !registered[name] { - t.Errorf("ATE_DEMOS lists %s, which no demo package registers", name) - } - } -} - -func TestShimTranslatesFlags(t *testing.T) { - const prefix = "run ./cmd/ate-setup " - - for _, tc := range []struct { - name string - env []string - args []string - want []string - }{{ - name: "deploy ate-system carries the CSI driver, and --setup-csi also acts on its own", - args: []string{"--deploy-ate-system", "--setup-csi=nfs"}, - want: []string{ - "deploy ate-system --setup-csi=nfs", - "setup csi nfs", - }, - }, { - name: "a bare --setup-csi means nfs", - args: []string{"--setup-csi"}, - want: []string{"setup csi nfs"}, - }, { - name: "--setup-csi takes a separated value", - args: []string{"--setup-csi", "hostpath"}, - want: []string{"setup csi hostpath"}, - }, { - name: "no CSI driver by default", - args: []string{"--deploy-ate-system"}, - want: []string{"deploy ate-system --setup-csi=none"}, - }, { - // The value-bearing flags were pre-scanned, so they shape every action - // regardless of where they appear. - name: "global flags apply to actions that precede them", - args: []string{"--deploy-atenet", "--atenet-dataplane", "agentgateway"}, - want: []string{"--atenet-dataplane=agentgateway deploy atenet"}, - }, { - name: "cluster profile flags are forwarded in either value form", - args: []string{"--deploy-ate-system", "--cluster-size", "size10", "--cordon-control-plane"}, - want: []string{"--cluster-size=size10 --cordon-control-plane deploy ate-system --setup-csi=none"}, - }, { - name: "--ingress-allowed-spiffe-ids is forwarded in either value form", - args: []string{"--deploy-ate-system", "--ingress-allowed-spiffe-ids", "spiffe://cluster.local/ns/a/sa/a", "--deploy-atenet", "--ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/b/sa/b"}, - want: []string{ - "--ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/a/sa/a --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/b/sa/b deploy ate-system --setup-csi=none", - "--ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/a/sa/a --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/b/sa/b deploy atenet", - }, - }, { - name: "--fake-data-plane is forwarded to every action", - args: []string{"--deploy-ate-system", "--fake-data-plane", "--deploy-atelet"}, - want: []string{ - "--fake-data-plane deploy ate-system --setup-csi=none", - "--fake-data-plane deploy atelet", - }, - }, { - name: "--cluster-size takes an attached value", - args: []string{"--cluster-size=size10", "--deploy-ate-apiserver"}, - want: []string{"--cluster-size=size10 deploy apiserver"}, - }, { - name: "--ateapi-log-level is forwarded in either value form", - args: []string{"--ateapi-log-level", "debug", "--deploy-ate-apiserver", "--ateapi-log-level=warn"}, - want: []string{"--ateapi-log-level=debug --ateapi-log-level=warn deploy apiserver"}, - }, { - name: "--credential-provider is forwarded with a separate value", - args: []string{"--deploy-atenet", "--credential-provider", `{"name":"custom","address":"cred.ate-system.svc:50051"}`}, - want: []string{`--credential-provider={"name":"custom","address":"cred.ate-system.svc:50051"} deploy atenet`}, - }, { - name: "--credential-provider takes an attached value", - args: []string{`--credential-provider={"name":"k8s.io"}`, "--deploy-ate-system"}, - want: []string{`--credential-provider={"name":"k8s.io"} deploy ate-system --setup-csi=none`}, - }, { - name: "actions run in command line order", - args: []string{"--deploy-ate-apiserver", "--deploy-atelet", "--delete-atenet"}, - want: []string{ - "deploy apiserver", - "deploy atelet", - "delete atenet", - }, - }, { - name: "every create flag maps to a create subcommand", - args: []string{ - "--create-jwt-authority-pool-secret", - "--create-actor-id-ca-pool-secret", - "--create-actor-id-ca-certs-secret", - "--create-egress-mitm-ca-pool-secret", - "--create-podcertificate-controller-cas", - "--create-api-server-env-vars", - "--create-api-authentication-config", - "--create-api-config", - }, - want: []string{ - "create jwt-authority-pool", - "create actor-id-ca-pool", - "create actor-id-ca-certs", - "create egress-mitm-ca-pool", - "create podcertificate-controller-cas", - "create api-server-env-vars", - "create api-authentication-config", - "create api-config", - }, - }, { - name: "delete flags", - args: []string{"--delete-ate-system", "--delete-all"}, - want: []string{"delete ate-system", "delete all"}, - }, { - name: "benchmark flags are renamed and apply to both benchmark actions", - args: []string{ - "--benchmark-worker-count=4", "--deploy-benchmarks", - "--benchmark-sandbox-class", "microvm", "--delete-benchmarks", - }, - want: []string{ - "deploy benchmarks --worker-count=4 --sandbox-class=microvm", - "delete benchmarks --worker-count=4 --sandbox-class=microvm", - }, - }, { - name: "demo flags drop the demo- prefix", - args: []string{"--deploy-demo-egress-microvm", "--delete-demo-counter-microvm"}, - want: []string{ - "deploy demo egress-microvm", - "delete demo counter-microvm", - }, - }, { - name: "the counter external volume defaults to the standard storage class", - args: []string{"--deploy-demo-counter-with-external-volume"}, - want: []string{"deploy demo counter --with-external-volume --storage-class=standard"}, - }, { - name: "the counter external volume follows --setup-csi", - args: []string{"--setup-csi=hostpath", "--deploy-demo-counter-with-external-volume"}, - want: []string{ - "setup csi hostpath", - "deploy demo counter --with-external-volume --storage-class=csi-hostpath-sc", - }, - }, { - name: "STORAGE_CLASS beats the driver --setup-csi installed", - env: []string{"STORAGE_CLASS=my-class"}, - args: []string{"--setup-csi=nfs", "--deploy-demo-counter-with-external-volume"}, - want: []string{ - "setup csi nfs", - "deploy demo counter --with-external-volume --storage-class=my-class", - }, - }} { - t.Run(tc.name, func(t *testing.T) { - got, exitCode := runShim(t, tc.env, tc.args...) - if exitCode != 0 { - t.Fatalf("exit code = %d, want 0", exitCode) - } - want := make([]string, len(tc.want)) - for i, w := range tc.want { - want[i] = prefix + w - } - if strings.Join(got, "\n") != strings.Join(want, "\n") { - t.Errorf("invocations:\n got %q\n want %q", got, want) - } - }) - } -} - -// --benchmark-actor-memory has no ate-setup flag; the script exports it, which -// the stub go cannot observe through its arguments. -func TestShimExportsBenchmarkActorMemory(t *testing.T) { - root := repoRoot(t) - bin := t.TempDir() - const reportEnv = `#!/usr/bin/env bash -printf '%s\n' "BENCHMARK_ACTOR_MEMORY=${BENCHMARK_ACTOR_MEMORY:-}" -` - if err := os.WriteFile(filepath.Join(bin, "go"), []byte(reportEnv), 0o755); err != nil { - t.Fatalf("writing the stub go: %v", err) - } - - script := exec.Command("bash", filepath.Join(root, "hack", "install-ate.sh"), - "--benchmark-actor-memory", "512Mi", "--deploy-benchmarks") - script.Dir = root - script.Env = []string{"PATH=" + bin + ":" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME")} - - out, err := script.CombinedOutput() - if err != nil { - t.Fatalf("running install-ate.sh: %v\n%s", err, out) - } - if want := "BENCHMARK_ACTOR_MEMORY=512Mi\n"; string(out) != want { - t.Errorf("stub go saw %q, want %q", out, want) - } -} - -func TestShimRejectsUnknownFlags(t *testing.T) { - for _, args := range [][]string{ - {"--no-such-flag"}, - {"--deploy-demo-no-such-demo"}, - {"--delete-demo-no-such-demo"}, - } { - t.Run(strings.Join(args, " "), func(t *testing.T) { - got, exitCode := runShim(t, nil, args...) - if exitCode != 1 { - t.Errorf("exit code = %d, want 1", exitCode) - } - if len(got) != 0 { - t.Errorf("ran %q, want nothing", got) - } - }) - } -} - -// An action before an unknown flag still runs: the script dispatches as it -// walks the command line, as it always has. -func TestShimRunsActionsBeforeAnUnknownFlag(t *testing.T) { - got, exitCode := runShim(t, nil, "--deploy-atelet", "--no-such-flag") - if exitCode != 1 { - t.Errorf("exit code = %d, want 1", exitCode) - } - if want := []string{"run ./cmd/ate-setup deploy atelet"}; strings.Join(got, "\n") != strings.Join(want, "\n") { - t.Errorf("invocations = %q, want %q", got, want) - } -} - -func TestShimUsage(t *testing.T) { - for _, tc := range []struct { - name string - args []string - exitCode int - }{ - {name: "no arguments is an error", args: nil, exitCode: 1}, - {name: "--help anywhere exits cleanly", args: []string{"--deploy-ate-system", "--help"}, exitCode: 0}, - {name: "-h anywhere exits cleanly", args: []string{"-h"}, exitCode: 0}, - } { - t.Run(tc.name, func(t *testing.T) { - got, exitCode := runShim(t, nil, tc.args...) - if exitCode != tc.exitCode { - t.Errorf("exit code = %d, want %d", exitCode, tc.exitCode) - } - if len(got) != 0 { - t.Errorf("ran %q, want nothing: usage must not touch the cluster", got) - } - }) - } -} - -// Every flag the script dispatches on has to be documented, since --help is -// the only place a user can discover them now that the demos are Go packages. -func TestShimUsageDocumentsEveryFlag(t *testing.T) { - script, err := os.ReadFile(filepath.Join(repoRoot(t), "hack", "install-ate.sh")) - if err != nil { - t.Fatalf("reading install-ate.sh: %v", err) - } - usage := shimUsage(t) - - // The dispatch arms below `run_demo`'s definition, minus the demo - // wildcards, which usage covers one demo at a time. - for _, flag := range regexp.MustCompile(`(?m)^ (--[a-z0-9-]+)[)|]`).FindAllStringSubmatch(string(script), -1) { - if !strings.Contains(usage, flag[1]) { - t.Errorf("--help does not mention %s", flag[1]) - } - } - for _, d := range demos.All() { - for _, flag := range []string{"--deploy-" + d.Name(), "--delete-" + d.Name()} { - if !strings.Contains(usage, flag) { - t.Errorf("--help does not mention %s", flag) - } - } - } -} - -func shimUsage(t *testing.T) string { - t.Helper() - root := repoRoot(t) - script := exec.Command("bash", filepath.Join(root, "hack", "install-ate.sh"), "--help") - script.Dir = root - script.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME")} - out, err := script.Output() - if err != nil { - t.Fatalf("install-ate.sh --help: %v", err) - } - return string(out) -} - -// Settings register from each command package's init, so the merged set only -// exists in a binary that links the whole command tree. Package config cannot -// check it: the commands that register are the ones config does not import. -// This package does import them, which is what makes the check possible here -// and nowhere below it. -func TestSettingRegistryIsValidAcrossEveryCommand(t *testing.T) { - if err := config.Validate(); err != nil { - t.Fatal(err) - } -} - -// Command-scoped settings exist so a flag appears only where it applies. A -// setting that named a command nobody built would silently never bind. -func TestEveryScopedSettingNamesARealCommand(t *testing.T) { - for _, s := range config.All() { - for _, path := range s.Commands { - t.Run(s.Key+" on "+path, func(t *testing.T) { - target, _, err := cmd.Root().Find(strings.Fields(path)) - if err != nil { - t.Fatalf("Find(%q): %v", path, err) - } - if target.CommandPath() != "ate-setup "+path { - t.Fatalf("%q resolved to %q, which is not that command", path, target.CommandPath()) - } - if target.Flags().Lookup(s.Flag) == nil { - t.Errorf("--%s is not bound to %q", s.Flag, path) - } - }) - } - } -} - -// docPath resolves a repository path from this package's directory. -func docPath(t *testing.T, rel string) string { - t.Helper() - root, err := config.RepoRoot() - if err != nil { - t.Fatalf("RepoRoot: %v", err) - } - return filepath.Join(root, rel) -} - -// The worked example is the first thing an operator copies, so a renamed or -// removed key has to break the build rather than their install. -func TestExampleConfigurationParses(t *testing.T) { - path := docPath(t, "docs/examples/substrate-install.yaml") - f, err := config.ParseFile(path) - if err != nil { - t.Fatalf("ParseFile(%s) = %v", path, err) - } - if len(f.Keys()) == 0 { - t.Error("the example sets no settings, so it demonstrates nothing") - } -} - -const operatorDoc = "docs/operator-install.md" - -// settingDocs maps a key prefix to the document that lists the settings under -// it. Demo, benchmark, and e2e settings are documented with what uses them, -// so the operator page lists only what an install uses. -var settingDocs = []struct{ prefix, doc string }{ - {"demo.counter.", "demos/counter/README.md"}, - {"demo.anthropicAPIKey", "demos/claude-code-multiplex/README.md"}, - {"benchmark.", "benchmarking/README.md"}, - {"e2e.", "internal/e2e/README.md"}, -} - -// settingDoc returns the document that lists the setting with this key. -func settingDoc(key string) string { - for _, d := range settingDocs { - if strings.HasPrefix(key, d.prefix) { - return d.doc - } - } - return operatorDoc -} - -// readDoc returns the contents of a repository file, by relative path. -func readDoc(t *testing.T, rel string) string { - t.Helper() - raw, err := os.ReadFile(docPath(t, rel)) - if err != nil { - t.Fatalf("ReadFile: %v", err) - } - return string(raw) -} - -// The settings tables are written by hand, so a setting added without -// documenting it is the drift this catches. A demo, benchmark, or e2e setting -// must also stay out of the operator page. -func TestDocsListEverySetting(t *testing.T) { - docs := map[string]string{operatorDoc: readDoc(t, operatorDoc)} - for _, d := range settingDocs { - docs[d.doc] = readDoc(t, d.doc) - } - - for _, s := range config.All() { - t.Run(s.Key, func(t *testing.T) { - rel := settingDoc(s.Key) - want := []string{"`" + s.Key + "`", "`" + s.Env + "`"} - // A secret has no flag, so the docs have none to name. - if s.Flag != "" { - want = append(want, "`--"+s.Flag+"`") - } - for _, want := range want { - if !strings.Contains(docs[rel], want) { - t.Errorf("%s does not mention %s", rel, want) - } - } - if rel != operatorDoc && strings.Contains(docs[operatorDoc], "`"+s.Key+"`") { - t.Errorf("%s lists %s, which belongs in %s", operatorDoc, s.Key, rel) - } - }) - } -} - -// The settings documentation must not name a setting that no longer exists: -// following it would configure nothing and the install would silently use a -// default. -func TestDocsNameNoRemovedSetting(t *testing.T) { - // The two controls that select how configuration is read and reported. - // They are not settings themselves, so no registry entry declares them. - notSettings := map[string]bool{ - "ATE_CONFIG": true, - "ATE_NO_REPORT_DEFAULTS": true, - } - - declared := map[string]bool{} - for _, s := range config.All() { - declared[s.Env] = true - } - - root, err := config.RepoRoot() - if err != nil { - t.Fatalf("RepoRoot: %v", err) - } - files := []string{operatorDoc, "docs/examples/substrate-install.yaml"} - for _, d := range settingDocs { - files = append(files, d.doc) - } - for _, rel := range files { - raw, err := os.ReadFile(filepath.Join(root, rel)) - if err != nil { - t.Fatalf("ReadFile(%s): %v", rel, err) - } - seen := map[string]bool{} - for _, m := range regexp.MustCompile(`\b(ATE_[A-Z0-9_]{3,})\b`).FindAllStringSubmatch(string(raw), -1) { - name := m[1] - if declared[name] || notSettings[name] || seen[name] { - continue - } - seen[name] = true - t.Errorf("%s names %s, which no setting declares", rel, name) - } - } -} diff --git a/cmd/ateapi/helm_test.go b/cmd/ateapi/helm_test.go index a3855c5f04..ca3fe4c213 100644 --- a/cmd/ateapi/helm_test.go +++ b/cmd/ateapi/helm_test.go @@ -316,6 +316,30 @@ func TestHelmObjectStorePlugins(t *testing.T) { if mode == "node" && (!slices.Contains(plugin.Args, "--root=/var/lib/ate") || !slices.ContainsFunc(plugin.VolumeMounts, func(m corev1.VolumeMount) bool { return m.Name == "ate-base" && m.MountPath == "/var/lib/ate" })) { t.Error("node plugin cannot access snapshot files") } + if mode == "node" { + for _, arg := range []string{"--exchange-address=test-api.custom.svc:8445", "--exchange-server-name=test-api.custom.svc"} { + if !slices.Contains(plugin.Args, arg) { + t.Errorf("node plugin is missing %s", arg) + } + } + if !slices.ContainsFunc(plugin.Env, func(e corev1.EnvVar) bool { + return e.Name == "POD_NAMESPACE" && e.ValueFrom != nil && e.ValueFrom.FieldRef != nil && e.ValueFrom.FieldRef.FieldPath == "metadata.namespace" + }) { + t.Error("node plugin is missing its namespace") + } + if !slices.ContainsFunc(plugin.VolumeMounts, func(m corev1.VolumeMount) bool { + return m.Name == "servicedns-ca" && m.MountPath == "/run/servicedns.podcert.ate.dev" && m.ReadOnly + }) { + t.Error("node plugin cannot verify the exchange certificate") + } + for _, container := range append(pod.Containers, plugin) { + if !slices.ContainsFunc(container.VolumeMounts, func(m corev1.VolumeMount) bool { + return m.Name == "ate-base" && m.MountPropagation != nil && *m.MountPropagation == corev1.MountPropagationHostToContainer + }) { + t.Errorf("%s cannot see mounts created after startup", container.Name) + } + } + } found[mode] = true } for _, name := range []string{"issuer", "control", "node"} { diff --git a/cmd/ateapi/internal/apivalidation/actor_template.go b/cmd/ateapi/internal/apivalidation/actor_template.go index 2931bb2a71..3e21dca0c1 100644 --- a/cmd/ateapi/internal/apivalidation/actor_template.go +++ b/cmd/ateapi/internal/apivalidation/actor_template.go @@ -57,10 +57,12 @@ func ValidateActorTemplateUpdate(ctx context.Context, fldPath *field.Path, newVa } // ValidateCustom_CreateActorTemplateRequest_ActorTemplate holds the rules -// that span more than one field of a template: container volume mounts must -// reference volumes the template declares. +// that span more than one field of a template. func ValidateCustom_CreateActorTemplateRequest_ActorTemplate(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *ateapipb.ActorTemplate) field.ErrorList { var errs field.ErrorList + if value.GetGoldenEgressPolicy() != nil && value.GetSnapshotConfig().GetGoldenSnapshotConfig().GetMode() == ateapipb.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED { + errs = append(errs, field.Forbidden(fldPath.Child("golden_egress_policy"), "requires golden snapshots to be enabled")) + } declared := make(map[string]bool, len(value.GetVolumes())) for _, vol := range value.GetVolumes() { declared[vol.GetName()] = true diff --git a/cmd/ateapi/internal/apivalidation/actor_template_test.go b/cmd/ateapi/internal/apivalidation/actor_template_test.go index 480e86f3a9..f5417f84e6 100644 --- a/cmd/ateapi/internal/apivalidation/actor_template_test.go +++ b/cmd/ateapi/internal/apivalidation/actor_template_test.go @@ -35,6 +35,48 @@ func TestValidateCreateActorTemplateRequest(t *testing.T) { "valid", &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate()}, nil, + }, { + "valid golden egress rules", + &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { + tmpl.GoldenEgressPolicy = &ateapipb.EgressPolicyTemplate{Rules: []*ateapipb.EgressRule{{Http: &ateapipb.HttpRule{Hostnames: []string{"api.example.com"}, Ports: &ateapipb.Ports{Numbers: []int32{80}}}}}} + })}, + nil, + }, { + "missing golden egress rules", + &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { + tmpl.GoldenEgressPolicy = &ateapipb.EgressPolicyTemplate{} + })}, + field.ErrorList{field.Required(field.NewPath("actor_template", "golden_egress_policy", "rules"), "")}, + }, { + "empty golden egress rules", + &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { + tmpl.GoldenEgressPolicy = &ateapipb.EgressPolicyTemplate{Rules: []*ateapipb.EgressRule{}} + })}, + field.ErrorList{field.Required(field.NewPath("actor_template", "golden_egress_policy", "rules"), "")}, + }, { + "invalid golden egress hostname", + &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { + tmpl.GoldenEgressPolicy = &ateapipb.EgressPolicyTemplate{Rules: []*ateapipb.EgressRule{{Http: &ateapipb.HttpRule{Hostnames: []string{"bad/host"}, Ports: &ateapipb.Ports{Numbers: []int32{80}}}}}} + })}, + field.ErrorList{field.Invalid(field.NewPath("actor_template", "golden_egress_policy", "rules").Index(0).Child("http", "hostnames").Index(0), nil, "")}, + }, { + "golden egress rules tie across protocols", + &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { + tmpl.GoldenEgressPolicy = &ateapipb.EgressPolicyTemplate{Rules: []*ateapipb.EgressRule{ + {Https: &ateapipb.HttpsRule{Hostnames: []string{"api.example.com"}, Ports: &ateapipb.Ports{Numbers: []int32{443}}}}, + {TlsPassthrough: &ateapipb.TlsPassthroughRule{Hostnames: []string{"api.example.com"}, Ports: &ateapipb.Ports{Numbers: []int32{443}}}}, + }} + })}, + field.ErrorList{field.Invalid(field.NewPath("actor_template", "golden_egress_policy", "rules").Index(1).Child("tls_passthrough", "hostnames").Index(0), nil, "")}, + }, { + "too many golden egress rules", + &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { + tmpl.GoldenEgressPolicy = &ateapipb.EgressPolicyTemplate{} + for i := range 257 { + tmpl.GoldenEgressPolicy.Rules = append(tmpl.GoldenEgressPolicy.Rules, &ateapipb.EgressRule{Http: &ateapipb.HttpRule{Hostnames: []string{fmt.Sprintf("host-%d.example.com", i)}, Ports: &ateapipb.Ports{Numbers: []int32{80}}}}) + } + })}, + field.ErrorList{field.TooMany(field.NewPath("actor_template", "golden_egress_policy", "rules"), 257, 256).WithOrigin("maxItems")}, }, { "missing actor_template", &ateapipb.CreateActorTemplateRequest{}, @@ -182,7 +224,15 @@ func TestValidateCreateActorTemplateRequest(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assertValidateErr(t, ValidateCreateActorTemplateRequest(context.Background(), tt.req), tt.want) + want := tt.want + var wantWithPreview map[string]field.ErrorList + if tt.req.GetActorTemplate().GetGoldenEgressPolicy() != nil { + want = field.ErrorList{field.Forbidden(field.NewPath("actor_template", "golden_egress_policy"), "")} + wantWithPreview = map[string]field.ErrorList{"GoldenEgressPolicy": tt.want, "ALL": tt.want} + } + assertValidate(t, func() field.ErrorList { + return ValidateCreateActorTemplateRequest(context.Background(), tt.req) + }, want, wantWithPreview) }) } } diff --git a/cmd/ateapi/internal/apivalidation/actor_test.go b/cmd/ateapi/internal/apivalidation/actor_test.go index 6223585c5a..652d4e8fb6 100644 --- a/cmd/ateapi/internal/apivalidation/actor_test.go +++ b/cmd/ateapi/internal/apivalidation/actor_test.go @@ -369,6 +369,27 @@ func TestValidateActorUpdate(t *testing.T) { want: field.ErrorList{ field.Invalid(field.NewPath("status", "worker_assignment", "worker_spiffe_id"), nil, ""), }, + }, { + name: "set valid actor.status.worker_assignment.sandbox_runtime", + oldVal: validInput(withStatus()), + newVal: validOutput(withStatus(withWorkerAssignment(func(wa *ateapipb.WorkerAssignment) { + wa.SandboxRuntime = &ateapipb.ActorSandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{SchemaVersion: "v1", Attributes: []*ateapipb.AttributeEntry{{Key: "architecture", Value: "amd64"}}}, + } + }))), + }, { + name: "invalid actor.status.worker_assignment.sandbox_runtime.sandbox_class", + oldVal: validInput(withStatus()), + newVal: validOutput(withStatus(withWorkerAssignment(func(wa *ateapipb.WorkerAssignment) { + wa.SandboxRuntime = &ateapipb.ActorSandboxRuntime{ + SandboxClass: "plain", + Version: &ateapipb.VersionedSandboxCompat{SchemaVersion: "v1", Attributes: []*ateapipb.AttributeEntry{{Key: "architecture", Value: "amd64"}}}, + } + }))), + want: field.ErrorList{ + field.NotSupported[string](field.NewPath("status", "worker_assignment", "sandbox_runtime", "sandbox_class"), nil, nil), + }, }, { name: "clear actor.status.worker_assignment", oldVal: validInput(withStatus(withWorkerAssignment())), diff --git a/cmd/ateapi/internal/apivalidation/egress_policy.go b/cmd/ateapi/internal/apivalidation/egress_policy.go index be444b47cb..4fa367ede0 100644 --- a/cmd/ateapi/internal/apivalidation/egress_policy.go +++ b/cmd/ateapi/internal/apivalidation/egress_policy.go @@ -126,6 +126,10 @@ func ValidateCustom_EgressPolicy_Rules(_ context.Context, _ operation.Operation, return errs } +func ValidateCustom_EgressPolicyTemplate_Rules(ctx context.Context, op operation.Operation, p *field.Path, rules, oldRules []*ateapipb.EgressRule) field.ErrorList { + return ValidateCustom_EgressPolicy_Rules(ctx, op, p, rules, oldRules) +} + // ruleMatchFields is what a rule matches on: the union member, its hostnames, // and its ports. Everything is empty for a rule that sets no member. func ruleMatchFields(rule *ateapipb.EgressRule) (member string, hostnames []string, ports *ateapipb.Ports) { diff --git a/cmd/ateapi/internal/apivalidation/worker.go b/cmd/ateapi/internal/apivalidation/worker.go index 6f53f82eb5..4d3ce0b5fa 100644 --- a/cmd/ateapi/internal/apivalidation/worker.go +++ b/cmd/ateapi/internal/apivalidation/worker.go @@ -126,11 +126,11 @@ func ValidateWorkerUpdate(ctx context.Context, fldPath *field.Path, newVal, oldV return errs } -func ValidateCustom_Worker_SandboxClass(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { +func ValidateCustom_ActorSandboxRuntime_SandboxClass(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { return resources.ValidateSandboxClass(fldPath, *value) } -func ValidateCustom_SandboxRuntime_SandboxClass(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { +func ValidateCustom_WorkerSandboxClass_SandboxClass(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { return resources.ValidateSandboxClass(fldPath, *value) } diff --git a/cmd/ateapi/internal/apivalidation/worker_test.go b/cmd/ateapi/internal/apivalidation/worker_test.go index a0b60ea8ea..05948affc6 100644 --- a/cmd/ateapi/internal/apivalidation/worker_test.go +++ b/cmd/ateapi/internal/apivalidation/worker_test.go @@ -159,17 +159,6 @@ func TestValidateCreateWorkerRequest(t *testing.T) { w.WorkerSpiffeId = "spiffe://cluster.local/" + strings.Repeat("a", 2048) })), want: field.ErrorList{field.TooLong(field.NewPath("worker", "worker_spiffe_id"), nil, 2048).WithOrigin("maxLength")}, - }, { - name: "sandbox_class too long", - req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { w.SandboxClass = strings.Repeat("x", 64) })), - want: field.ErrorList{ - field.TooLong(field.NewPath("worker", "sandbox_class"), nil, 63).WithOrigin("maxLength"), - field.NotSupported[string](field.NewPath("worker", "sandbox_class"), nil, nil), - }, - }, { - name: "unsupported sandbox_class", - req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { w.SandboxClass = "kata" })), - want: field.ErrorList{field.NotSupported[string](field.NewPath("worker", "sandbox_class"), nil, nil)}, }, { name: "valid labels", req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { @@ -349,7 +338,6 @@ func TestValidateWorkerUpdate_RequireStatus(t *testing.T) { } func TestValidateRegisterWorkerRequest(t *testing.T) { - validRuntime := validSandboxRuntime valid := func(mutate ...func(*ateapipb.RegisterWorkerRequest)) *ateapipb.RegisterWorkerRequest { r := &ateapipb.RegisterWorkerRequest{ Worker: workerRef(apiWorkerName), @@ -357,15 +345,18 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { Actors: 10, CpuMillicores: 2000, MemoryBytes: 4 << 30, }, - DefaultRuntime: validRuntime(), + SandboxClasses: []*ateapipb.WorkerSandboxClass{validSandboxClass()}, } for _, m := range mutate { m(r) } return r } - defaultRuntimePath := field.NewPath("default_runtime") - restorablePath := field.NewPath("restorable_runtimes") + classesPath := field.NewPath("sandbox_classes") + classPath := classesPath.Index(0) + defaultRuntimePath := classPath.Child("default_runtime") + defaultVersionPath := defaultRuntimePath.Child("version") + restorablePath := classPath.Child("restorable_runtimes") tests := []struct { name string req *ateapipb.RegisterWorkerRequest @@ -376,15 +367,12 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { }, { name: "valid with restorable runtimes", req: valid(func(r *ateapipb.RegisterWorkerRequest) { - r.RestorableRuntimes = []*ateapipb.SandboxRuntime{{ - SandboxClass: "gvisor", - Version: &ateapipb.VersionedSandboxCompat{ - SchemaVersion: "v1", - Attributes: []*ateapipb.AttributeEntry{ - {Key: "architecture", Value: "amd64"}, - }, + r.SandboxClasses[0].RestorableRuntimes = []*ateapipb.WorkerSandboxRuntime{{Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, }, - }} + }}} }), }, { name: "valid empty capacity", @@ -418,109 +406,126 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.Capacity.MemoryBytes = -1 }), want: field.ErrorList{field.Invalid(field.NewPath("capacity", "memory_bytes"), nil, "").WithOrigin("minimum")}, }, { - name: "missing default_runtime", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.DefaultRuntime = nil }), - want: field.ErrorList{field.Required(defaultRuntimePath, "")}, + name: "missing sandbox_classes", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses = nil }), + want: field.ErrorList{field.Required(classesPath, "")}, + }, { + name: "empty sandbox_classes", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses = []*ateapipb.WorkerSandboxClass{} }), + want: field.ErrorList{field.Required(classesPath, "")}, + }, { + name: "too many sandbox_classes", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { + second := validSandboxClass() + second.SandboxClass = "microvm" + r.SandboxClasses = append(r.SandboxClasses, second) + }), + want: field.ErrorList{field.TooMany(classesPath, 2, 1).WithOrigin("maxItems")}, }, { - name: "missing default_runtime.sandbox_class", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.DefaultRuntime.SandboxClass = "" }), - want: field.ErrorList{field.Required(defaultRuntimePath.Child("sandbox_class"), "")}, + name: "missing sandbox_classes.sandbox_class", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses[0].SandboxClass = "" }), + want: field.ErrorList{field.Required(classPath.Child("sandbox_class"), "")}, }, { - name: "default_runtime.sandbox_class too long", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.DefaultRuntime.SandboxClass = strings.Repeat("c", 64) }), + name: "sandbox_classes.sandbox_class too long", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses[0].SandboxClass = strings.Repeat("c", 64) }), want: field.ErrorList{ - field.TooLong(defaultRuntimePath.Child("sandbox_class"), "", 63).WithOrigin("maxLength"), - field.NotSupported[string](defaultRuntimePath.Child("sandbox_class"), nil, nil), + field.TooLong(classPath.Child("sandbox_class"), "", 63).WithOrigin("maxLength"), + field.NotSupported[string](classPath.Child("sandbox_class"), nil, nil), }, }, { - name: "unsupported default_runtime.sandbox_class", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.DefaultRuntime.SandboxClass = "kata" }), - want: field.ErrorList{field.NotSupported[string](defaultRuntimePath.Child("sandbox_class"), nil, nil)}, + name: "unsupported sandbox_classes.sandbox_class", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses[0].SandboxClass = "kata" }), + want: field.ErrorList{field.NotSupported[string](classPath.Child("sandbox_class"), nil, nil)}, + }, { + name: "missing sandbox_classes.default_runtime", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses[0].DefaultRuntime = nil }), + want: field.ErrorList{field.Required(defaultRuntimePath, "")}, }, { name: "missing default_runtime.version", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.DefaultRuntime.Version = nil }), - want: field.ErrorList{field.Required(defaultRuntimePath.Child("version"), "")}, + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses[0].DefaultRuntime.Version = nil }), + want: field.ErrorList{field.Required(defaultVersionPath, "")}, }, { - name: "missing default_runtime.version.schema_version", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.DefaultRuntime.Version.SchemaVersion = "" }), - want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "schema_version"), "")}, + name: "missing default_runtime.schema_version", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses[0].DefaultRuntime.Version.SchemaVersion = "" }), + want: field.ErrorList{field.Required(defaultVersionPath.Child("schema_version"), "")}, }, { - name: "default_runtime.version.schema_version too long", + name: "default_runtime.schema_version too long", req: valid(func(r *ateapipb.RegisterWorkerRequest) { - r.DefaultRuntime.Version.SchemaVersion = strings.Repeat("v", 65) + r.SandboxClasses[0].DefaultRuntime.Version.SchemaVersion = strings.Repeat("v", 65) }), - want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("version", "schema_version"), "", 64).WithOrigin("maxLength")}, + want: field.ErrorList{field.TooLong(defaultVersionPath.Child("schema_version"), "", 64).WithOrigin("maxLength")}, }, { - name: "missing default_runtime.version.attributes", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.DefaultRuntime.Version.Attributes = nil }), - want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "attributes"), "")}, + name: "missing default_runtime.attributes", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses[0].DefaultRuntime.Version.Attributes = nil }), + want: field.ErrorList{field.Required(defaultVersionPath.Child("attributes"), "")}, }, { name: "duplicate attribute key", req: valid(func(r *ateapipb.RegisterWorkerRequest) { - r.DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{ + r.SandboxClasses[0].DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{ {Key: "architecture", Value: "amd64"}, {Key: "architecture", Value: "arm64"}, } }), - want: field.ErrorList{field.Duplicate(defaultRuntimePath.Child("version", "attributes").Index(1), nil)}, + want: field.ErrorList{field.Duplicate(defaultVersionPath.Child("attributes").Index(1), nil)}, }, { name: "missing attribute key", req: valid(func(r *ateapipb.RegisterWorkerRequest) { - r.DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Value: "amd64"}} + r.SandboxClasses[0].DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Value: "amd64"}} }), - want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "attributes").Index(0).Child("key"), "")}, + want: field.ErrorList{field.Required(defaultVersionPath.Child("attributes").Index(0).Child("key"), "")}, }, { name: "attribute key too long", req: valid(func(r *ateapipb.RegisterWorkerRequest) { - r.DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Key: strings.Repeat("k", 129), Value: "v"}} + r.SandboxClasses[0].DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Key: strings.Repeat("k", 129), Value: "v"}} }), - want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("version", "attributes").Index(0).Child("key"), "", 128).WithOrigin("maxLength")}, + want: field.ErrorList{field.TooLong(defaultVersionPath.Child("attributes").Index(0).Child("key"), "", 128).WithOrigin("maxLength")}, }, { name: "missing attribute value", req: valid(func(r *ateapipb.RegisterWorkerRequest) { - r.DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Key: "k"}} + r.SandboxClasses[0].DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Key: "k"}} }), - want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "attributes").Index(0).Child("value"), "")}, + want: field.ErrorList{field.Required(defaultVersionPath.Child("attributes").Index(0).Child("value"), "")}, }, { name: "attribute value too long", req: valid(func(r *ateapipb.RegisterWorkerRequest) { - r.DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Key: "k", Value: strings.Repeat("v", 257)}} + r.SandboxClasses[0].DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Key: "k", Value: strings.Repeat("v", 257)}} }), - want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("version", "attributes").Index(0).Child("value"), "", 256).WithOrigin("maxLength")}, + want: field.ErrorList{field.TooLong(defaultVersionPath.Child("attributes").Index(0).Child("value"), "", 256).WithOrigin("maxLength")}, }, { name: "attributes at the bound", req: valid(func(r *ateapipb.RegisterWorkerRequest) { - r.DefaultRuntime.Version.Attributes = attributeEntries(32) + r.SandboxClasses[0].DefaultRuntime.Version.Attributes = attributeEntries(32) }), }, { name: "too many attributes", req: valid(func(r *ateapipb.RegisterWorkerRequest) { - r.DefaultRuntime.Version.Attributes = attributeEntries(33) + r.SandboxClasses[0].DefaultRuntime.Version.Attributes = attributeEntries(33) }), - want: field.ErrorList{field.TooMany(defaultRuntimePath.Child("version", "attributes"), 33, 32).WithOrigin("maxItems")}, + want: field.ErrorList{field.TooMany(defaultVersionPath.Child("attributes"), 33, 32).WithOrigin("maxItems")}, }, { name: "nil restorable_runtimes entry", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.RestorableRuntimes = []*ateapipb.SandboxRuntime{nil} }), + req: valid(func(r *ateapipb.RegisterWorkerRequest) { + r.SandboxClasses[0].RestorableRuntimes = []*ateapipb.WorkerSandboxRuntime{nil} + }), want: field.ErrorList{field.Required(restorablePath.Index(0), "")}, }, { name: "invalid restorable_runtimes entry", req: valid(func(r *ateapipb.RegisterWorkerRequest) { - r.RestorableRuntimes = []*ateapipb.SandboxRuntime{{SandboxClass: "gvisor"}} + r.SandboxClasses[0].RestorableRuntimes = []*ateapipb.WorkerSandboxRuntime{{Version: &ateapipb.VersionedSandboxCompat{Attributes: attributeEntries(1)}}} }), - want: field.ErrorList{field.Required(restorablePath.Index(0).Child("version"), "")}, + want: field.ErrorList{field.Required(restorablePath.Index(0).Child("version", "schema_version"), "")}, }, { - name: "unsupported restorable_runtimes sandbox_class", + name: "restorable_runtimes entry without a version", req: valid(func(r *ateapipb.RegisterWorkerRequest) { - r.RestorableRuntimes = []*ateapipb.SandboxRuntime{validSandboxRuntime()} - r.RestorableRuntimes[0].SandboxClass = "GVISOR" + r.SandboxClasses[0].RestorableRuntimes = []*ateapipb.WorkerSandboxRuntime{{}} }), - want: field.ErrorList{field.NotSupported[string](restorablePath.Index(0).Child("sandbox_class"), nil, nil)}, + want: field.ErrorList{field.Required(restorablePath.Index(0).Child("version"), "")}, }, { name: "restorable_runtimes at the bound", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.RestorableRuntimes = sandboxRuntimes(32) }), + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses[0].RestorableRuntimes = sandboxRuntimes(32) }), }, { name: "too many restorable_runtimes", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.RestorableRuntimes = sandboxRuntimes(33) }), + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses[0].RestorableRuntimes = sandboxRuntimes(33) }), want: field.ErrorList{field.TooMany(restorablePath, 33, 32).WithOrigin("maxItems")}, }} for _, tt := range tests { @@ -530,12 +535,17 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { } } -// TestValidateWorkerUpdate_SandboxRuntimes pins the status-side checks on the -// runtimes RegisterWorker records. They reach the store as status, which no -// request validator sees, so ValidateWorkerUpdate is the only check on them. -func TestValidateWorkerUpdate_SandboxRuntimes(t *testing.T) { - statusPath := field.NewPath("worker", "status") - oldVal := validWorker(apiWorkerName, withWorkerStatus()) +// TestValidateWorkerUpdate_SandboxClasses pins the status-side checks on the +// sandbox classes RegisterWorker records. They reach the store as status, +// which no request validator sees, so ValidateWorkerUpdate is the only check +// on them. The old worker already reports a class so that every mutation, +// clearing the list included, differs from it and is validated. +func TestValidateWorkerUpdate_SandboxClasses(t *testing.T) { + classesPath := field.NewPath("worker", "status", "sandbox_classes") + classPath := classesPath.Index(0) + oldVal := validWorker(apiWorkerName, withWorkerStatus(func(s *ateapipb.WorkerStatus) { + s.SandboxClasses = []*ateapipb.WorkerSandboxClass{validSandboxClass()} + })) tests := []struct { name string mutate func(s *ateapipb.WorkerStatus) @@ -543,48 +553,53 @@ func TestValidateWorkerUpdate_SandboxRuntimes(t *testing.T) { }{{ name: "valid runtimes", mutate: func(s *ateapipb.WorkerStatus) { - s.DefaultRuntime = validSandboxRuntime() - s.RestorableRuntimes = sandboxRuntimes(32) + s.SandboxClasses[0].RestorableRuntimes = sandboxRuntimes(32) }, }, { - name: "missing default_runtime.sandbox_class", - mutate: func(s *ateapipb.WorkerStatus) { - s.DefaultRuntime = validSandboxRuntime() - s.DefaultRuntime.SandboxClass = "" - }, - want: field.ErrorList{field.Required(statusPath.Child("default_runtime", "sandbox_class"), "")}, + name: "empty sandbox_classes", + mutate: func(s *ateapipb.WorkerStatus) { s.SandboxClasses = []*ateapipb.WorkerSandboxClass{} }, }, { - name: "unsupported default_runtime.sandbox_class", + name: "too many sandbox_classes", mutate: func(s *ateapipb.WorkerStatus) { - s.DefaultRuntime = validSandboxRuntime() - s.DefaultRuntime.SandboxClass = "kata" + second := validSandboxClass() + second.SandboxClass = "microvm" + s.SandboxClasses = append(s.SandboxClasses, second) }, - want: field.ErrorList{field.NotSupported[string](statusPath.Child("default_runtime", "sandbox_class"), nil, nil)}, + want: field.ErrorList{field.TooMany(classesPath, 2, 1).WithOrigin("maxItems")}, }, { - name: "missing default_runtime.version", - mutate: func(s *ateapipb.WorkerStatus) { - s.DefaultRuntime = &ateapipb.SandboxRuntime{SandboxClass: "gvisor"} - }, - want: field.ErrorList{field.Required(statusPath.Child("default_runtime", "version"), "")}, + name: "missing sandbox_classes.sandbox_class", + mutate: func(s *ateapipb.WorkerStatus) { s.SandboxClasses[0].SandboxClass = "" }, + want: field.ErrorList{field.Required(classPath.Child("sandbox_class"), "")}, + }, { + name: "unsupported sandbox_classes.sandbox_class", + mutate: func(s *ateapipb.WorkerStatus) { s.SandboxClasses[0].SandboxClass = "kata" }, + want: field.ErrorList{field.NotSupported[string](classPath.Child("sandbox_class"), nil, nil)}, + }, { + name: "missing sandbox_classes.default_runtime", + mutate: func(s *ateapipb.WorkerStatus) { s.SandboxClasses[0].DefaultRuntime = nil }, + want: field.ErrorList{field.Required(classPath.Child("default_runtime"), "")}, + }, { + name: "missing sandbox_classes.default_runtime.version", + mutate: func(s *ateapipb.WorkerStatus) { s.SandboxClasses[0].DefaultRuntime.Version = nil }, + want: field.ErrorList{field.Required(classPath.Child("default_runtime", "version"), "")}, }, { name: "too many default_runtime attributes", mutate: func(s *ateapipb.WorkerStatus) { - s.DefaultRuntime = validSandboxRuntime() - s.DefaultRuntime.Version.Attributes = attributeEntries(33) + s.SandboxClasses[0].DefaultRuntime.Version.Attributes = attributeEntries(33) }, - want: field.ErrorList{field.TooMany(statusPath.Child("default_runtime", "version", "attributes"), 33, 32).WithOrigin("maxItems")}, + want: field.ErrorList{field.TooMany(classPath.Child("default_runtime", "version", "attributes"), 33, 32).WithOrigin("maxItems")}, }, { name: "invalid restorable_runtimes entry", mutate: func(s *ateapipb.WorkerStatus) { - s.RestorableRuntimes = []*ateapipb.SandboxRuntime{{SandboxClass: "gvisor"}} + s.SandboxClasses[0].RestorableRuntimes = []*ateapipb.WorkerSandboxRuntime{{Version: &ateapipb.VersionedSandboxCompat{Attributes: attributeEntries(1)}}} }, - want: field.ErrorList{field.Required(statusPath.Child("restorable_runtimes").Index(0).Child("version"), "")}, + want: field.ErrorList{field.Required(classPath.Child("restorable_runtimes").Index(0).Child("version", "schema_version"), "")}, }, { name: "too many restorable_runtimes", mutate: func(s *ateapipb.WorkerStatus) { - s.RestorableRuntimes = sandboxRuntimes(33) + s.SandboxClasses[0].RestorableRuntimes = sandboxRuntimes(33) }, - want: field.ErrorList{field.TooMany(statusPath.Child("restorable_runtimes"), 33, 32).WithOrigin("maxItems")}, + want: field.ErrorList{field.TooMany(classPath.Child("restorable_runtimes"), 33, 32).WithOrigin("maxItems")}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -802,7 +817,6 @@ func validWorker(name string, mods ...func(*ateapipb.Worker)) *ateapipb.Worker { WorkerPodUid: name, NodeName: "node-1", Ips: []string{"10.1.2.3"}, - SandboxClass: "gvisor", } for _, m := range mods { m(w) @@ -833,15 +847,20 @@ func workerRef(name string) *ateapipb.ObjectRef { return &ateapipb.ObjectRef{Name: name} } -// validSandboxRuntime returns a SandboxRuntime in the shape RegisterWorker -// accepts: a class and a v1 identity with one attribute. -func validSandboxRuntime() *ateapipb.SandboxRuntime { - return &ateapipb.SandboxRuntime{ - SandboxClass: "gvisor", - Version: &ateapipb.VersionedSandboxCompat{ - SchemaVersion: "v1", - Attributes: []*ateapipb.AttributeEntry{{Key: "architecture", Value: "amd64"}}, - }, +// validSandboxClass returns a WorkerSandboxClass in the shape RegisterWorker +// accepts: a sandbox class and a default runtime, with nothing restorable. +func validSandboxClass() *ateapipb.WorkerSandboxClass { + return &ateapipb.WorkerSandboxClass{ + SandboxClass: "gvisor", + DefaultRuntime: &ateapipb.WorkerSandboxRuntime{Version: validSandboxCompat()}, + } +} + +// validSandboxCompat returns a v1 compatibility identity with one attribute. +func validSandboxCompat() *ateapipb.VersionedSandboxCompat { + return &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{{Key: "architecture", Value: "amd64"}}, } } @@ -855,10 +874,10 @@ func attributeEntries(n int) []*ateapipb.AttributeEntry { } // sandboxRuntimes returns n valid runtimes. -func sandboxRuntimes(n int) []*ateapipb.SandboxRuntime { - rts := make([]*ateapipb.SandboxRuntime, n) - for i := range rts { - rts[i] = validSandboxRuntime() +func sandboxRuntimes(n int) []*ateapipb.WorkerSandboxRuntime { + runtimes := make([]*ateapipb.WorkerSandboxRuntime, n) + for i := range runtimes { + runtimes[i] = &ateapipb.WorkerSandboxRuntime{Version: validSandboxCompat()} } - return rts + return runtimes } diff --git a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go index e10bfb8a0f..3bd48c9c84 100644 --- a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go +++ b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go @@ -631,6 +631,82 @@ func Validate_ActorMetadataItem( return errs } +// Validate_ActorSandboxRuntime validates an instance of ActorSandboxRuntime according +// to declarative validation rules in the API schema. +func Validate_ActorSandboxRuntime( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.ActorSandboxRuntime) (errs field.ErrorList) { + + { // field ateapipb.ActorSandboxRuntime.SandboxClass + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // custom validation + if e := ValidateCustom_ActorSandboxRuntime_SandboxClass(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 63); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.ActorSandboxRuntime) *string { + return &oldObj.SandboxClass + }) + errs = append(errs, fn(fldPath.Child("sandbox_class"), &obj.SandboxClass, oldVal, oldObj != nil)...) + } + + { // field ateapipb.ActorSandboxRuntime.Version + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.VersionedSandboxCompat, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_VersionedSandboxCompat(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.ActorSandboxRuntime) *ateapipb.VersionedSandboxCompat { + return oldObj.Version + }) + errs = append(errs, fn(fldPath.Child("version"), obj.Version, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_ActorSnapshotState validates an instance of ActorSnapshotState according // to declarative validation rules in the API schema. func Validate_ActorSnapshotState( @@ -1256,6 +1332,43 @@ func Validate_ActorTemplate( errs = append(errs, fn(fldPath.Child("resources"), obj.Resources, oldVal, oldObj != nil)...) } + { // field ateapipb.ActorTemplate.GoldenEgressPolicy + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.EgressPolicyTemplate, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.IfOption(ctx, op, fldPath, obj, oldObj, "GoldenEgressPolicy", false, validate.ForbiddenPointer).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.IfOption(ctx, op, fldPath, obj, oldObj, "GoldenEgressPolicy", false, validate.OptionalPointer).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_EgressPolicyTemplate(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.ActorTemplate) *ateapipb.EgressPolicyTemplate { + return oldObj.GoldenEgressPolicy + }) + errs = append(errs, fn(fldPath.Child("golden_egress_policy"), obj.GoldenEgressPolicy, oldVal, oldObj != nil)...) + } + { // field ateapipb.ActorTemplate.Status fn := func( fldPath *field.Path, @@ -3477,6 +3590,60 @@ func Validate_EgressPolicy( return errs } +// Validate_EgressPolicyTemplate validates an instance of EgressPolicyTemplate according +// to declarative validation rules in the API schema. +func Validate_EgressPolicyTemplate( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.EgressPolicyTemplate) (errs field.ErrorList) { + + { // field ateapipb.EgressPolicyTemplate.Rules + fn := func( + fldPath *field.Path, + obj, oldObj []*ateapipb.EgressRule, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateapipb.EgressRule](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 256).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.RequiredSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // custom validation + if e := ValidateCustom_EgressPolicyTemplate_Rules(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, nil, nil, Validate_EgressRule); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.EgressPolicyTemplate) []*ateapipb.EgressRule { + return oldObj.Rules + }) + errs = append(errs, fn(fldPath.Child("rules"), obj.Rules, oldVal, oldObj != nil)...) + } + + return errs +} + var unionMembershipFor_go_ate_dev_substrate_pkg_proto_ateapipb_EgressRule_ = validate.NewUnionMembership(validate.NewUnionMember("http"), validate.NewUnionMember("https"), validate.NewUnionMember("tls_passthrough")) // Validate_EgressRule validates an instance of EgressRule according @@ -6486,10 +6653,10 @@ func Validate_RegisterWorkerRequest( errs = append(errs, fn(fldPath.Child("capacity"), obj.Capacity, oldVal, oldObj != nil)...) } - { // field ateapipb.RegisterWorkerRequest.DefaultRuntime + { // field ateapipb.RegisterWorkerRequest.SandboxClasses fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.SandboxRuntime, + obj, oldObj []*ateapipb.WorkerSandboxClass, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -6499,62 +6666,45 @@ func Validate_RegisterWorkerRequest( } // call field-attached validations earlyReturn := false - if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.PtrSliceNoNils[ateapipb.WorkerSandboxClass](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } - if earlyReturn { - return // do not proceed - } - // call the type's validation function - errs = append(errs, Validate_SandboxRuntime(ctx, op, fldPath, obj, oldObj)...) - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.RegisterWorkerRequest) *ateapipb.SandboxRuntime { - return oldObj.DefaultRuntime - }) - errs = append(errs, fn(fldPath.Child("default_runtime"), obj.DefaultRuntime, oldVal, oldObj != nil)...) - } - - { // field ateapipb.RegisterWorkerRequest.RestorableRuntimes - fn := func( - fldPath *field.Path, - obj, oldObj []*ateapipb.SandboxRuntime, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.PtrSliceNoNils[ateapipb.SandboxRuntime](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 1).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } - if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + if e := validate.RequiredSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } - if e := validate.OptionalSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } if earlyReturn { return // do not proceed } + if e := validate.MinItems(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + // lists with map semantics require unique keys + if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, + func(a *ateapipb.WorkerSandboxClass, b *ateapipb.WorkerSandboxClass) bool { + return a.SandboxClass == b.SandboxClass + }); len(e) != 0 { + errs = append(errs, e...) + } // iterate the list and call the type's validation function - if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, nil, nil, Validate_SandboxRuntime); len(e) != 0 { + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, + func(a *ateapipb.WorkerSandboxClass, b *ateapipb.WorkerSandboxClass) bool { + return a.SandboxClass == b.SandboxClass + }, ateDeepEqual, Validate_WorkerSandboxClass); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.RegisterWorkerRequest) []*ateapipb.SandboxRuntime { - return oldObj.RestorableRuntimes + func(oldObj *ateapipb.RegisterWorkerRequest) []*ateapipb.WorkerSandboxClass { + return oldObj.SandboxClasses }) - errs = append(errs, fn(fldPath.Child("restorable_runtimes"), obj.RestorableRuntimes, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("sandbox_classes"), obj.SandboxClasses, oldVal, oldObj != nil)...) } return errs @@ -7204,6 +7354,38 @@ func Validate_ResourceMetadata( errs = append(errs, fn(fldPath.Child("update_time"), obj.UpdateTime, oldVal, oldObj != nil)...) } + { // field ateapipb.ResourceMetadata.DeleteTime + fn := func( + fldPath *field.Path, + obj, oldObj *timestamppb.Timestamp, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if e := validate.UpdatePointer(ctx, op, fldPath, obj, oldObj, validate.NoUnset, validate.NoModify).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.ResourceMetadata) *timestamppb.Timestamp { + return oldObj.DeleteTime + }) + errs = append(errs, fn(fldPath.Child("delete_time"), obj.DeleteTime, oldVal, oldObj != nil)...) + } + return errs } @@ -7430,82 +7612,6 @@ func Validate_RevertActorRequest( return errs } -// Validate_SandboxRuntime validates an instance of SandboxRuntime according -// to declarative validation rules in the API schema. -func Validate_SandboxRuntime( - ctx context.Context, op operation.Operation, fldPath *field.Path, - obj, oldObj *ateapipb.SandboxRuntime) (errs field.ErrorList) { - - { // field ateapipb.SandboxRuntime.SandboxClass - fn := func( - fldPath *field.Path, - obj, oldObj *string, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - // custom validation - if e := ValidateCustom_SandboxRuntime_SandboxClass(ctx, op, fldPath, obj, oldObj); len(e) != 0 { - errs = append(errs, e...) - } - if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 63); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.SandboxRuntime) *string { - return &oldObj.SandboxClass - }) - errs = append(errs, fn(fldPath.Child("sandbox_class"), &obj.SandboxClass, oldVal, oldObj != nil)...) - } - - { // field ateapipb.SandboxRuntime.Version - fn := func( - fldPath *field.Path, - obj, oldObj *ateapipb.VersionedSandboxCompat, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - // call the type's validation function - errs = append(errs, Validate_VersionedSandboxCompat(ctx, op, fldPath, obj, oldObj)...) - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.SandboxRuntime) *ateapipb.VersionedSandboxCompat { - return oldObj.Version - }) - errs = append(errs, fn(fldPath.Child("version"), obj.Version, oldVal, oldObj != nil)...) - } - - return errs -} - // Validate_SecurityContext validates an instance of SecurityContext according // to declarative validation rules in the API schema. func Validate_SecurityContext( @@ -7842,7 +7948,7 @@ func Validate_Snapshot( { // field ateapipb.Snapshot.SandboxRuntime fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.SandboxRuntime, + obj, oldObj *ateapipb.ActorSandboxRuntime, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -7859,11 +7965,11 @@ func Validate_Snapshot( return // do not proceed } // call the type's validation function - errs = append(errs, Validate_SandboxRuntime(ctx, op, fldPath, obj, oldObj)...) + errs = append(errs, Validate_ActorSandboxRuntime(ctx, op, fldPath, obj, oldObj)...) return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.Snapshot) *ateapipb.SandboxRuntime { + func(oldObj *ateapipb.Snapshot) *ateapipb.ActorSandboxRuntime { return oldObj.SandboxRuntime }) errs = append(errs, fn(fldPath.Child("sandbox_runtime"), obj.SandboxRuntime, oldVal, oldObj != nil)...) @@ -10096,45 +10202,6 @@ func Validate_Worker( errs = append(errs, fn(fldPath.Child("worker_spiffe_id"), &obj.WorkerSpiffeId, oldVal, oldObj != nil)...) } - { // field ateapipb.Worker.SandboxClass - fn := func( - fldPath *field.Path, - obj, oldObj *string, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.Immutable(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - // custom validation - if e := ValidateCustom_Worker_SandboxClass(ctx, op, fldPath, obj, oldObj); len(e) != 0 { - errs = append(errs, e...) - } - if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 63); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.Worker) *string { - return &oldObj.SandboxClass - }) - errs = append(errs, fn(fldPath.Child("sandbox_class"), &obj.SandboxClass, oldVal, oldObj != nil)...) - } - { // field ateapipb.Worker.Labels fn := func( fldPath *field.Path, @@ -10533,7 +10600,7 @@ func Validate_WorkerAssignment( { // field ateapipb.WorkerAssignment.SandboxRuntime fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.SandboxRuntime, + obj, oldObj *ateapipb.ActorSandboxRuntime, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -10550,11 +10617,11 @@ func Validate_WorkerAssignment( return // do not proceed } // call the type's validation function - errs = append(errs, Validate_SandboxRuntime(ctx, op, fldPath, obj, oldObj)...) + errs = append(errs, Validate_ActorSandboxRuntime(ctx, op, fldPath, obj, oldObj)...) return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.WorkerAssignment) *ateapipb.SandboxRuntime { + func(oldObj *ateapipb.WorkerAssignment) *ateapipb.ActorSandboxRuntime { return oldObj.SandboxRuntime }) errs = append(errs, fn(fldPath.Child("sandbox_runtime"), obj.SandboxRuntime, oldVal, oldObj != nil)...) @@ -10703,6 +10770,162 @@ func Validate_WorkerResources( return errs } +// Validate_WorkerSandboxClass validates an instance of WorkerSandboxClass according +// to declarative validation rules in the API schema. +func Validate_WorkerSandboxClass( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.WorkerSandboxClass) (errs field.ErrorList) { + + { // field ateapipb.WorkerSandboxClass.SandboxClass + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // custom validation + if e := ValidateCustom_WorkerSandboxClass_SandboxClass(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 63); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.WorkerSandboxClass) *string { + return &oldObj.SandboxClass + }) + errs = append(errs, fn(fldPath.Child("sandbox_class"), &obj.SandboxClass, oldVal, oldObj != nil)...) + } + + { // field ateapipb.WorkerSandboxClass.DefaultRuntime + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.WorkerSandboxRuntime, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_WorkerSandboxRuntime(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.WorkerSandboxClass) *ateapipb.WorkerSandboxRuntime { + return oldObj.DefaultRuntime + }) + errs = append(errs, fn(fldPath.Child("default_runtime"), obj.DefaultRuntime, oldVal, oldObj != nil)...) + } + + { // field ateapipb.WorkerSandboxClass.RestorableRuntimes + fn := func( + fldPath *field.Path, + obj, oldObj []*ateapipb.WorkerSandboxRuntime, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateapipb.WorkerSandboxRuntime](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, nil, nil, Validate_WorkerSandboxRuntime); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.WorkerSandboxClass) []*ateapipb.WorkerSandboxRuntime { + return oldObj.RestorableRuntimes + }) + errs = append(errs, fn(fldPath.Child("restorable_runtimes"), obj.RestorableRuntimes, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_WorkerSandboxRuntime validates an instance of WorkerSandboxRuntime according +// to declarative validation rules in the API schema. +func Validate_WorkerSandboxRuntime( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.WorkerSandboxRuntime) (errs field.ErrorList) { + + { // field ateapipb.WorkerSandboxRuntime.Version + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.VersionedSandboxCompat, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_VersionedSandboxCompat(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.WorkerSandboxRuntime) *ateapipb.VersionedSandboxCompat { + return oldObj.Version + }) + errs = append(errs, fn(fldPath.Child("version"), obj.Version, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_WorkerStatus validates an instance of WorkerStatus according // to declarative validation rules in the API schema. func Validate_WorkerStatus( @@ -10835,10 +11058,10 @@ func Validate_WorkerStatus( errs = append(errs, fn(fldPath.Child("observed_epoch"), &obj.ObservedEpoch, oldVal, oldObj != nil)...) } - { // field ateapipb.WorkerStatus.DefaultRuntime + { // field ateapipb.WorkerStatus.SandboxClasses fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.SandboxRuntime, + obj, oldObj []*ateapipb.WorkerSandboxClass, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -10848,41 +11071,11 @@ func Validate_WorkerStatus( } // call field-attached validations earlyReturn := false - if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - // call the type's validation function - errs = append(errs, Validate_SandboxRuntime(ctx, op, fldPath, obj, oldObj)...) - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.WorkerStatus) *ateapipb.SandboxRuntime { - return oldObj.DefaultRuntime - }) - errs = append(errs, fn(fldPath.Child("default_runtime"), obj.DefaultRuntime, oldVal, oldObj != nil)...) - } - - { // field ateapipb.WorkerStatus.RestorableRuntimes - fn := func( - fldPath *field.Path, - obj, oldObj []*ateapipb.SandboxRuntime, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.PtrSliceNoNils[ateapipb.SandboxRuntime](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.PtrSliceNoNils[ateapipb.WorkerSandboxClass](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } - if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 1).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } @@ -10892,17 +11085,27 @@ func Validate_WorkerStatus( if earlyReturn { return // do not proceed } + // lists with map semantics require unique keys + if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, + func(a *ateapipb.WorkerSandboxClass, b *ateapipb.WorkerSandboxClass) bool { + return a.SandboxClass == b.SandboxClass + }); len(e) != 0 { + errs = append(errs, e...) + } // iterate the list and call the type's validation function - if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, nil, nil, Validate_SandboxRuntime); len(e) != 0 { + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, + func(a *ateapipb.WorkerSandboxClass, b *ateapipb.WorkerSandboxClass) bool { + return a.SandboxClass == b.SandboxClass + }, ateDeepEqual, Validate_WorkerSandboxClass); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.WorkerStatus) []*ateapipb.SandboxRuntime { - return oldObj.RestorableRuntimes + func(oldObj *ateapipb.WorkerStatus) []*ateapipb.WorkerSandboxClass { + return oldObj.SandboxClasses }) - errs = append(errs, fn(fldPath.Child("restorable_runtimes"), obj.RestorableRuntimes, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("sandbox_classes"), obj.SandboxClasses, oldVal, oldObj != nil)...) } return errs diff --git a/cmd/ateapi/internal/authz/interceptor.go b/cmd/ateapi/internal/authz/interceptor.go index ee20395535..586c16f35e 100644 --- a/cmd/ateapi/internal/authz/interceptor.go +++ b/cmd/ateapi/internal/authz/interceptor.go @@ -23,10 +23,9 @@ import ( ) // UnaryServerInterceptor returns a gRPC unary interceptor that enforces per-RPC -// permissions registered in defaultRPCPermissions using authorizer. When -// enforce is false, only rules marked alwaysEnforce are checked. When enforce -// is true, an RPC with no rule is denied. -func UnaryServerInterceptor(authorizer *Authorizer, enforce bool) grpc.UnaryServerInterceptor { +// permissions registered in defaultRPCPermissions using authorizer. An RPC +// with no rule is denied. +func UnaryServerInterceptor(authorizer *Authorizer) grpc.UnaryServerInterceptor { return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { if IsBypassed(ctx) { return handler(ctx, req) @@ -35,9 +34,6 @@ func UnaryServerInterceptor(authorizer *Authorizer, enforce bool) grpc.UnaryServ if registered && rule.authorizedInHandler != "" { return handler(ctx, req) } - if !enforce && !rule.alwaysEnforce { - return handler(ctx, req) - } if !registered { return nil, apierror.PermissionDenied("permission denied: no authorization rule for %s", info.FullMethod) } diff --git a/cmd/ateapi/internal/authz/interceptor_test.go b/cmd/ateapi/internal/authz/interceptor_test.go index 52df3035c7..adc5ab63b1 100644 --- a/cmd/ateapi/internal/authz/interceptor_test.go +++ b/cmd/ateapi/internal/authz/interceptor_test.go @@ -16,7 +16,6 @@ package authz import ( "context" - "strings" "testing" "go.ate.dev/substrate/internal/apierror" @@ -57,7 +56,7 @@ func TestUnaryServerInterceptor_QuickRejectionAndDispatch(t *testing.T) { Kind: principal.KindJWT, }) - interceptor := UnaryServerInterceptor(authorizer, true) + interceptor := UnaryServerInterceptor(authorizer) tests := []struct { name string @@ -152,7 +151,7 @@ func TestUnaryServerInterceptor_QuickRejectionAndDispatch(t *testing.T) { func TestUnaryServerInterceptor_MalformedRequestRequiresPrincipalThenFailsClosed(t *testing.T) { authorizer := setupTestAuthorizer(t) - interceptor := UnaryServerInterceptor(authorizer, true) + interceptor := UnaryServerInterceptor(authorizer) // 1. Unauthenticated caller with empty atespace name -> Unauthenticated _, err := interceptor(context.Background(), &ateapipb.GetAtespaceRequest{}, &grpc.UnaryServerInfo{ @@ -196,74 +195,8 @@ func TestUnaryServerInterceptor_MalformedRequestRequiresPrincipalThenFailsClosed } -func TestUnaryServerInterceptor_EnforcementDisabled(t *testing.T) { - authorizer := setupTestAuthorizer(t) - interceptor := UnaryServerInterceptor(authorizer, false) - bobCtx := principal.InjectContext(context.Background(), principal.PrincipalInfo{ - ID: "bob@example.com", - Kind: principal.KindJWT, - }) - aliceCtx := principal.InjectContext(context.Background(), principal.PrincipalInfo{ - ID: "alice@example.com", - Kind: principal.KindJWT, - }) - - tests := []struct { - name string - ctx context.Context - fullMethod string - req any - wantCode codes.Code - }{ - { - name: "regular RPC skips the check", - ctx: bobCtx, - fullMethod: ateapipb.Control_CreateAtespace_FullMethodName, - req: &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: "team1"}}}, - wantCode: codes.OK, - }, - { - name: "governance RPC is still denied", - ctx: bobCtx, - fullMethod: ateapipb.Control_UpdateGlobalAccessPolicy_FullMethodName, - req: &ateapipb.UpdateGlobalAccessPolicyRequest{}, - wantCode: codes.PermissionDenied, - }, - { - name: "governance RPC is still allowed for an owner", - ctx: aliceCtx, - fullMethod: ateapipb.Control_UpdateGlobalAccessPolicy_FullMethodName, - req: &ateapipb.UpdateGlobalAccessPolicyRequest{}, - wantCode: codes.OK, - }, - { - name: "governance RPC still requires a principal", - ctx: context.Background(), - fullMethod: ateapipb.Control_GetAtespaceAccessPolicy_FullMethodName, - req: &ateapipb.GetAtespaceAccessPolicyRequest{}, - wantCode: codes.Unauthenticated, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - handlerCalled := false - _, err := interceptor(tc.ctx, tc.req, &grpc.UnaryServerInfo{FullMethod: tc.fullMethod}, func(ctx context.Context, req any) (any, error) { - handlerCalled = true - return "ok", nil - }) - if apierror.Code(err) != tc.wantCode { - t.Fatalf("apierror.Code(err) = %v, want %v (err: %v)", apierror.Code(err), tc.wantCode, err) - } - if wantHandler := tc.wantCode == codes.OK; handlerCalled != wantHandler { - t.Fatalf("handlerCalled = %v, want %v", handlerCalled, wantHandler) - } - }) - } -} - -// An RPC with no rule is denied when enforcement is on and passes through when -// it is off; an RPC its handler authorizes passes through either way. Neither -// reaches OpenFGA, so the authorizer is nil. +// An RPC with no rule is denied, and an RPC its handler authorizes passes +// through. Neither reaches OpenFGA, so the authorizer is nil. func TestUnaryServerInterceptor_UnregisteredAndHandlerAuthorizedRPCs(t *testing.T) { const unregistered = "/ateapi.Control/NotARealMethod" aliceCtx := principal.InjectContext(context.Background(), principal.PrincipalInfo{ @@ -273,23 +206,20 @@ func TestUnaryServerInterceptor_UnregisteredAndHandlerAuthorizedRPCs(t *testing. tests := []struct { name string - enforce bool ctx context.Context fullMethod string wantCode codes.Code }{ - {"unregistered RPC is denied with enforcement on", true, aliceCtx, unregistered, codes.PermissionDenied}, - {"unregistered RPC is denied without a principal", true, context.Background(), unregistered, codes.PermissionDenied}, - {"unregistered RPC passes with enforcement off", false, aliceCtx, unregistered, codes.OK}, - {"RegisterWorker passes with enforcement on", true, context.Background(), ateapipb.WorkerService_RegisterWorker_FullMethodName, codes.OK}, - {"RequestActorSuspend passes with enforcement on", true, context.Background(), ateapipb.WorkerService_RequestActorSuspend_FullMethodName, codes.OK}, - {"MintWorkerActorCertificate passes with enforcement on", true, context.Background(), ateapipb.WorkerService_MintWorkerActorCertificate_FullMethodName, codes.OK}, - {"MintWorkerActorJwt passes with enforcement on", true, context.Background(), ateapipb.WorkerService_MintWorkerActorJwt_FullMethodName, codes.OK}, - {"RegisterWorker passes with enforcement off", false, context.Background(), ateapipb.WorkerService_RegisterWorker_FullMethodName, codes.OK}, + {"unregistered RPC is denied", aliceCtx, unregistered, codes.PermissionDenied}, + {"unregistered RPC is denied without a principal", context.Background(), unregistered, codes.PermissionDenied}, + {"RegisterWorker passes", context.Background(), ateapipb.WorkerService_RegisterWorker_FullMethodName, codes.OK}, + {"RequestActorSuspend passes", context.Background(), ateapipb.WorkerService_RequestActorSuspend_FullMethodName, codes.OK}, + {"MintWorkerActorCertificate passes", context.Background(), ateapipb.WorkerService_MintWorkerActorCertificate_FullMethodName, codes.OK}, + {"MintWorkerActorJwt passes", context.Background(), ateapipb.WorkerService_MintWorkerActorJwt_FullMethodName, codes.OK}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - interceptor := UnaryServerInterceptor(nil, tc.enforce) + interceptor := UnaryServerInterceptor(nil) handlerCalled := false _, err := interceptor(tc.ctx, struct{}{}, &grpc.UnaryServerInfo{FullMethod: tc.fullMethod}, func(context.Context, any) (any, error) { handlerCalled = true @@ -305,32 +235,6 @@ func TestUnaryServerInterceptor_UnregisteredAndHandlerAuthorizedRPCs(t *testing. } } -// TestAccessPolicyRPCsAlwaysEnforced guards against an AccessPolicy RPC being -// added without a governance rule, which would leave it unchecked while -// enforcement is disabled. -func TestAccessPolicyRPCsAlwaysEnforced(t *testing.T) { - desc := ateapipb.Control_ServiceDesc - found := 0 - for _, m := range desc.Methods { - if !strings.Contains(m.MethodName, "AccessPolicy") { - continue - } - found++ - fullMethod := "/" + desc.ServiceName + "/" + m.MethodName - rule, ok := defaultRPCPermissions[fullMethod] - if !ok { - t.Errorf("%s has no permission rule", fullMethod) - continue - } - if !rule.alwaysEnforce { - t.Errorf("%s is not marked alwaysEnforce", fullMethod) - } - } - if found == 0 { - t.Fatal("found no AccessPolicy RPCs in Control_ServiceDesc") - } -} - func TestUnaryServerInterceptor_ActorAndTemplateChecks(t *testing.T) { ctx := context.Background() pool := startPostgres(t) @@ -460,7 +364,7 @@ func TestUnaryServerInterceptor_ActorAndTemplateChecks(t *testing.T) { {"global viewer cannot delete template", "global-viewer", deleteTemplate, codes.PermissionDenied}, } - interceptor := UnaryServerInterceptor(authorizer, true) + interceptor := UnaryServerInterceptor(authorizer) for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { userCtx := principal.InjectContext(ctx, principal.PrincipalInfo{ID: tc.user, Kind: principal.KindJWT}) @@ -652,7 +556,7 @@ func TestUnaryServerInterceptor_LifecycleEgressWorkerAndTagChecks(t *testing.T) {"bootstrap owner cannot mint actor certificate", "bootstrap-owner", mintCert, codes.PermissionDenied}, } - interceptor := UnaryServerInterceptor(authorizer, true) + interceptor := UnaryServerInterceptor(authorizer) for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { userCtx := principal.InjectContext(ctx, principal.PrincipalInfo{ID: tc.user, Kind: principal.KindJWT}) @@ -808,7 +712,7 @@ func TestUnaryServerInterceptor_SystemPrincipals(t *testing.T) { {"other mTLS identity cannot create worker", mtls(otherID), createWorker, codes.PermissionDenied}, } - interceptor := UnaryServerInterceptor(authorizer, true) + interceptor := UnaryServerInterceptor(authorizer) for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { userCtx := principal.InjectContext(ctx, tc.principal) diff --git a/cmd/ateapi/internal/authz/registry.go b/cmd/ateapi/internal/authz/registry.go index fb25590eb9..d850d03911 100644 --- a/cmd/ateapi/internal/authz/registry.go +++ b/cmd/ateapi/internal/authz/registry.go @@ -144,10 +144,6 @@ func both(aChecks []check, aErrs field.ErrorList, bChecks []check, bErrs field.E // rpcRule is the permission rule for one RPC. type rpcRule struct { extract checkExtractor - // alwaysEnforce marks RPCs that are checked even when enforcement is - // disabled. AccessPolicy RPCs set it so that nobody can grant themselves - // access while enforcement is off and keep that grant once it is turned on. - alwaysEnforce bool // authorizedInHandler, when set, says how the RPC's handler authorizes // the caller; the interceptor passes such RPCs through unchecked. authorizedInHandler string @@ -157,10 +153,6 @@ func rule(extract checkExtractor) rpcRule { return rpcRule{extract: extract} } -func governance(extract checkExtractor) rpcRule { - return rpcRule{extract: extract, alwaysEnforce: true} -} - // inHandler registers an RPC whose handler authorizes the caller, as how // describes. func inHandler(how string) rpcRule { @@ -168,8 +160,8 @@ func inHandler(how string) rpcRule { } // defaultRPCPermissions is the declarative registry mapping gRPC full method -// names to their permission rules. With enforcement on, the interceptor denies -// any RPC missing from it. +// names to their permission rules. The interceptor denies any RPC missing +// from it. var defaultRPCPermissions = map[string]rpcRule{ // Atespaces. ateapipb.Control_CreateAtespace_FullMethodName: rule(checksOf(func(*ateapipb.CreateAtespaceRequest) ([]check, field.ErrorList) { @@ -186,25 +178,25 @@ var defaultRPCPermissions = map[string]rpcRule{ })), // Access policies. - ateapipb.Control_GetGlobalAccessPolicy_FullMethodName: governance(checksOf(func(*ateapipb.GetGlobalAccessPolicyRequest) ([]check, field.ErrorList) { + ateapipb.Control_GetGlobalAccessPolicy_FullMethodName: rule(checksOf(func(*ateapipb.GetGlobalAccessPolicyRequest) ([]check, field.ErrorList) { return onGlobal(RelationCanGetAccessPolicy) })), - ateapipb.Control_CreateGlobalAccessPolicy_FullMethodName: governance(checksOf(func(*ateapipb.CreateGlobalAccessPolicyRequest) ([]check, field.ErrorList) { + ateapipb.Control_CreateGlobalAccessPolicy_FullMethodName: rule(checksOf(func(*ateapipb.CreateGlobalAccessPolicyRequest) ([]check, field.ErrorList) { return onGlobal(RelationCanCreateAccessPolicy) })), - ateapipb.Control_UpdateGlobalAccessPolicy_FullMethodName: governance(checksOf(func(*ateapipb.UpdateGlobalAccessPolicyRequest) ([]check, field.ErrorList) { + ateapipb.Control_UpdateGlobalAccessPolicy_FullMethodName: rule(checksOf(func(*ateapipb.UpdateGlobalAccessPolicyRequest) ([]check, field.ErrorList) { return onGlobal(RelationCanUpdateAccessPolicy) })), - ateapipb.Control_GetAtespaceAccessPolicy_FullMethodName: governance(checksOf(func(r *ateapipb.GetAtespaceAccessPolicyRequest) ([]check, field.ErrorList) { + ateapipb.Control_GetAtespaceAccessPolicy_FullMethodName: rule(checksOf(func(r *ateapipb.GetAtespaceAccessPolicyRequest) ([]check, field.ErrorList) { return onAtespace(RelationCanGetAccessPolicy, r.GetAtespace().GetName(), field.NewPath("atespace", "name")) })), - ateapipb.Control_CreateAtespaceAccessPolicy_FullMethodName: governance(checksOf(func(r *ateapipb.CreateAtespaceAccessPolicyRequest) ([]check, field.ErrorList) { + ateapipb.Control_CreateAtespaceAccessPolicy_FullMethodName: rule(checksOf(func(r *ateapipb.CreateAtespaceAccessPolicyRequest) ([]check, field.ErrorList) { return onAtespace(RelationCanCreateAccessPolicy, r.GetAtespace().GetName(), field.NewPath("atespace", "name")) })), - ateapipb.Control_UpdateAtespaceAccessPolicy_FullMethodName: governance(checksOf(func(r *ateapipb.UpdateAtespaceAccessPolicyRequest) ([]check, field.ErrorList) { + ateapipb.Control_UpdateAtespaceAccessPolicy_FullMethodName: rule(checksOf(func(r *ateapipb.UpdateAtespaceAccessPolicyRequest) ([]check, field.ErrorList) { return onAtespace(RelationCanUpdateAccessPolicy, r.GetAtespace().GetName(), field.NewPath("atespace", "name")) })), - ateapipb.Control_DeleteAtespaceAccessPolicy_FullMethodName: governance(checksOf(func(r *ateapipb.DeleteAtespaceAccessPolicyRequest) ([]check, field.ErrorList) { + ateapipb.Control_DeleteAtespaceAccessPolicy_FullMethodName: rule(checksOf(func(r *ateapipb.DeleteAtespaceAccessPolicyRequest) ([]check, field.ErrorList) { return onAtespace(RelationCanDeleteAccessPolicy, r.GetAtespace().GetName(), field.NewPath("atespace", "name")) })), diff --git a/cmd/ateapi/internal/authz/registry_test.go b/cmd/ateapi/internal/authz/registry_test.go index 05d4e26386..b6821ca7d3 100644 --- a/cmd/ateapi/internal/authz/registry_test.go +++ b/cmd/ateapi/internal/authz/registry_test.go @@ -709,9 +709,6 @@ func TestDefaultRPCPermissions_CoversEveryServedRPC(t *testing.T) { if (r.extract == nil) == (r.authorizedInHandler == "") { t.Errorf("%s: want exactly one of a check extractor or authorizedInHandler", fullMethod) } - if r.authorizedInHandler != "" && r.alwaysEnforce { - t.Errorf("%s: authorizedInHandler and alwaysEnforce are exclusive", fullMethod) - } } for fullMethod := range defaultRPCPermissions { if !served[fullMethod] { diff --git a/cmd/ateapi/internal/controlapi/access_policy_test.go b/cmd/ateapi/internal/controlapi/access_policy_test.go index f3e39d81aa..81c29864d3 100644 --- a/cmd/ateapi/internal/controlapi/access_policy_test.go +++ b/cmd/ateapi/internal/controlapi/access_policy_test.go @@ -49,7 +49,7 @@ func TestAccessPolicy_GlobalAndAtespaceGovernance(t *testing.T) { persistence.SetPolicyManager(policyManager) svc := NewRPCService(persistence, nil, nil, nil, nil, nil, "", 0, nil, nil, "", nil, nil, "") - interceptor := authz.UnaryServerInterceptor(authorizer, true) + interceptor := authz.UnaryServerInterceptor(authorizer) userCtx := func(id string) context.Context { return principal.InjectContext(ctx, principal.PrincipalInfo{ @@ -200,7 +200,7 @@ func TestAccessPolicy_GlobalAndAtespaceGovernance(t *testing.T) { if err != nil { t.Fatalf("authz.New after restart failed: %v", err) } - restartedInterceptor := authz.UnaryServerInterceptor(restarted, true) + restartedInterceptor := authz.UnaryServerInterceptor(restarted) getGlobalAfterRestart := func(c context.Context) error { _, err := restartedInterceptor(c, &ateapipb.GetGlobalAccessPolicyRequest{Name: resources.SingletonName}, &grpc.UnaryServerInfo{FullMethod: ateapipb.Control_GetGlobalAccessPolicy_FullMethodName}, func(c context.Context, r any) (any, error) { return svc.GetGlobalAccessPolicy(c, r.(*ateapipb.GetGlobalAccessPolicyRequest)) diff --git a/cmd/ateapi/internal/controlapi/actor.go b/cmd/ateapi/internal/controlapi/actor.go index 35e1fef471..d212e93666 100644 --- a/cmd/ateapi/internal/controlapi/actor.go +++ b/cmd/ateapi/internal/controlapi/actor.go @@ -149,7 +149,7 @@ func (s *ServiceImpl) CreateActor(ctx context.Context, inActor *ateapipb.Actor) outActor.Status.LastAssignedGeneration = 1 } if errs := apivalidation.ValidateActorUpdate(ctx, field.NewPath("actor"), outActor, inActor, true); len(errs) > 0 { - return nil, toGRPCInternalError(errs) + return nil, resources.ToInternalError(errs) } // Save the data in the storage layer. @@ -191,6 +191,8 @@ func (s *ServiceImpl) resolveTagSource(ctx context.Context, actorAtespace string default: return nil, apierror.FailedPrecondition("source Tag has an invalid scope") } + // TODO: Refuse a tag with delete_time set. DeleteTag may already have + // collected its snapshot. // A tag might have an empty Snapshot URI if the tag creation failed or is ongoing. if tagDurableSnapshotURI(tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) == "" { return nil, apierror.FailedPrecondition("source Tag is still being created or failed creation") @@ -328,7 +330,7 @@ func (s *ServiceImpl) UpdateActor(ctx context.Context, actorRef resources.ActorR // Validate the final value before storing it. if errs := apivalidation.ValidateActorUpdate(ctx, field.NewPath("actor"), newVal, oldVal, true); len(errs) > 0 { - return toGRPCInternalError(errs) + return resources.ToInternalError(errs) } return nil @@ -454,6 +456,10 @@ func (s *RPCService) DeleteActor(ctx context.Context, req *ateapipb.DeleteActorR return deleted, nil } +func (s *ServiceImpl) MarkActorForDeletion(ctx context.Context, actorRef resources.ActorRef, precondition store.DeletePreconditions, mutate func(*ateapipb.Actor) error) (*ateapipb.Actor, error) { + return s.store.MarkActorForDeletion(ctx, actorRef, precondition, mutate) +} + func (s *ServiceImpl) DeleteActor(ctx context.Context, actorRef resources.ActorRef, precondition store.DeletePreconditions) (*ateapipb.Actor, error) { return s.store.DeleteActor(ctx, actorRef, precondition) } diff --git a/cmd/ateapi/internal/controlapi/actor_template.go b/cmd/ateapi/internal/controlapi/actor_template.go index 45df2feb5a..cf2c77fab1 100644 --- a/cmd/ateapi/internal/controlapi/actor_template.go +++ b/cmd/ateapi/internal/controlapi/actor_template.go @@ -67,7 +67,7 @@ func (s *ServiceImpl) CreateActorTemplate(ctx context.Context, inTemplate *ateap // Validate the final value before storing it. if errs := apivalidation.ValidateActorTemplateUpdate(ctx, field.NewPath("actor_template"), outTemplate, inTemplate); len(errs) > 0 { - return nil, toGRPCInternalError(errs) + return nil, resources.ToInternalError(errs) } // Reject a template whose actors could never be snapshotted, rather than @@ -127,6 +127,10 @@ func (s *RPCService) DeleteActorTemplate(ctx context.Context, req *ateapipb.Dele return s.actorWorkflow.DeleteActorTemplate(ctx, resources.ActorTemplateRefFromObjectRef(req.GetActorTemplate()), toDeletePreconditions(req.GetOptions())) } +func (s *ServiceImpl) MarkActorTemplateForDeletion(ctx context.Context, templateRef resources.ActorTemplateRef, precondition store.DeletePreconditions, mutate func(*ateapipb.ActorTemplate) error) (*ateapipb.ActorTemplate, error) { + return s.store.MarkActorTemplateForDeletion(ctx, templateRef, precondition, mutate) +} + func (s *ServiceImpl) DeleteActorTemplate(ctx context.Context, templateRef resources.ActorTemplateRef, precondition store.DeletePreconditions) (*ateapipb.ActorTemplate, error) { // TODO: implement this return s.store.DeleteActorTemplate(ctx, templateRef, precondition) diff --git a/cmd/ateapi/internal/controlapi/actor_template_test.go b/cmd/ateapi/internal/controlapi/actor_template_test.go index bef9a961c0..36964f2026 100644 --- a/cmd/ateapi/internal/controlapi/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/actor_template_test.go @@ -313,6 +313,9 @@ func TestDeleteActorTemplate(t *testing.T) { if _, err := persistence.GetTag(ctx, tagRef); err != nil { t.Fatalf("golden tag after the refused delete: %v", err) } + if current, err := persistence.GetActorTemplate(ctx, templateRef); err != nil || current.GetMetadata().GetDeleteTime() != nil { + t.Fatalf("template after the refused delete = %v, %v; want it present and not marked", current, err) + } } req := &ateapipb.DeleteActorTemplateRequest{ActorTemplate: templateRef.ToObjectRef()} deleted, err := svc.DeleteActorTemplate(ctx, req) @@ -320,8 +323,12 @@ func TestDeleteActorTemplate(t *testing.T) { if !isObjectStoreErr(err) { t.Fatalf("DeleteActorTemplate = %v, want object storage error", err) } - if _, err := persistence.GetActorTemplate(ctx, templateRef); err != nil { - t.Fatalf("template lost after cleanup failure: %v", err) + admitted, getErr := persistence.GetActorTemplate(ctx, templateRef) + if getErr != nil { + t.Fatalf("template lost after cleanup failure: %v", getErr) + } + if admitted.GetMetadata().GetDeleteTime() == nil { + t.Fatal("a failed delete left delete_time unset") } if _, err := persistence.GetTag(ctx, tagRef); err != nil { t.Fatalf("tag lost after cleanup failure: %v", err) @@ -330,15 +337,28 @@ func TestDeleteActorTemplate(t *testing.T) { if tt.wantActorAfterFailure && actorErr != nil || !tt.wantActorAfterFailure && !errors.Is(actorErr, store.ErrNotFound) { t.Fatalf("GetActor after failure = %v, want present %v", actorErr, tt.wantActorAfterFailure) } + if _, err := svc.DeleteActorTemplate(ctx, req); !isObjectStoreErr(err) { + t.Fatalf("retried DeleteActorTemplate = %v, want object storage error", err) + } + retried, retryErr := persistence.GetActorTemplate(ctx, templateRef) + if retryErr != nil { + t.Fatalf("template lost after a retried cleanup failure: %v", retryErr) + } + if got, want := retried.GetMetadata().GetVersion(), admitted.GetMetadata().GetVersion(); got != want { + t.Errorf("a retried delete moved the version to %d, want %d", got, want) + } objects.OnDelete = nil deleted, err = svc.DeleteActorTemplate(ctx, req) } if err != nil { t.Fatal(err) } - if diff := cmp.Diff(tmpl, deleted, protocmp.Transform()); diff != "" { + if diff := cmp.Diff(tmpl, deleted, protocmp.Transform(), ignoreVersion, ignoreTimestamps); diff != "" { t.Fatalf("deleted template mismatch (-want +got):\n%s", diff) } + if deleted.GetMetadata().GetDeleteTime() == nil { + t.Fatal("the deleted template carries no delete_time") + } if _, err := persistence.GetActorTemplate(ctx, templateRef); !errors.Is(err, store.ErrNotFound) { t.Fatalf("GetActorTemplate after delete = %v, want NotFound", err) } diff --git a/cmd/ateapi/internal/controlapi/actor_test.go b/cmd/ateapi/internal/controlapi/actor_test.go index 3faf3001bf..ac2b3726ca 100644 --- a/cmd/ateapi/internal/controlapi/actor_test.go +++ b/cmd/ateapi/internal/controlapi/actor_test.go @@ -21,6 +21,7 @@ import ( "crypto/rand" "crypto/tls" "crypto/x509" + "errors" "fmt" "strings" "testing" @@ -40,6 +41,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/peer" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/known/timestamppb" "k8s.io/apimachinery/pkg/util/validation/field" @@ -361,8 +363,8 @@ func TestUpdateActor_RepointTemplate(t *testing.T) { t.Fatalf("UpdateActor repointing a running actor = %v, want FailedPrecondition (err: %v)", got, err) } - // Repointing a paused actor is allowed while suspended, even when it holds - // a node-local snapshot. + // Repointing a suspended actor is allowed even when it holds a node-local + // snapshot. for _, name := range []string{"paused-actor", "uploaded-paused-actor"} { paused := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, @@ -468,10 +470,10 @@ func TestUpdateActor_RepointTemplateStorageLocation(t *testing.T) { name string template string snapshot func(*testing.T, *ateapipb.Actor) string - // pending leaves the snapshot's upload PENDING, as a pause does. + // pending leaves the snapshot's upload PENDING, as an async suspend does. pending bool // lastAssignedGen overrides status.LastAssignedGeneration when > 0 - // (e.g. after a reverted pause or crashed suspend pruned a higher + // (e.g. after a reverted or crashed suspend pruned a higher // pending generation). lastAssignedGen int32 repointTo string @@ -757,11 +759,8 @@ func TestUpdateActor_DeleteRecreateRace(t *testing.T) { racing := &conflictInjectingStore{ Interface: persistence, inject: func() { - if _, err := persistence.UpdateActor(ctx, actorRef, store.PreconditionFrom(original), func(toUpdate *ateapipb.Actor) error { - toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_DELETING - return nil - }); err != nil { - t.Fatalf("racing writer: mark deleting: %v", err) + if _, err := persistence.MarkActorForDeletion(ctx, actorRef, store.DeletePreconditions{}, nil); err != nil { + t.Fatalf("racing writer: mark for deletion: %v", err) } if _, err := persistence.DeleteActor(ctx, actorRef, store.DeletePreconditions{}); err != nil { t.Fatalf("racing writer: DeleteActor: %v", err) @@ -896,6 +895,69 @@ func rpcServiceWithActor(t *testing.T, actor *ateapipb.Actor) (*RPCService, *ate return &RPCService{impl: newServiceImpl(persistence, nil)}, created } +func TestCreateActor_DoesNotInheritGoldenEgressPolicy(t *testing.T) { + preview.InitForTest(t, "GoldenEgressPolicy") + for _, tt := range []struct { + name string + policy *ateapipb.EgressPolicyTemplate + }{ + {name: "absent"}, + {name: "configured", policy: &ateapipb.EgressPolicyTemplate{Rules: []*ateapipb.EgressRule{ + {Http: &ateapipb.HttpRule{Hostnames: []string{"api.example.com"}}}, + {Https: &ateapipb.HttpsRule{Hostnames: []string{"api.example.com"}}}, + }}}, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := t.Context() + persistence := newTestPersistence(t) + impl := newServiceImpl(persistence, nil) + impl.clusterSnapshotLocation = "gs://cluster-bucket/" + svc := &RPCService{impl: impl} + storetest.MustCreateAtespace(t, ctx, persistence, "templates") + storetest.MustCreateAtespace(t, ctx, persistence, "actors") + tmpl, err := svc.CreateActorTemplate(ctx, &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { + tmpl.Metadata = &ateapipb.ResourceMetadata{Atespace: "templates", Name: "tmpl"} + tmpl.GoldenEgressPolicy = tt.policy + })}) + if err != nil { + t.Fatal(err) + } + tmplRef := resources.ActorTemplateRefFromActorTemplate(tmpl).ToObjectRef() + // Existing templates remain usable without the preview gate. + preview.InitForTest(t) + for _, target := range []resources.ActorRef{ + {Atespace: "actors", Name: "alpha"}, + {Atespace: "actors", Name: tmpl.GetMetadata().GetUid()}, + {Atespace: tmpl.GetMetadata().GetAtespace(), Name: tmpl.GetMetadata().GetUid()}, + } { + actor, err := svc.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: target.Atespace, Name: target.Name}, ActorTemplate: tmplRef, + }}) + if err != nil { + t.Fatal(err) + } + ref := resources.ActorRefFromActor(actor).ToObjectRef() + if _, err := svc.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: ref, Name: resources.SingletonName}); apierror.Code(err) != codes.NotFound { + t.Fatalf("actor %v inherited golden policy: %v, want NotFound", target, err) + } + if _, err := svc.CreateActorEgressPolicy(ctx, &ateapipb.CreateActorEgressPolicyRequest{ + Actor: ref, + EgressPolicy: &ateapipb.EgressPolicy{ + Metadata: &ateapipb.ResourceMetadata{Atespace: target.Atespace, Name: resources.SingletonName}, + Rules: []*ateapipb.EgressRule{{Https: &ateapipb.HttpsRule{Hostnames: []string{"runtime.example.com"}}}}, + }, + }); err != nil { + t.Fatalf("creating the actor's runtime policy: %v", err) + } + } + gotTemplate, err := svc.GetActorTemplate(ctx, &ateapipb.GetActorTemplateRequest{ActorTemplate: tmplRef}) + if err != nil || !proto.Equal(gotTemplate, tmpl) { + t.Fatalf("creating runtime policies changed the template: %v, %v", gotTemplate, err) + } + }) + } +} + func TestCreateActor_GoldenTagDefault(t *testing.T) { for _, scenario := range []string{"default", "explicit tag", "own snapshot", "missing", "pending", "wrong template", "data scope"} { t.Run(scenario, func(t *testing.T) { @@ -933,6 +995,7 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { } if _, err := persistence.UpdateActorTemplate(ctx, resources.ActorTemplateRefFromActorTemplate(tmpl), store.PreconditionFrom(tmpl), func(db *ateapipb.ActorTemplate) error { db.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{GoldenTag: ref}} + db.GoldenEgressPolicy = &ateapipb.EgressPolicyTemplate{Rules: []*ateapipb.EgressRule{{Https: &ateapipb.HttpsRule{Hostnames: []string{"init.example.com"}, Ports: &ateapipb.Ports{Numbers: []int32{443}}}}}} return nil }); err != nil { t.Fatal(err) @@ -958,6 +1021,9 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { if gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() != tagDurableSnapshotURI(tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) || gotSnap.GetActorTemplateUid() != tmpl.GetMetadata().GetUid() { t.Fatalf("incorrect initial status: %v", created.GetStatus()) } + if _, err := persistence.GetEgressPolicy(ctx, resources.ActorRefFromActor(created)); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("actor restored from a tag inherited golden policy: %v, want NotFound", err) + } if scenario == "own snapshot" { uri, err := resources.NewActorSnapshotURI(tmpl.GetSnapshotConfig().GetStorageLocation(), "team-a", created.GetMetadata().GetUid(), "snapshot") if err != nil { @@ -974,11 +1040,12 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { } } workflow := &ActorWorkflow{store: persistence} - _, _, src, err := workflow.loadActorForResume(ctx, resources.ActorRefFromActor(created)) + actorRef := resources.ActorRefFromActor(created) + loaded, _, err := workflow.loadActorForResume(ctx, actorRef) if err != nil { t.Fatal(err) } - if src.SnapshotURI.IsZero() { + if snapshotToResume(loaded.GetStatus()) == nil { t.Fatalf("missing snapshot source for %s", scenario) } }) diff --git a/cmd/ateapi/internal/controlapi/atespace_test.go b/cmd/ateapi/internal/controlapi/atespace_test.go index 4564485076..6a649fda48 100644 --- a/cmd/ateapi/internal/controlapi/atespace_test.go +++ b/cmd/ateapi/internal/controlapi/atespace_test.go @@ -73,7 +73,7 @@ func TestAtespace_EndToEndOpenFGAScenarios(t *testing.T) { svc := &RPCService{ impl: newServiceImpl(persistence, nil), } - interceptor := authz.UnaryServerInterceptor(authorizer, true) + interceptor := authz.UnaryServerInterceptor(authorizer) asUser := func(id string) context.Context { return principal.InjectContext(ctx, principal.PrincipalInfo{ diff --git a/cmd/ateapi/internal/controlapi/common_test.go b/cmd/ateapi/internal/controlapi/common_test.go index 2d5ec8bd7a..2b294108e9 100644 --- a/cmd/ateapi/internal/controlapi/common_test.go +++ b/cmd/ateapi/internal/controlapi/common_test.go @@ -32,7 +32,8 @@ const ( var ( ignoreUID = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "uid") - ignoreTimestamps = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "create_time", "update_time") + ignoreVersion = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "version") + ignoreTimestamps = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "create_time", "update_time", "delete_time") ) func assertValidateErr(t *testing.T, got field.ErrorList, want field.ErrorList) { diff --git a/cmd/ateapi/internal/controlapi/crash.go b/cmd/ateapi/internal/controlapi/crash.go index 9aeee6aea0..05b8236a2a 100644 --- a/cmd/ateapi/internal/controlapi/crash.go +++ b/cmd/ateapi/internal/controlapi/crash.go @@ -37,6 +37,7 @@ import ( const ( crashMessageWorkerAssignmentMissing = "actor has no worker assignment in a state that requires one" crashMessageLocalSnapshotNodeUnknown = "node holding the actor's local snapshot is unknown" + crashMessageLocalSnapshotNodeGone = "node holding the actor's latest snapshot no longer exists" crashMessageLocalSnapshotMissing = "actor has no local snapshot" crashMessageDurableSnapshotMissing = "actor has no durable snapshot" crashMessageSnapshotStateConflict = "atelet's snapshot state conflicts with the actor's record" @@ -127,7 +128,12 @@ func crashActor(ctx context.Context, st crashActorStore, actorRef resources.Acto if err != nil { return fmt.Errorf("while loading actor to crash: %w", err) } + return crashActorVersion(ctx, st, actorRef, actor, opName, message) +} +// crashActorVersion is crashActor for actor as already read. It fails with +// store.ErrVersionConflict if the actor changed since. +func crashActorVersion(ctx context.Context, st crashActorStore, actorRef resources.ActorRef, actor *ateapipb.Actor, opName, message string) error { wasAlreadyCrashed := actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_CRASHED opName = ateattr.NormalizeOperationName(opName) @@ -139,16 +145,16 @@ func crashActor(ctx context.Context, st crashActorStore, actorRef resources.Acto // instead leaves the actor (and its assignment) intact so the caller retries // crashActor, which re-attempts the release. releaseWorker is idempotent, so // a retry after a release that already succeeded is a no-op. - sandboxClass, _, err := releaseWorker(ctx, st, actor) - if err != nil { + if _, err := releaseWorker(ctx, st, actor); err != nil { return fmt.Errorf("while releasing worker to crash actor: %w", err) } // Snapshot crash attributes before pod and pool pointers are cleared below; // the counter itself is emitted only after the transition commits. + sandboxClass := actor.GetStatus().GetWorkerAssignment().GetSandboxRuntime().GetSandboxClass() crashAttrs := ateattr.ActorMetricAttributes(actor, sandboxClass, opName) - _, err = st.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { + if _, err := st.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_CRASHED // An actor crashed concurrently, e.g. by worker deletion, keeps its first crash. if !wasAlreadyCrashed { @@ -161,8 +167,7 @@ func crashActor(ctx context.Context, st crashActorStore, actorRef resources.Acto // failed workflow steps must never promote them to COMPLETED. toUpdate.Status.WorkerAssignment = nil return nil - }) - if err != nil { + }); err != nil { return fmt.Errorf("while marking actor crashed: %w", err) } @@ -212,37 +217,36 @@ type crashActorStore interface { // releaseWorker clears the worker's assignment if it still points at the given // actor. A missing worker or an already-cleared assignment is not an error. -// It returns the worker's sandboxClass if found, and the worker as it stands -// after the release, which callers holding a cache of workers hand to it. -func releaseWorker(ctx context.Context, st crashActorStore, actor *ateapipb.Actor) (string, *ateapipb.Worker, error) { +// It returns the worker as it stands after the release, which callers holding +// a cache of workers hand to it. +func releaseWorker(ctx context.Context, st crashActorStore, actor *ateapipb.Actor) (*ateapipb.Worker, error) { assignment := actor.GetStatus().GetWorkerAssignment() if assignment == nil { slog.WarnContext(ctx, "Actor's worker assignment is already cleared") - return "", nil, nil + return nil, nil } workerName := assignment.GetWorker().GetName() - worker, err := st.GetWorker(ctx, workerName) + _, err := st.GetWorker(ctx, workerName) if errors.Is(err, store.ErrNotFound) { // No need to release if the worker is not found. slog.WarnContext(ctx, "Worker already gone while crashing actor, skipping release", slog.String("worker", workerName)) - return "", nil, nil + return nil, nil } if err != nil { - return "", nil, fmt.Errorf("while getting worker to release: %w", err) + return nil, fmt.Errorf("while getting worker to release: %w", err) } - sandboxClass := worker.GetSandboxClass() // Release only this actor's assignment; the worker may be hosting others, // and they are unaffected by this one crashing. A worker that is no longer // hosting it has already been released. released, err := st.ReleaseActorFromWorker(ctx, workerName, actor.GetMetadata().GetUid()) if err != nil { - return sandboxClass, nil, fmt.Errorf("while releasing worker: %w", err) + return nil, fmt.Errorf("while releasing worker: %w", err) } if released == nil { slog.WarnContext(ctx, "Worker is not hosting this Actor, skipping release", slog.String("worker", workerName)) } - return sandboxClass, released, nil + return released, nil } diff --git a/cmd/ateapi/internal/controlapi/crash_test.go b/cmd/ateapi/internal/controlapi/crash_test.go index 8e39921a7e..079a993d58 100644 --- a/cmd/ateapi/internal/controlapi/crash_test.go +++ b/cmd/ateapi/internal/controlapi/crash_test.go @@ -567,7 +567,6 @@ func TestCrashActor_Metrics(t *testing.T) { WorkerPool: "pool-1", WorkerPod: "pod-1", WorkerPodUid: "pod-uid-1", - SandboxClass: "gvisor", Status: &ateapipb.WorkerStatus{}, } if _, err := st.CreateWorker(ctx, worker); err != nil { @@ -589,6 +588,7 @@ func TestCrashActor_Metrics(t *testing.T) { WorkerPool: "pool-1", WorkerPod: "pod-1", WorkerPodUid: "pod-uid-1", + SandboxRuntime: &ateapipb.ActorSandboxRuntime{SandboxClass: "gvisor"}, }, }, } diff --git a/cmd/ateapi/internal/controlapi/dialer.go b/cmd/ateapi/internal/controlapi/dialer.go index 07160498b1..0d02b9d70c 100644 --- a/cmd/ateapi/internal/controlapi/dialer.go +++ b/cmd/ateapi/internal/controlapi/dialer.go @@ -15,6 +15,7 @@ package controlapi import ( + "context" "crypto/tls" "crypto/x509" "errors" @@ -36,6 +37,7 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/tools/cache" "k8s.io/utils/lru" ) @@ -54,6 +56,9 @@ type ateletConn struct { // AteletDialer handles gRPC connections to Atelet pods. type AteletDialer struct { ateletIndexer cache.Indexer + // getNode, when set, reads a node from the API server. NodeGone uses it to + // tell a deleted node from an atelet that is only missing for now. + getNode func(ctx context.Context, nodeName string) error // mu makes the lookup, stale eviction and insert in DialForAteletOnNode // atomic, so concurrent callers cannot evict each other's fresh conn. mu sync.Mutex @@ -80,6 +85,30 @@ func WithInsecureCredentials() DialerOption { return WithDialCredentials(func(string) (credentials.TransportCredentials, error) { return insecure.NewCredentials(), nil }) } +// WithNodeGetter sets how NodeGone reads a node. getNode returns the API +// server's error, so a NotFound error means that the node is deleted. +func WithNodeGetter(getNode func(ctx context.Context, nodeName string) error) DialerOption { + return func(d *AteletDialer) { d.getNode = getNode } +} + +// NodeGone reports whether nodeName is deleted from the cluster and no atelet +// pod is known on it. It reads the node live, so a stale cache cannot report a +// node as gone. Without a node getter it always reports false. +func (d *AteletDialer) NodeGone(ctx context.Context, nodeName string) (bool, error) { + if d.getNode == nil { + return false, nil + } + if err := d.getNode(ctx, nodeName); !apierrors.IsNotFound(err) { + return false, err // nil when the node exists + } + // An atelet that registered while the node was read means the node is back. + atelets, err := d.ateletIndexer.ByIndex(byNode, nodeName) + if err != nil { + return false, fmt.Errorf("while finding atelet on node %q: %w", nodeName, err) + } + return len(atelets) == 0, nil +} + // NewAteletDialer creates a new AteletDialer. clientBundlePath and serverCAPath // are used to build the per-atelet mTLS credentials used for every atelet // connection, and ateletSPIFFEID is the identity those credentials expect on diff --git a/cmd/ateapi/internal/controlapi/dialer_test.go b/cmd/ateapi/internal/controlapi/dialer_test.go index a13fe86b0e..9d67ba6891 100644 --- a/cmd/ateapi/internal/controlapi/dialer_test.go +++ b/cmd/ateapi/internal/controlapi/dialer_test.go @@ -15,6 +15,7 @@ package controlapi import ( + "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" @@ -24,6 +25,7 @@ import ( "errors" "math/big" "net/url" + "slices" "strconv" "sync" "testing" @@ -39,7 +41,9 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/cache" ) @@ -313,6 +317,48 @@ func TestVerifyAteletServerCert(t *testing.T) { }) } +// nodeGetter answers like the API server: the named nodes exist, others are +// NotFound, and lookupErr, when set, fails every read. +func nodeGetter(lookupErr error, names ...string) func(context.Context, string) error { + return func(_ context.Context, node string) error { + if lookupErr != nil { + return lookupErr + } + if slices.Contains(names, node) { + return nil + } + return apierrors.NewNotFound(schema.GroupResource{Resource: "nodes"}, node) + } +} + +func TestNodeGone(t *testing.T) { + spiffeID := installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace) + atelet := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ate-system", Name: "atelet-1"}, + Spec: corev1.PodSpec{NodeName: "node1"}, + } + unavailable := apierrors.NewServiceUnavailable("etcd is down") + for _, tc := range []struct { + name string + dialer *AteletDialer + wantGone bool + wantErr bool + }{ + {name: "node deleted", dialer: NewAteletDialer(newTestAteletIndexer(t), spiffeID, "", "", WithNodeGetter(nodeGetter(nil, "node2"))), wantGone: true}, + {name: "node exists", dialer: NewAteletDialer(newTestAteletIndexer(t), spiffeID, "", "", WithNodeGetter(nodeGetter(nil, "node1")))}, + {name: "lookup fails", dialer: NewAteletDialer(newTestAteletIndexer(t), spiffeID, "", "", WithNodeGetter(nodeGetter(unavailable))), wantErr: true}, + {name: "atelet registered meanwhile", dialer: NewAteletDialer(newTestAteletIndexer(t, atelet), spiffeID, "", "", WithNodeGetter(nodeGetter(nil)))}, + {name: "no node getter", dialer: NewAteletDialer(newTestAteletIndexer(t), spiffeID, "", "")}, + } { + t.Run(tc.name, func(t *testing.T) { + gone, err := tc.dialer.NodeGone(context.Background(), "node1") + if gone != tc.wantGone || (err != nil) != tc.wantErr { + t.Fatalf("NodeGone = %v, %v; want %v, error %v", gone, err, tc.wantGone, tc.wantErr) + } + }) + } +} + // newTestAteletIndexer builds an indexer with the production byNode index // shape, holding the given atelet pods. func newTestAteletIndexer(t *testing.T, pods ...*corev1.Pod) cache.Indexer { diff --git a/cmd/ateapi/internal/controlapi/egress_policy_test.go b/cmd/ateapi/internal/controlapi/egress_policy_test.go index 207aa3adfb..99552be053 100644 --- a/cmd/ateapi/internal/controlapi/egress_policy_test.go +++ b/cmd/ateapi/internal/controlapi/egress_policy_test.go @@ -142,8 +142,15 @@ func TestActorEgressPolicy(t *testing.T) { Actor: actorRef, Name: resources.SingletonName, }) - if err != nil || !proto.Equal(deleted, updated) { - t.Fatalf("deleted policy = %v, %v; want %v", deleted, err, updated) + if err != nil { + t.Fatalf("DeleteActorEgressPolicy: %v", err) + } + if deleted.GetMetadata().GetDeleteTime() == nil { + t.Error("the deleted policy carries no delete_time") + } + deleted.Metadata.DeleteTime = nil + if !proto.Equal(deleted, updated) { + t.Fatalf("deleted policy = %v; want %v with delete_time set", deleted, updated) } if _, err := service.GetActorEgressPolicy(t.Context(), &ateapipb.GetActorEgressPolicyRequest{ Actor: actorRef, diff --git a/cmd/ateapi/internal/controlapi/export_test.go b/cmd/ateapi/internal/controlapi/export_test.go index 3d0a435cac..daeed181c6 100644 --- a/cmd/ateapi/internal/controlapi/export_test.go +++ b/cmd/ateapi/internal/controlapi/export_test.go @@ -53,7 +53,7 @@ type ReleaseWindowWorkflow struct { // Snapshot, when set, is the actor's latest snapshot. Snapshot func(t *testing.T) *ateapipb.Snapshot // Finalize runs the workflow or step that releases the worker and then - // records the actor's new state against st. Pause and suspend reach the + // records the actor's new state against st. Async and sync suspend reach the // fake atelet newWireCaptureWorkflow serves for the worker // WireTestAssignment names, so the actor must be assigned that worker // and run from the template SeedSuspendTemplate stores. @@ -64,11 +64,11 @@ type ReleaseWindowWorkflow struct { WantFinalState ateapipb.ActorState } -// ReleaseWindowWorkflows returns pause, suspend, crash, and a resume onto an -// ineligible worker: the workflows that release a worker before rewriting the -// actor's status while the actor is in a state HostingNode resolves through -// the worker assignment. Revert and delete release theirs in REVERTING and -// DELETING, which HostingNode never resolves to a node. +// ReleaseWindowWorkflows returns async suspend ("pause"), sync suspend, crash, +// and a resume onto an ineligible worker: the workflows that release a worker +// before rewriting the actor's status while the actor is in a state HostingNode +// resolves through the worker assignment. Revert and delete release theirs in +// REVERTING and DELETING, which HostingNode never resolves to a node. func ReleaseWindowWorkflows() []ReleaseWindowWorkflow { // The workers these tests create carry no labels, so a template // selecting a pool makes any of them ineligible. @@ -80,10 +80,10 @@ func ReleaseWindowWorkflows() []ReleaseWindowWorkflow { Name: "pause", State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, Snapshot: func(t *testing.T) *ateapipb.Snapshot { - // A pause in progress: a PENDING durable copy, not yet captured. + // An async suspend in progress: a PENDING durable copy, not yet captured. return newInProgressSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "local-snap-1") }, - // The actor is seeded SUSPENDING, so the pause skips its mark step + // The actor is seeded SUSPENDING, so the suspend skips its mark step // and its first actor update is the finalize commit. Finalize: func(t *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, _ *ateapipb.Actor) error { w, _ := newWireCaptureWorkflow(t, st) @@ -99,7 +99,7 @@ func ReleaseWindowWorkflows() []ReleaseWindowWorkflow { Snapshot: func(t *testing.T) *ateapipb.Snapshot { return newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) }, - // As for the pause, the finalize commit is the first actor update. + // As for the async suspend, the finalize commit is the first actor update. Finalize: func(t *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, _ *ateapipb.Actor) error { w, _ := newWireCaptureWorkflow(t, st) _, err := w.SuspendActor(ctx, actorRef, suspendSync) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go index 047996805c..42f33e9117 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go @@ -24,6 +24,7 @@ import ( "go.ate.dev/substrate/internal/preview" "go.ate.dev/substrate/internal/resources" "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/util/wait" "github.com/google/go-cmp/cmp" @@ -118,9 +119,12 @@ func TestActorTemplateCRUD(t *testing.T) { if err != nil { t.Fatalf("DeleteActorTemplate failed: %v", err) } - if diff := cmp.Diff(created, deleted, protocmp.Transform()); diff != "" { + if diff := cmp.Diff(created, deleted, protocmp.Transform(), ignoreVersion, ignoreTimestamps); diff != "" { t.Errorf("DeleteActorTemplate response mismatch (-created +deleted):\n%s", diff) } + if deleted.GetMetadata().GetDeleteTime() == nil { + t.Errorf("the deleted template carries no delete_time") + } _, err = tc.client.GetActorTemplate(ctx, &ateapipb.GetActorTemplateRequest{ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl-a"}}) assertGrpcError(t, err, codes.NotFound, "ActorTemplate "+testAtespace+"/tmpl-a not found") _, err = tc.client.DeleteActorTemplate(ctx, &ateapipb.DeleteActorTemplateRequest{ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl-a"}}) @@ -468,3 +472,69 @@ func TestCreateActorTemplate_PerTemplateSnapshotLocationPreview(t *testing.T) { }) } } + +func TestCreateActorTemplate_GoldenEgressPolicyPreview(t *testing.T) { + tc := setupPreviewTest(t, namespaceForTest("ns-template-golden-egress-preview")) + t.Cleanup(tc.cleanup) + base := createTemplateWithSelector(t, tc, "base", nil) + + for i, tt := range []struct { + name string + gates []string + optIn bool + snapshotsDisabled bool + wantCode codes.Code + }{ + {name: "gate off", optIn: true, wantCode: codes.InvalidArgument}, + {name: "no opt-in", gates: []string{"GoldenEgressPolicy"}, wantCode: codes.InvalidArgument}, + {name: "unrelated gate", gates: []string{"ExternalVolumes"}, optIn: true, wantCode: codes.InvalidArgument}, + {name: "gate on and opt-in", gates: []string{"GoldenEgressPolicy"}, optIn: true, wantCode: codes.OK}, + {name: "all gates and opt-in", gates: []string{"ALL"}, optIn: true, wantCode: codes.OK}, + {name: "golden snapshots disabled", gates: []string{"GoldenEgressPolicy"}, optIn: true, snapshotsDisabled: true, wantCode: codes.InvalidArgument}, + } { + t.Run(tt.name, func(t *testing.T) { + preview.InitForTest(t, tt.gates...) + client := tc.noOptInClient + if tt.optIn { + client = tc.client + } + tmpl := proto.Clone(base).(*ateapipb.ActorTemplate) + tmpl.Metadata = &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: fmt.Sprintf("golden-egress-%d", i)} + tmpl.Status = nil + tmpl.GoldenEgressPolicy = &ateapipb.EgressPolicyTemplate{Rules: functionalEgressPolicy().Rules} + if tt.snapshotsDisabled { + tmpl.SnapshotConfig.GoldenSnapshotConfig = &ateapipb.GoldenSnapshotConfig{Mode: ateapipb.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED} + } + ref := resources.ActorTemplateRefFromActorTemplate(tmpl).ToObjectRef() + created, err := client.CreateActorTemplate(t.Context(), &ateapipb.CreateActorTemplateRequest{ActorTemplate: tmpl}) + if tt.wantCode != codes.OK { + assertGrpcErrorRegex(t, err, tt.wantCode, "golden_egress_policy") + _, err = client.GetActorTemplate(t.Context(), &ateapipb.GetActorTemplateRequest{ActorTemplate: ref}) + assertGrpcErrorRegex(t, err, codes.NotFound, "not found") + return + } + if err != nil { + t.Fatalf("CreateActorTemplate: %v", err) + } + wantPolicy := &ateapipb.EgressPolicyTemplate{Rules: []*ateapipb.EgressRule{{ + Http: &ateapipb.HttpRule{Hostnames: []string{"api.example.com"}, Ports: &ateapipb.Ports{Numbers: []int32{80}}}, + }}} + if diff := cmp.Diff(wantPolicy, created.GetGoldenEgressPolicy(), protocmp.Transform()); diff != "" { + t.Fatalf("golden policy (-want +got):\n%s", diff) + } + + // Opting out or disabling the gate must not strand existing templates. + preview.InitForTest(t) + got, err := tc.noOptInClient.GetActorTemplate(t.Context(), &ateapipb.GetActorTemplateRequest{ActorTemplate: ref}) + if err != nil { + t.Fatalf("GetActorTemplate with gate disabled: %v", err) + } + if diff := cmp.Diff(created, got, protocmp.Transform()); diff != "" { + t.Fatalf("template changed with gate disabled (-want +got):\n%s", diff) + } + if _, err := tc.noOptInClient.DeleteActorTemplate(t.Context(), &ateapipb.DeleteActorTemplateRequest{ActorTemplate: ref}); err != nil { + t.Fatalf("DeleteActorTemplate with gate disabled: %v", err) + } + }) + } +} diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index da875809a9..891efc22cb 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -35,6 +35,7 @@ import ( "go.ate.dev/substrate/internal/actoridjwt" "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/installdefaults" "go.ate.dev/substrate/internal/objectstore" "go.ate.dev/substrate/internal/preview" "go.ate.dev/substrate/internal/proto/ateletpb" @@ -48,7 +49,9 @@ import ( "google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/testing/protocmp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/ptr" ) // TestCreateActor_Success tests the happy path for creating an actor. @@ -1146,6 +1149,9 @@ func TestDeleteActor_Success(t *testing.T) { if got := deleted.GetMetadata().GetName(); got != "id1" { t.Errorf("deleted actor name = %q, want id1", got) } + if deleted.GetMetadata().GetDeleteTime() == nil { + t.Errorf("the deleted actor carries no delete_time") + } if got := deleted.GetMetadata().GetAtespace(); got != testAtespace { t.Errorf("deleted actor atespace = %q, want %q", got, testAtespace) } @@ -1379,6 +1385,69 @@ func TestDeleteActor_Crashed(t *testing.T) { assertGrpcError(t, err, codes.NotFound, "Actor test-atespace/id1 not found") } +// A delete that fails partway leaves the actor DELETING. Losing its worker +// before the retry must keep it DELETING, since a CRASHED actor can be +// reverted and resumed, and the retry must still finish the delete. +func TestDeleteActor_RetryAfterWorkerDelete(t *testing.T) { + ns := namespaceForTest("ns-delete-retry-worker-gone") + tc := setupTest(t, ns) + defer tc.cleanup() + + createTemplate(t, tc, ns) + createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + + ctx := context.Background() + actorRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"} + if _, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor failed: %v", err) + } + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { + t.Fatalf("ResumeActor failed: %v", err) + } + + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.FailTerminate = status.Error(codes.Unavailable, "node unavailable") + tc.fakeAtelet.Lock.Unlock() + if _, err := tc.client.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: actorRef}); err == nil { + t.Fatal("DeleteActor succeeded, want it to fail on Terminate") + } + + // This would crash a RUNNING Actor. But, should keep a DELETING actor + // unchanged. + deleteWorkerPod(t, tc, ns, "worker-1") + actor, err := tc.client.GetActor(ctx, &ateapipb.GetActorRequest{Actor: actorRef}) + if err != nil { + t.Fatalf("GetActor failed: %v", err) + } + if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_DELETING { + t.Fatalf("state after worker delete = %v, want DELETING", got) + } + + // Retry deleting the actor after its worker is gone. + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.FailTerminate = nil + tc.fakeAtelet.TerminateRequest = nil + tc.fakeAtelet.Lock.Unlock() + if _, err := tc.client.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: actorRef}); err != nil { + t.Fatalf("retried DeleteActor failed: %v", err) + } + + // The worker is gone, so the retry must not name its pod. + tc.fakeAtelet.Lock.Lock() + terminate := tc.fakeAtelet.TerminateRequest + tc.fakeAtelet.Lock.Unlock() + if terminate == nil { + t.Error("retried DeleteActor did not call atelet Terminate") + } else if gotUID := terminate.GetWorkerPodUid(); gotUID != "" { + t.Errorf("TerminateRequest.WorkerPodUid = %q, want empty", gotUID) + } + _, err = tc.client.GetActor(ctx, &ateapipb.GetActorRequest{Actor: actorRef}) + assertGrpcError(t, err, codes.NotFound, "Actor test-atespace/id1 not found") +} + func TestDeleteActor_NotFound(t *testing.T) { ns := namespaceForTest("ns-delete-notfound") tc := setupTest(t, ns) @@ -1435,6 +1504,9 @@ func TestDeleteActor_StateDeleting(t *testing.T) { if _, err := tc.persistence.CreateActor(context.Background(), deletingActor); err != nil { t.Fatalf("CreateActor: %v", err) } + if _, err := tc.persistence.MarkActorForDeletion(context.Background(), resources.ActorRef{Atespace: testAtespace, Name: "deleting-actor"}, store.DeletePreconditions{}, nil); err != nil { + t.Fatalf("MarkActorForDeletion: %v", err) + } if _, err := tc.service.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "deleting-actor"}, @@ -1447,9 +1519,9 @@ func TestDeleteActor_StateDeleting(t *testing.T) { } } -// TestDeleteActor_Paused deletes a paused actor: a paused actor is SUSPENDED -// (holding a node-local snapshot), and the delete terminates it on its -// assigned node so the local snapshot is removed. +// TestDeleteActor_Paused deletes an asynchronously suspended actor: it is +// SUSPENDED (holding a node-local snapshot), and the delete terminates it on +// its assigned node so the local snapshot is removed. func TestDeleteActor_Paused(t *testing.T) { ns := namespaceForTest("ns-delete-paused") tc := setupTest(t, ns) @@ -1797,7 +1869,7 @@ func TestActorLifecycle_WithExternalVolumes(t *testing.T) { } waitForWorkerAvailable(t, tc, workerName) - // 4. ResumeActor from paused + // 4. ResumeActor from node-local snapshot waitForWorkerAvailable(t, tc, workerName) resumeResp2, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-vol-lc"}, @@ -2796,7 +2868,7 @@ func TestSuspendActor_VolumeDetachFailure_DeleteActor(t *testing.T) { // 1. Creates a template with an external volume and a mock worker pod on node1. // 2. Creates and resumes an actor to ACTOR_STATE_RUNNING; verifies volume is attached to node1 and restore request carries node1 publishContext. // 3. Calls async SuspendActor; verifies volume is detached from node1 and actor transitions to ACTOR_STATE_SUSPENDED with a local snapshot. -// 4. Calls ResumeActor from pause; verifies volume is re-attached to node1 and restore request carries node1 publishContext. +// 4. Calls ResumeActor from local snapshot; verifies volume is re-attached to node1 and restore request carries node1 publishContext. // 5. Calls SuspendActor, replaces worker-1 (node1) with worker-2 (node2), and calls ResumeActor; verifies volume is attached to node2 and restore request carries node2 publishContext. // 6. Calls SuspendActor and DeleteActor; verifies volume is detached from node2 and deleted. func TestSuspendActorAsync_VolumeLifecycle_DetachResumeAndMigrate(t *testing.T) { @@ -2880,7 +2952,7 @@ func TestSuspendActorAsync_VolumeLifecycle_DetachResumeAndMigrate(t *testing.T) plugin.mu.Unlock() waitForWorkerAvailable(t, tc, workerName) - // 4. ResumeActor from pause: volume is re-attached to node1 and actor transitions to ACTOR_STATE_RUNNING. + // 4. ResumeActor from local snapshot: volume is re-attached to node1 and actor transitions to ACTOR_STATE_RUNNING. tc.fakeAtelet.Reset() _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "pause-vol-actor"}, @@ -2899,7 +2971,7 @@ func TestSuspendActorAsync_VolumeLifecycle_DetachResumeAndMigrate(t *testing.T) t.Errorf("actor state after retry = %v, want ACTOR_STATE_RUNNING", getResp.GetStatus().GetState()) } plugin.mu.Lock() - // Should have attached twice: first resume + resume from pause + // Should have attached twice: first resume + resume from local snapshot if diff := cmp.Diff([]string{"node1", "node1"}, plugin.attachedNodes); diff != "" { t.Errorf("attached nodes after resume from pause mismatch (-want +got):\n%s", diff) } @@ -3182,7 +3254,6 @@ func TestResumeActor(t *testing.T) { WorkerPodUid: podUID, Ips: []string{"127.0.0.1"}, NodeName: "node1", - SandboxClass: "gvisor", Labels: map[string]string{poolLabelKey: ns}, Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, @@ -3191,12 +3262,11 @@ func TestResumeActor(t *testing.T) { // All a listing reports of the assignments: one actor and the // compute its template declares. Allocated: &ateapipb.WorkerResources{Actors: 1, CpuMillicores: 1000, MemoryBytes: 512 << 20}, - DefaultRuntime: defaultWorkerRuntime("gvisor"), + SandboxClasses: workerSandboxClasses(defaultWorkerRuntime("gvisor")), }, } - if diff := cmp.Diff(wantWorker, actorWorker, protocmp.Transform(), ignoreServerMetadata, - protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "version")); diff != "" { + if diff := cmp.Diff(wantWorker, actorWorker, protocmp.Transform(), ignoreUID, ignoreVersion, ignoreTimestamps); diff != "" { t.Errorf("Worker state mismatch (-want +got):\n%s", diff) } } @@ -3365,6 +3435,11 @@ func TestResumeActor_DataSnapshotIgnoresGolden(t *testing.T) { if got := restoreReq.GetSnapshot().GetObject().GetSnapshotUri(); got != actorSnapshotURI { t.Errorf("restore config snapshot uri = %q, want the actor's data snapshot %q", got, actorSnapshotURI) } + // The actor's own snapshot is replaced by its next suspend: atelet must + // not cache it. + if got := restoreReq.GetSnapshot().GetObject().GetSharing(); got != ateletpb.SnapshotSharing_SNAPSHOT_SHARING_PRIVATE { + t.Errorf("restore of the actor's own snapshot declared sharing %v, want PRIVATE", got) + } } // TestSuspendActor_ReplacedSnapshotReleaseFailure verifies that object storage @@ -3711,7 +3786,7 @@ func TestResumeActor_AteletTransportErrorLeavesActorResuming(t *testing.T) { } // TestResumeActor_LocalRestoreFailureCrashesActor: an atelet error restoring a -// paused actor from its local snapshot crashes it and releases its worker. +// suspended actor from its local snapshot crashes it and releases its worker. func TestResumeActor_LocalRestoreFailureCrashesActor(t *testing.T) { ns := namespaceForTest("ns-resume-local-crash") tc := setupTest(t, ns) @@ -4027,6 +4102,213 @@ func TestSuspendActor(t *testing.T) { assertSnapshotPresent(t, tc, cloneSnapshotURI) } +func TestResumeActor_SyncSuspendKeepsLocalCopy(t *testing.T) { + for _, tt := range []struct { + name string + repoint bool + relocate bool + wantNode string + wantTemplate string + wantFidelity ateletpb.SnapshotFidelity + }{ + { + name: "same template on the original node", + wantNode: "node1", + wantTemplate: "tmpl1", + wantFidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + }, + { + name: "new template on the original node", + repoint: true, + wantNode: "node1", + wantTemplate: "tmpl2", + wantFidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + }, + { + name: "new template on another node", + repoint: true, + relocate: true, + wantNode: "node2", + wantTemplate: "tmpl2", + wantFidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + }, + } { + t.Run(tt.name, func(t *testing.T) { + ns := namespaceForTest("ns-sync-local-resume") + tc := setupTest(t, ns) + defer tc.cleanup() + ctx := context.Background() + resumeKinds := func() map[string]uint64 { + var rm metricdata.ResourceMetrics + if err := tc.metricReader.Collect(ctx, &rm); err != nil { + t.Fatal(err) + } + counts := make(map[string]uint64) + for _, scope := range rm.ScopeMetrics { + for _, metric := range scope.Metrics { + if metric.Name != "ate.actor.lifecycle.operation.duration" { + continue + } + hist, ok := metric.Data.(metricdata.Histogram[float64]) + if !ok { + t.Fatalf("%s data = %T, want float64 histogram", metric.Name, metric.Data) + } + for _, point := range hist.DataPoints { + op, _ := point.Attributes.Value(ateattr.ActorOperationNameKey) + kind, present := point.Attributes.Value(ateattr.SnapshotKindKey) + if op.AsString() == ateattr.OperationResume && present { + counts[kind.AsString()] += point.Count + } + } + } + } + return counts + } + tmpl := createTemplate(t, tc, ns) + if tt.repoint { + replacement := proto.CloneOf(tmpl) + replacement.Metadata = &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "tmpl2"} + replacement.Status = nil + if _, err := tc.client.CreateActorTemplate(ctx, &ateapipb.CreateActorTemplateRequest{ActorTemplate: replacement}); err != nil { + t.Fatalf("CreateActorTemplate(tmpl2): %v", err) + } + } + worker := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.KeepCheckpointLocal = true + tc.fakeAtelet.Lock.Unlock() + + actorRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-1"} + if _, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: actorRef.GetName()}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor: %v", err) + } + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { + t.Fatalf("initial ResumeActor: %v", err) + } + if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: actorRef, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, + }); err != nil { + t.Fatalf("sync SuspendActor: %v", err) + } + suspended, err := tc.client.GetActor(ctx, &ateapipb.GetActorRequest{Actor: actorRef}) + if err != nil { + t.Fatalf("GetActor after suspend: %v", err) + } + if got := suspended.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + t.Fatalf("state after suspend = %v, want SUSPENDED", got) + } + if got := suspended.GetStatus().GetAssignedNode(); got != "node1" { + t.Fatalf("AssignedNode after sync suspend = %q, want node1", got) + } + if got := suspended.GetStatus().GetWorkerAssignment(); got != nil { + t.Fatalf("WorkerAssignment after sync suspend = %v, want none", got) + } + snap := localSnapshot(suspended.GetStatus()) + if snap == nil || snap.GetLocality() != "node1" { + t.Fatalf("sync snapshot = %v, want its local copy on node1 recorded", snap) + } + if got := snap.GetDurableSnapshot().GetStatus(); got != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + t.Fatalf("sync snapshot durable status = %v, want COMPLETED", got) + } + if got := snap.GetActorTemplateUid(); got != tmpl.GetMetadata().GetUid() { + t.Fatalf("snapshot template UID = %q, want original template %q", got, tmpl.GetMetadata().GetUid()) + } + if snap.GetUuid() == "" || snap.GetGeneration() == 0 || snap.GetDurableSnapshot().GetObject().GetSnapshotUri() == "" { + t.Fatalf("snapshot has incomplete identity: %v", snap) + } + assertSnapshotPresent(t, tc, snap.GetDurableSnapshot().GetObject().GetSnapshotUri()) + waitForWorkerAvailable(t, tc, worker) + + if tt.repoint { + toUpdate := proto.CloneOf(suspended) + toUpdate.ActorTemplate = &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl2"} + updated, err := tc.client.UpdateActor(ctx, &ateapipb.UpdateActorRequest{Actor: toUpdate}) + if err != nil { + t.Fatalf("UpdateActor with durable local snapshot: %v", err) + } + if got := localSnapshot(updated.GetStatus()); !proto.Equal(got, snap) { + t.Fatalf("snapshot after template update = %v, want unchanged %v", got, snap) + } + } + if tt.relocate { + // Fill node1 so resume must detach before moving to node2. + squatterRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "squatter"} + if _, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: squatterRef.GetName()}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor(squatter): %v", err) + } + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: squatterRef}); err != nil { + t.Fatalf("ResumeActor(squatter): %v", err) + } + createWorkerPod(t, tc, ns, "worker-2", "node2", "pool1") + setupAteletOnNode(t, tc, "atelet-node2", "node2") + } + + before := resumeKinds() + resumed, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}) + if err != nil { + t.Fatalf("ResumeActor with durable local snapshot: %v", err) + } + if got := resumed.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RUNNING { + t.Errorf("state after resume = %v, want RUNNING", got) + } + if got := resumed.GetActor().GetStatus().GetWorkerAssignment().GetNodeName(); got != tt.wantNode { + t.Errorf("resumed on node %q, want %q", got, tt.wantNode) + } + restore := tc.fakeAtelet.lastRestoreRequest() + if restore == nil || restore.GetActorName() != actorRef.GetName() { + t.Fatalf("Restore request = %v, want actor-1", restore) + } + if got := restore.GetWorkerPodUid(); got != resumed.GetActor().GetStatus().GetWorkerAssignment().GetWorkerPodUid() { + t.Errorf("Restore worker UID = %q, want the assigned worker", got) + } + if got := restore.GetActorTemplateName(); got != tt.wantTemplate { + t.Errorf("Restore template = %q, want %q", got, tt.wantTemplate) + } + if got := restore.GetSnapshot().GetFidelity(); got != tt.wantFidelity { + t.Errorf("Restore fidelity = %v, want %v", got, tt.wantFidelity) + } + if got := restore.GetSnapshot(); got.GetSnapshotUuid() != snap.GetUuid() || + got.GetGeneration() != snap.GetGeneration() || + got.GetObject().GetSnapshotUri() != snap.GetDurableSnapshot().GetObject().GetSnapshotUri() { + t.Errorf("Restore snapshot = %v, want original checkpoint identity %v", got, snap) + } + wantLocality := "node1" + wantKind := ateattr.SnapshotKindLocal + if tt.relocate { + wantLocality = "" + wantKind = ateattr.SnapshotKindLatest + } + if got := localSnapshot(resumed.GetActor().GetStatus()).GetLocality(); got != wantLocality { + t.Errorf("locality after resume = %q, want %q", got, wantLocality) + } + tc.fakeAtelet.Lock.Lock() + detached := tc.fakeAtelet.DetachCalled + tc.fakeAtelet.Lock.Unlock() + if detached != tt.relocate { + t.Errorf("detach called = %v, want %v", detached, tt.relocate) + } + + counts := resumeKinds() + for kind, count := range before { + counts[kind] -= count + if counts[kind] == 0 { + delete(counts, kind) + } + } + if diff := cmp.Diff(map[string]uint64{wantKind: 1}, counts); diff != "" { + t.Errorf("restore snapshot kinds (-want +got):\n%s", diff) + } + }) + } +} + // TestResumeActor_RepointTemplateBeforeResume checks what an Actor cloned from // a tag asks atelet to restore on its first resume. // @@ -4149,9 +4431,9 @@ func TestResumeActor_RepointTemplateBeforeResume(t *testing.T) { } // TestResumeActor_PausedAfterRepointUsesLocalProvenance verifies that an actor -// paused after a template repoint restores from its pause checkpoint in FULL, -// not DATA. At that point the actor holds an external snapshot captured on v1 -// and a local checkpoint captured on v2; judging the local restore by the +// suspended after a template repoint restores from its local checkpoint in +// FULL, not DATA. At that point the actor holds an external snapshot captured +// on v1 and a local checkpoint captured on v2; judging the local restore by the // external snapshot's provenance would wrongly discard the v2 memory image. func TestResumeActor_PausedAfterRepointUsesLocalProvenance(t *testing.T) { ns := namespaceForTest("ns-repoint-pause") @@ -4193,8 +4475,8 @@ func TestResumeActor_PausedAfterRepointUsesLocalProvenance(t *testing.T) { } // Resume on v2 (this restore is DATA, from the v1 external snapshot), then - // pause; the pause checkpoint is captured on v2 while the external - // snapshot still says v1. + // suspend asynchronously; the local checkpoint is captured on v2 while the + // external snapshot still says v1. if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { t.Fatalf("ResumeActor(v2 from v1 snapshot) failed: %v", err) } @@ -4211,8 +4493,8 @@ func TestResumeActor_PausedAfterRepointUsesLocalProvenance(t *testing.T) { } waitForWorkerAvailable(t, tc, worker) - // Resume from pause: the local checkpoint was captured on v2 and the - // actor's template is v2, so the restore must be FULL. + // Resume from local snapshot: the local checkpoint was captured on v2 and + // the actor's template is v2, so the restore must be FULL. if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { t.Fatalf("ResumeActor(from v2 pause) failed: %v", err) } @@ -4225,6 +4507,374 @@ func TestResumeActor_PausedAfterRepointUsesLocalProvenance(t *testing.T) { } } +func TestResumeActor_PausedLocalSnapshotRuntimeFallback(t *testing.T) { + rtA := &ateapipb.ActorSandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "sandbox_bundle_hash", Value: "hash-a"}, + }, + }, + } + rtB := proto.CloneOf(rtA) + rtB.Version.Attributes[1].Value = "hash-b" + + for _, tt := range []struct { + name string + compatibleNode bool + failDetach bool + wantWorker string + wantNode string + wantRuntime *ateapipb.ActorSandboxRuntime + wantFidelity ateletpb.SnapshotFidelity + }{ + { + name: "compatible worker on another node preserves memory", + compatibleNode: true, + wantWorker: "worker-2", + wantNode: "node2", + wantRuntime: rtA, + wantFidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + }, + { + name: "no compatible worker falls back after detach", + wantWorker: "worker-1", + wantNode: "node1", + wantRuntime: rtB, + wantFidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, + }, + { + name: "unreachable node uses the uploaded checkpoint", + compatibleNode: true, + failDetach: true, + wantWorker: "worker-2", + wantNode: "node2", + wantRuntime: rtA, + wantFidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + }, + } { + t.Run(tt.name, func(t *testing.T) { + ns := namespaceForTest("ns-paused-runtime-fallback") + tc := setupTest(t, ns) + t.Cleanup(tc.cleanup) + ctx := context.Background() + createTemplate(t, tc, ns) + + workerPod := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + reportWorkerRuntime(t, tc, workerPod, rtA) + + actorRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-1"} + if _, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: actorRef.GetName()}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor: %v", err) + } + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { + t.Fatalf("initial ResumeActor: %v", err) + } + pauseResp, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: actorRef, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, + }) + if err != nil { + t.Fatalf("SuspendActor: %v", err) + } + paused := localSnapshot(pauseResp.GetActor().GetStatus()) + if paused == nil || !proto.Equal(paused.GetSandboxRuntime(), rtA) { + t.Fatalf("pause snapshot = %v, want runtime A", paused) + } + waitForWorkerAvailable(t, tc, workerPod) + + upload := &ateapipb.SnapshotReport{ + Uuid: paused.GetUuid(), + Generation: paused.GetGeneration(), + Fidelity: paused.GetFidelity(), + Object: paused.GetDurableSnapshot().GetObject(), + } + if err := tc.service.ApplyActorSnapshotState(ctx, resources.ActorRefFromActor(pauseResp.GetActor()), pauseResp.GetActor().GetMetadata().GetUid(), "node1", &ateapipb.ActorSnapshotState{ + LatestLocal: upload, + LatestDurable: upload, + }); err != nil { + t.Fatalf("ApplyActorSnapshotState(upload): %v", err) + } + reportWorkerRuntime(t, tc, workerPod, rtB) + if tt.compatibleNode { + other := createWorkerPod(t, tc, ns, "worker-2", "node2", "pool1") + reportWorkerRuntime(t, tc, other, rtA) + setupAteletOnNode(t, tc, "atelet-node2", "node2") + } + if tt.failDetach { + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.FailDetach = status.Error(codes.Unavailable, "node unavailable") + tc.fakeAtelet.Lock.Unlock() + } + + resumed, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}) + if err != nil { + t.Fatalf("ResumeActor after runtime change: %v", err) + } + assignment := resumed.GetActor().GetStatus().GetWorkerAssignment() + if assignment.GetWorkerPod() != tt.wantWorker || assignment.GetNodeName() != tt.wantNode { + t.Errorf("assignment = %v, want %s on %s", assignment, tt.wantWorker, tt.wantNode) + } + if !proto.Equal(assignment.GetSandboxRuntime(), tt.wantRuntime) { + t.Errorf("assigned runtime = %v, want %v", assignment.GetSandboxRuntime(), tt.wantRuntime) + } + tc.fakeAtelet.Lock.Lock() + detached := tc.fakeAtelet.DetachCalled + tc.fakeAtelet.Lock.Unlock() + if !detached { + t.Error("resume did not detach the incompatible local checkpoint") + } + if snap := localSnapshot(resumed.GetActor().GetStatus()); snap != nil { + t.Errorf("local snapshot after detach = %v, want none", snap) + } + + restore := tc.fakeAtelet.lastRestoreRequest().GetSnapshot() + if restore.GetSnapshotUuid() != paused.GetUuid() || restore.GetGeneration() != paused.GetGeneration() || + restore.GetObject().GetSnapshotUri() != paused.GetDurableSnapshot().GetObject().GetSnapshotUri() { + t.Errorf("restore snapshot = %v, want the paused checkpoint's identity", restore) + } + if restore.GetFidelity() != tt.wantFidelity { + t.Errorf("restore fidelity = %v, want %v", restore.GetFidelity(), tt.wantFidelity) + } + }) + } +} + +// An older upload cannot authorize fallback for a newer checkpoint. +func TestResumeActor_PausedPendingSnapshotRefusesRuntimeFallback(t *testing.T) { + ns := namespaceForTest("ns-paused-pending-refusal") + tc := setupTest(t, ns) + defer tc.cleanup() + + ctx := context.Background() + createTemplate(t, tc, ns) + + rtA := &ateapipb.ActorSandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "sandbox_bundle_hash", Value: "hash-a"}, + }, + }, + } + rtB := &ateapipb.ActorSandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "sandbox_bundle_hash", Value: "hash-b"}, + }, + }, + } + + workerPod := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + reportWorkerRuntime(t, tc, workerPod, rtA) + + const name = "actor-1" + actorRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: name} + if _, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor failed: %v", err) + } + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { + t.Fatalf("ResumeActor failed: %v", err) + } + // Keep an older durable generation beside the pending checkpoint. + if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: actorRef, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, + }); err != nil { + t.Fatalf("sync SuspendActor failed: %v", err) + } + waitForWorkerAvailable(t, tc, workerPod) + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { + t.Fatalf("ResumeActor from the durable snapshot failed: %v", err) + } + pauseResp, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: actorRef, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, + }) + if err != nil { + t.Fatalf("pause SuspendActor failed: %v", err) + } + paused := localSnapshot(pauseResp.GetActor().GetStatus()) + if paused == nil { + t.Fatalf("pause left no local snapshot: %v", pauseResp.GetActor().GetStatus()) + } + if got := paused.GetDurableSnapshot().GetStatus(); got != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING { + t.Fatalf("pause checkpoint durable status = %v, want PENDING (not yet uploaded)", got) + } + durable := durableSnapshot(pauseResp.GetActor().GetStatus()) + if durable == nil || durable.GetGeneration() >= paused.GetGeneration() { + t.Fatalf("want an older COMPLETED durable snapshot beside the pause checkpoint, got %v", pauseResp.GetActor().GetStatus().GetSnapshots()) + } + waitForWorkerAvailable(t, tc, workerPod) + + // The pending checkpoint still needs upload before runtime fallback. + reportWorkerRuntime(t, tc, workerPod, rtB) + tc.fakeAtelet.Reset() + tc.fakeAtelet.FailDetach = status.Error(codes.Unavailable, "checkpoint upload unavailable") + + _, err = tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}) + assertGrpcError(t, err, codes.Unavailable, "while calling atelet DetachActor: rpc error: code = Unavailable desc = checkpoint upload unavailable") + tc.fakeAtelet.Lock.Lock() + detachCalled := tc.fakeAtelet.DetachCalled + tc.fakeAtelet.Lock.Unlock() + if !detachCalled { + t.Error("ResumeActor did not attempt to detach the pending checkpoint") + } + + if restore := tc.fakeAtelet.lastRestoreRequest(); restore != nil { + t.Errorf("ResumeActor sent a Restore request: %v, want none (neither the checkpoint nor the older generation)", restore) + } + after, err := tc.persistence.GetActor(ctx, resources.ActorRefFromActor(pauseResp.GetActor())) + if err != nil { + t.Fatalf("GetActor: %v", err) + } + if got := after.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + t.Errorf("state after the refused resume = %v, want SUSPENDED", got) + } + if got := after.GetStatus().GetWorkerAssignment(); got != nil { + t.Errorf("WorkerAssignment after the refused resume = %v, want none", got) + } + stillLocal := localSnapshot(after.GetStatus()) + if stillLocal == nil || stillLocal.GetUuid() != paused.GetUuid() || + stillLocal.GetDurableSnapshot().GetStatus() != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING { + t.Errorf("pause checkpoint after the refused resume = %v, want %v still PENDING", stillLocal, paused) + } + if got := durableSnapshotURI(after.GetStatus()); got != durable.GetDurableSnapshot().GetObject().GetSnapshotUri() { + t.Errorf("older durable snapshot = %q, want it untouched at %q", got, durable.GetDurableSnapshot().GetObject().GetSnapshotUri()) + } +} + +func TestResumeActor_RetryRelaxedPauseAssignmentRestoresSameSnapshot(t *testing.T) { + ns := namespaceForTest("ns-paused-relaxed-retry") + tc := setupTest(t, ns) + defer tc.cleanup() + + ctx := context.Background() + createTemplate(t, tc, ns) + + rtA := &ateapipb.ActorSandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "sandbox_bundle_hash", Value: "hash-a"}, + }, + }, + } + rtB := &ateapipb.ActorSandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "sandbox_bundle_hash", Value: "hash-b"}, + }, + }, + } + + workerPod := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + reportWorkerRuntime(t, tc, workerPod, rtA) + + const name = "actor-1" + actorRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: name} + if _, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor failed: %v", err) + } + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { + t.Fatalf("ResumeActor failed: %v", err) + } + pauseResp, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: actorRef, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, + }) + if err != nil { + t.Fatalf("SuspendActor failed: %v", err) + } + paused := localSnapshot(pauseResp.GetActor().GetStatus()) + if paused == nil { + t.Fatalf("pause left no local snapshot: %v", pauseResp.GetActor().GetStatus()) + } + waitForWorkerAvailable(t, tc, workerPod) + + // Finish the upload before changing the worker's runtime. + upload := &ateapipb.SnapshotReport{ + Uuid: paused.GetUuid(), + Generation: paused.GetGeneration(), + Fidelity: paused.GetFidelity(), + Object: paused.GetDurableSnapshot().GetObject(), + } + if err := tc.service.ApplyActorSnapshotState(ctx, resources.ActorRefFromActor(pauseResp.GetActor()), pauseResp.GetActor().GetMetadata().GetUid(), "node1", &ateapipb.ActorSnapshotState{ + LatestLocal: upload, + LatestDurable: upload, + }); err != nil { + t.Fatalf("ApplyActorSnapshotState(upload) failed: %v", err) + } + reportWorkerRuntime(t, tc, workerPod, rtB) + + // Interrupt restore after detach and worker assignment. + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.FailRestore = status.Error(codes.Unavailable, "restore reply lost") + tc.fakeAtelet.Lock.Unlock() + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); status.Code(err) != codes.Unavailable { + t.Fatalf("first ResumeActor status code = %v, want Unavailable (err: %v)", status.Code(err), err) + } + interrupted, err := tc.persistence.GetActor(ctx, resources.ActorRefFromActor(pauseResp.GetActor())) + if err != nil { + t.Fatalf("GetActor after the interrupted resume: %v", err) + } + if got := interrupted.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RESUMING { + t.Fatalf("state after the interrupted resume = %v, want RESUMING", got) + } + if got := interrupted.GetStatus().GetWorkerAssignment().GetWorkerPod(); got != "worker-1" { + t.Fatalf("assigned worker after the interrupted resume = %q, want worker-1", got) + } + if crash := interrupted.GetStatus().GetCrash(); crash != nil { + t.Fatalf("crash recorded after the interrupted resume: %v", crash) + } + + // Re-entry keeps the assignment and restores the same snapshot. + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.FailRestore = nil + tc.fakeAtelet.Lock.Unlock() + resumed, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}) + if err != nil { + t.Fatalf("retried ResumeActor failed: %v", err) + } + if got := resumed.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RUNNING { + t.Errorf("state after the retry = %v, want RUNNING", got) + } + if got := resumed.GetActor().GetStatus().GetWorkerAssignment().GetWorkerPod(); got != "worker-1" { + t.Errorf("retried resume moved to worker %q, want the kept assignment on worker-1", got) + } + restoreReq := tc.fakeAtelet.lastRestoreRequest() + if got := restoreReq.GetSnapshot().GetSnapshotUuid(); got != paused.GetUuid() { + t.Errorf("RestoreRequest.SnapshotUuid = %q, want the pause checkpoint's %q", got, paused.GetUuid()) + } + if got := restoreReq.GetSnapshot().GetObject().GetSnapshotUri(); got != paused.GetDurableSnapshot().GetObject().GetSnapshotUri() { + t.Errorf("RestoreRequest.SnapshotUri = %q, want the pause checkpoint's upload %q", got, paused.GetDurableSnapshot().GetObject().GetSnapshotUri()) + } + if got := restoreReq.GetSnapshot().GetFidelity(); got != ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS { + t.Errorf("RestoreRequest.Fidelity = %v, want ROOTFS (assigned worker cannot run the checkpoint's runtime)", got) + } +} + // TestSuspendActorAsynchronous tests the full workflow of suspending a running // actor asynchronously. // Workflow: @@ -4327,9 +4977,9 @@ func TestSuspendActorAsynchronous(t *testing.T) { } // TestResumeActor_PausedLocalSnapshotMissing_Crashes tests that if the local checkpoint -// files on the worker node are missing (e.g. node reboot, /tmp wipe) when resuming a paused actor, -// atelet returns a terminal file system error and ateapi marks the actor ACTOR_STATE_CRASHED -// while releasing the assigned worker pod. +// files on the worker node are missing (e.g. node reboot, /tmp wipe) when resuming a +// suspended actor from its local snapshot, atelet returns a terminal file system error +// and ateapi marks the actor ACTOR_STATE_CRASHED while releasing the assigned worker pod. func TestResumeActor_PausedLocalSnapshotMissing_Crashes(t *testing.T) { ns := namespaceForTest("ns-resume-paused-missing") tc := setupTest(t, ns) @@ -4944,8 +5594,8 @@ func TestSuspendActor_DanglingWorker(t *testing.T) { } // TestSuspendActor_FromPaused verifies SuspendActor, in either mode, returns a -// paused actor as it is: atelet is not called, and the actor keeps its -// node-local snapshot and stays pinned to its node. +// suspended actor as it is: atelet is not called, and the actor keeps its +// node-local snapshot on its node. func TestSuspendActor_FromPaused(t *testing.T) { tests := []struct { name string @@ -4986,8 +5636,8 @@ func TestSuspendActor_FromPaused(t *testing.T) { if localSnapshot(paused.GetStatus()) == nil { t.Fatal("paused actor has no local snapshot") } - // Drop the pause's Checkpoint call so the suspend's atelet traffic is - // observable in isolation. + // Drop the first suspend's Checkpoint call so the second suspend's + // atelet traffic is observable in isolation. tc.fakeAtelet.Reset() suspended, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: ref, Options: tt.opts}) @@ -5203,8 +5853,8 @@ func TestSuspendActor_SnapshotConflictPreservesState(t *testing.T) { } } -// TestCheckpointFailureCrashes: an atelet Checkpoint error while pausing or -// suspending a running actor crashes it, records atelet's error text, and +// TestCheckpointFailureCrashes: an atelet Checkpoint error while suspending a +// running actor (in either mode) crashes it, records atelet's error text, and // releases its worker. func TestCheckpointFailureCrashes(t *testing.T) { tests := []struct { @@ -5284,9 +5934,9 @@ func TestCheckpointFailureCrashes(t *testing.T) { } // TestResumeActor_RelocatesAfterDetach covers the capacity-recovery flow: a -// paused actor is pinned to the node holding its local snapshot, so it cannot -// resume while that node is full. Once atelet reports the snapshot uploaded and -// detached from the node, the pin is dropped and the actor is scheduled onto a +// suspended actor with a local snapshot is placed on the node holding its +// local snapshot. Once atelet reports the snapshot uploaded and detached from +// the node, the node placement is cleared and the actor is scheduled onto a // worker on a different node. func TestResumeActor_RelocatesAfterDetach(t *testing.T) { ns := namespaceForTest("ns-resume-relocate") @@ -5306,8 +5956,9 @@ func TestResumeActor_RelocatesAfterDetach(t *testing.T) { } } - // The actor under test runs on node1's only worker, then pauses — which - // frees that worker but pins the actor to node1 via LocalSnapshot. + // The actor under test runs on node1's only worker, then suspends + // asynchronously — which frees that worker but pins the actor to node1 via + // the local snapshot. if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: pinned}, }); err != nil { @@ -5345,7 +5996,7 @@ func TestResumeActor_RelocatesAfterDetach(t *testing.T) { createWorkerPod(t, tc, ns, "worker-2", "node2", "pool1") setupAteletOnNode(t, tc, "atelet-node2", "node2") - // node1's atelet reports the paused snapshot uploaded and detached: a report + // node1's atelet reports the local snapshot uploaded and detached: a report // with only latest_durable. ls := localSnapshot(paused.GetStatus()) if err := tc.service.ApplyActorSnapshotState(context.Background(), resources.ActorRefFromActor(paused), paused.GetMetadata().GetUid(), "node1", &ateapipb.ActorSnapshotState{ @@ -5390,7 +6041,7 @@ func TestResumeActor_RelocatesAfterDetach(t *testing.T) { } } -// TestResumeActor_DetachesFromFullNode resumes a paused actor whose node is +// TestResumeActor_DetachesFromFullNode resumes a suspended actor whose node is // full while another has room: it is detached and restored on the other node. func TestResumeActor_DetachesFromFullNode(t *testing.T) { ns := namespaceForTest("ns-resume-detach") @@ -5464,7 +6115,7 @@ func TestResumeActor_DetachesFromFullNode(t *testing.T) { } // TestLifecycleOpPoolAttributesOnSuccess is the regression test for #957: a -// successful suspend and pause must stamp the pool they ran on. Both recorded +// successful sync or async suspend must stamp the pool it ran on. Both recorded // the histogram from a defer that read the finalized record, whose assignment // the finalize step had already cleared, so the pair landed only on failures. func TestLifecycleOpPoolAttributesOnSuccess(t *testing.T) { @@ -5794,8 +6445,115 @@ func TestRevertActor(t *testing.T) { } } -// TestRevertActor_FromPaused reverts a paused actor back to its external -// snapshot, discarding the local pause snapshot info on the actor record. +func TestRevertActor_UnreachableNode(t *testing.T) { + const sourceNode = "node-revert-source" + const sourceAtelet = "atelet-revert-source" + for _, tt := range []struct { + name string + missing bool + pending bool + }{ + {name: "missing atelet", missing: true}, + {name: "unavailable atelet"}, + {name: "pending checkpoint remains pinned", pending: true}, + } { + t.Run(tt.name, func(t *testing.T) { + ns := namespaceForTest("ns-revert-unreachable") + tc := setupTest(t, ns) + t.Cleanup(tc.cleanup) + ctx := context.Background() + createTemplate(t, tc, ns) + worker := createWorkerPod(t, tc, ns, "worker-1", sourceNode, "pool1") + setupAteletOnNode(t, tc, sourceAtelet, sourceNode) + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.KeepCheckpointLocal = true + tc.fakeAtelet.Lock.Unlock() + ref := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-1"} + if _, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: ref.GetName()}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatal(err) + } + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: ref}); err != nil { + t.Fatal(err) + } + mode := ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS + if tt.pending { + mode = ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS + } + suspended, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: ref, Options: &ateapipb.SuspendOptions{Mode: mode}, + }) + if err != nil { + t.Fatal(err) + } + snap := proto.CloneOf(localSnapshot(suspended.GetActor().GetStatus())) + if snap == nil || suspended.GetActor().GetStatus().GetAssignedNode() != sourceNode { + t.Fatalf("suspended actor has no retained checkpoint: %v", suspended.GetActor().GetStatus()) + } + waitForWorkerAvailable(t, tc, worker) + deleteWorkerPod(t, tc, ns, "worker-1") + if tt.missing { + if err := tc.k8sClient.CoreV1().Pods(installdefaults.SystemNamespace).Delete(ctx, sourceAtelet, metav1.DeleteOptions{GracePeriodSeconds: ptr.To[int64](0)}); err != nil { + t.Fatal(err) + } + if err := wait.PollUntilContextTimeout(ctx, 10*time.Millisecond, 5*time.Second, true, func(context.Context) (bool, error) { + pods, err := tc.ateletIndexer.ByIndex(byNode, sourceNode) + return len(pods) == 0, err + }); err != nil { + t.Fatal(err) + } + } else { + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.FailTerminate = status.Error(codes.Unavailable, "node unavailable") + tc.fakeAtelet.Lock.Unlock() + } + reverted, err := tc.client.RevertActor(ctx, &ateapipb.RevertActorRequest{Actor: ref}) + if tt.pending { + if status.Code(err) != codes.Unavailable { + t.Fatalf("RevertActor = %v, want Unavailable", err) + } + stored, err := tc.client.GetActor(ctx, &ateapipb.GetActorRequest{Actor: ref}) + if err != nil { + t.Fatal(err) + } + if stored.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_REVERTING || + stored.GetStatus().GetAssignedNode() != sourceNode || !proto.Equal(localSnapshot(stored.GetStatus()), snap) { + t.Fatalf("pending checkpoint changed: %v", stored.GetStatus()) + } + return + } + if err != nil { + t.Fatalf("RevertActor on unreachable node: %v", err) + } + got := reverted.GetActor().GetStatus() + if got.GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED || got.GetAssignedNode() != "" || + got.GetWorkerAssignment() != nil || localSnapshot(got) != nil { + t.Fatalf("reverted status = %v, want unbound SUSPENDED", got) + } + createWorkerPod(t, tc, ns, "worker-2", "node2", "pool1") + setupAteletOnNode(t, tc, "atelet-node2", "node2") + resumed, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: ref}) + if err != nil { + t.Fatalf("ResumeActor after revert: %v", err) + } + if resumed.GetActor().GetStatus().GetAssignedNode() != "node2" { + t.Fatalf("resumed on wrong node: %v", resumed.GetActor().GetStatus()) + } + restore := tc.fakeAtelet.lastRestoreRequest().GetSnapshot() + if restore.GetSnapshotUuid() != snap.GetUuid() || restore.GetGeneration() != snap.GetGeneration() || + restore.GetObject().GetSnapshotUri() != snap.GetDurableSnapshot().GetObject().GetSnapshotUri() { + t.Fatalf("restored snapshot = %v, want retained checkpoint %v", restore, snap) + } + assertSnapshotPresent(t, tc, snap.GetDurableSnapshot().GetObject().GetSnapshotUri()) + }) + } +} + +// TestRevertActor_FromPaused reverts a suspended actor with a local snapshot +// back to its external snapshot, discarding the local snapshot info on the +// actor record. func TestRevertActor_FromPaused(t *testing.T) { ns := namespaceForTest("ns-revert-paused") tc := setupTest(t, ns) @@ -6213,20 +6971,17 @@ func TestDeleteActor_Preconditions(t *testing.T) { } // reportWorkerRuntime writes defaultRT and restorableRTs to workerName's status -// and waits for the worker cache to observe the update. -func reportWorkerRuntime(t *testing.T, tc *testContext, workerName string, defaultRT *ateapipb.SandboxRuntime, restorableRTs ...*ateapipb.SandboxRuntime) { +// as its one sandbox class and waits for the worker cache to observe the update. +func reportWorkerRuntime(t *testing.T, tc *testContext, workerName string, defaultRT *ateapipb.ActorSandboxRuntime, restorableRTs ...*ateapipb.ActorSandboxRuntime) { t.Helper() ctx := context.Background() cur, err := tc.persistence.GetWorker(ctx, workerName) if err != nil { t.Fatalf("GetWorker(%s): %v", workerName, err) } + want := workerSandboxClasses(defaultRT, restorableRTs...) if _, err := tc.persistence.UpdateWorker(ctx, workerName, store.PreconditionFrom(cur), func(toUpdate *ateapipb.Worker) error { - toUpdate.Status.DefaultRuntime = proto.Clone(defaultRT).(*ateapipb.SandboxRuntime) - toUpdate.Status.RestorableRuntimes = nil - for _, r := range restorableRTs { - toUpdate.Status.RestorableRuntimes = append(toUpdate.Status.RestorableRuntimes, proto.Clone(r).(*ateapipb.SandboxRuntime)) - } + toUpdate.Status.SandboxClasses = workerSandboxClasses(defaultRT, restorableRTs...) return nil }); err != nil { t.Fatalf("UpdateWorker(%s): %v", workerName, err) @@ -6234,8 +6989,7 @@ func reportWorkerRuntime(t *testing.T, tc *testContext, workerName string, defau if err := wait.PollUntilContextTimeout(ctx, 10*time.Millisecond, 5*time.Second, true, func(context.Context) (bool, error) { got, err := tc.workerCache.Worker(workerName) return err == nil && - proto.Equal(got.GetStatus().GetDefaultRuntime(), defaultRT) && - slices.EqualFunc(got.GetStatus().GetRestorableRuntimes(), restorableRTs, func(a, b *ateapipb.SandboxRuntime) bool { return proto.Equal(a, b) }), nil + slices.EqualFunc(got.GetStatus().GetSandboxClasses(), want, func(a, b *ateapipb.WorkerSandboxClass) bool { return proto.Equal(a, b) }), nil }); err != nil { t.Fatalf("worker %s runtime never reached worker cache: %v", workerName, err) } @@ -6255,7 +7009,8 @@ func reportWorkerRuntime(t *testing.T, tc *testContext, workerName string, defau // 4. When the other worker advertises the snapshot's runtime in // restorable_runtimes, ResumeActor restores at full MEMORY fidelity, // records the matched restorable runtime on WorkerAssignment, and preserves -// that stamp through async SuspendActor and sync SuspendActor-from-PAUSED. +// that stamp through async SuspendActor and sync SuspendActor on an +// already-SUSPENDED actor. func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { ns := namespaceForTest("ns-runtime-fallback") tc := setupTest(t, ns) @@ -6264,7 +7019,7 @@ func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { ctx := context.Background() tmpl := createTemplate(t, tc, ns) - rtA := &ateapipb.SandboxRuntime{ + rtA := &ateapipb.ActorSandboxRuntime{ SandboxClass: "gvisor", Version: &ateapipb.VersionedSandboxCompat{ SchemaVersion: "v1", @@ -6274,7 +7029,7 @@ func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { }, }, } - rtB := &ateapipb.SandboxRuntime{ + rtB := &ateapipb.ActorSandboxRuntime{ SandboxClass: "gvisor", Version: &ateapipb.VersionedSandboxCompat{ SchemaVersion: "v1", @@ -6401,7 +7156,7 @@ func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { // worker-a is full and it falls back onto worker-b, restores the golden at // ROOTFS fidelity like any other durable snapshot rather than cold-booting. goldenSnap := newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), goldenSnapshotName, goldenSnapshotURI(t)) - goldenSnap.SandboxRuntime = proto.Clone(rtA).(*ateapipb.SandboxRuntime) + goldenSnap.SandboxRuntime = proto.Clone(rtA).(*ateapipb.ActorSandboxRuntime) stampedGoldenTag, err := tc.persistence.CreateTag(ctx, &ateapipb.Tag{ Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "golden-rt-a"}, SourceActor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: tmpl.GetMetadata().GetUid()}, @@ -6466,12 +7221,18 @@ func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { if got, want := goldenRestore.GetSnapshot().GetObject().GetSnapshotUri(), goldenSnapshotURI(t); got != want { t.Fatalf("golden-actor RestoreRequest.SnapshotUri = %q, want golden %q", got, want) } + // A borrowed golden snapshot is shared by every actor created from it, + // and the request must say so: atelet caches only declared-SHARED sources. + if got := goldenRestore.GetSnapshot().GetObject().GetSharing(); got != ateletpb.SnapshotSharing_SNAPSHOT_SHARING_SHARED { + t.Fatalf("golden-actor RestoreRequest.Sharing = %v, want SHARED", got) + } // Free worker-b and have it advertise rtA in restorable_runtimes while // keeping rtB as its default. Resuming an actor whose snapshot requires rtA // onto worker-b (with worker-a still full) must restore at MEMORY fidelity, // record the matched restorable entry (rtA, not rtB) on WorkerAssignment, - // and preserve rtA through async SuspendActor and sync SuspendActor-from-PAUSED. + // and preserve rtA through async SuspendActor and sync SuspendActor on the + // already-SUSPENDED actor. if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: goldenActorRef}); err != nil { t.Fatalf("SuspendActor(golden-actor) failed: %v", err) } @@ -6509,8 +7270,8 @@ func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { t.Fatalf("paused local Snapshot.SandboxRuntime = %v, want %v", got, rtA) } - // A sync suspend of the paused actor returns it as it is: the node-local - // snapshot keeps its runtime stamp. + // A sync suspend of the already-SUSPENDED actor returns it as it is: the + // node-local snapshot keeps its runtime stamp. suspendedFromPaused, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: restorableRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) if err != nil { t.Fatalf("SuspendActor from PAUSED(restorable-actor) failed: %v", err) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_volume_detach_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_volume_detach_test.go new file mode 100644 index 0000000000..e12b10fb75 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_volume_detach_test.go @@ -0,0 +1,318 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package functionaltest + +import ( + "context" + "slices" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/volume" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// calls returns copies of the nodes the plugin attached to and detached from, +// in call order. +func (a *attachFailVolumePlugin) calls() (attached, detached []string) { + a.mu.Lock() + defer a.mu.Unlock() + return slices.Clone(a.attachedNodes), slices.Clone(a.detachedNodes) +} + +// assertAssignedNode checks the actor's persisted assigned_node in the store +// against want. +func assertAssignedNode(t *testing.T, tc *testContext, name, step, want string) { + t.Helper() + actor, err := tc.persistence.GetActor(context.Background(), resources.ActorRef{Atespace: testAtespace, Name: name}) + if err != nil { + t.Fatalf("GetActor(%s) from store: %v", name, err) + } + if got := actor.GetStatus().GetAssignedNode(); got != want { + t.Errorf("persisted assigned_node %s = %q, want %q", step, got, want) + } +} + +// assertDetachedNodes checks the nodes the plugin detached from, in call +// order. An empty want means no detach. +func assertDetachedNodes(t *testing.T, plugin *attachFailVolumePlugin, step string, want ...string) { + t.Helper() + _, detached := plugin.calls() + if diff := cmp.Diff(want, detached, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("detached nodes %s (-want +got):\n%s", step, diff) + } +} + +// createSingleVolumeActor creates a template mounting one external volume, +// worker-1 on node1, and a SUSPENDED actor named name from the template. It +// returns worker-1's Worker name. +func createSingleVolumeActor(t *testing.T, tc *testContext, ns, name string) string { + t.Helper() + preview.InitForTest(t, "ExternalVolumes") + volumes := []*ateapipb.Volume{{ + Name: "vol1", + External: &ateapipb.ExternalVolumeSource{Template: &ateapipb.ExternalVolumeTemplate{ + StorageClassName: "standard", + CapacityBytes: 10 << 30, + }}, + }} + mounts := []*ateapipb.VolumeMount{{Name: "vol1", MountPath: "/mnt/vol1"}} + createTemplateWithVolumes(t, tc, ns, volumes, mounts) + workerName := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + + if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor failed: %v", err) + } + return workerName +} + +// resumeToRunningOnNode1 resumes the actor and checks that its volume was +// published to node1 and that the store records node1 as its assigned_node. +func resumeToRunningOnNode1(t *testing.T, tc *testContext, plugin *attachFailVolumePlugin, name string) { + t.Helper() + if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + }); err != nil { + t.Fatalf("ResumeActor failed: %v", err) + } + if attached, _ := plugin.calls(); !cmp.Equal(attached, []string{"node1"}) { + t.Fatalf("attached nodes after resume = %v, want [node1]", attached) + } + assertAssignedNode(t, tc, name, "after resume", "node1") +} + +// assertCrashedKeepingAssignedNode checks that the actor is CRASHED with no +// worker assignment, and that its status still records node as assigned_node. +func assertCrashedKeepingAssignedNode(t *testing.T, tc *testContext, name, node string) { + t.Helper() + actor, err := tc.persistence.GetActor(context.Background(), resources.ActorRef{Atespace: testAtespace, Name: name}) + if err != nil { + t.Fatalf("GetActor(%s) from store: %v", name, err) + } + if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_CRASHED { + t.Fatalf("state = %v, want CRASHED", got) + } + if got := actor.GetStatus().GetWorkerAssignment(); got != nil { + t.Fatalf("worker assignment = %v, want nil after the crash", got) + } + assertAssignedNode(t, tc, name, "after the crash", node) +} + +// TestDeleteActor_CrashedWorkerGone_DetachesAssignedNode verifies that +// deleting an actor that crashed because its worker went away unpublishes its +// volume from the node recorded in assigned_node, although the crash cleared +// the worker assignment and the worker no longer exists. +func TestDeleteActor_CrashedWorkerGone_DetachesAssignedNode(t *testing.T) { + ns := namespaceForTest("ns-del-crashed-detach") + plugin := &attachFailVolumePlugin{} + tc := setupTestWithVolumePlugins(t, ns, map[string]volume.VolumePluginControlPlane{ + "substrate.io/mock": plugin, + }) + defer tc.cleanup() + + const name = "crashed-delete-actor" + createSingleVolumeActor(t, tc, ns, name) + resumeToRunningOnNode1(t, tc, plugin, name) + + // Deregistering the worker crashes the actor it hosts. + deleteWorkerPod(t, tc, ns, "worker-1") + assertCrashedKeepingAssignedNode(t, tc, name, "node1") + assertDetachedNodes(t, plugin, "after the crash") + + // CRASHED is deletable without AnyState. + if _, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + }); err != nil { + t.Fatalf("DeleteActor failed: %v", err) + } + assertDetachedNodes(t, plugin, "after delete", "node1") + plugin.mu.Lock() + if len(plugin.deleted) != 1 { + t.Errorf("deleted volumes = %v, want exactly one", plugin.deleted) + } + plugin.mu.Unlock() +} + +// TestResumeActor_RestoreCrash_DeleteDetaches verifies that when Restore fails +// with an error a retry cannot clear after the volume was published, the +// crashed actor keeps assigned_node on record and a later delete unpublishes +// the volume from it. +func TestResumeActor_RestoreCrash_DeleteDetaches(t *testing.T) { + ns := namespaceForTest("ns-restore-crash-detach") + plugin := &attachFailVolumePlugin{} + tc := setupTestWithVolumePlugins(t, ns, map[string]volume.VolumePluginControlPlane{ + "substrate.io/mock": plugin, + }) + defer tc.cleanup() + + const name = "restore-crash-actor" + createSingleVolumeActor(t, tc, ns, name) + tc.fakeAtelet.FailRestore = status.Error(codes.Internal, "injected restore failure") + + _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + }) + if got := status.Code(err); got != codes.Internal { + t.Fatalf("ResumeActor code = %v, want %v (err: %v)", got, codes.Internal, err) + } + if attached, _ := plugin.calls(); !cmp.Equal(attached, []string{"node1"}) { + t.Fatalf("attached nodes = %v, want [node1]: the volume must be published before Restore fails", attached) + } + assertCrashedKeepingAssignedNode(t, tc, name, "node1") + assertDetachedNodes(t, plugin, "after the crash") + + if _, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + }); err != nil { + t.Fatalf("DeleteActor failed: %v", err) + } + assertDetachedNodes(t, plugin, "after delete", "node1") +} + +// TestSuspendActor_WorkerRowGone_StillDetaches verifies that a suspend +// unpublishes the volume from the node recorded on the actor even when the +// worker row the assignment names is already gone, and clears assigned_node +// afterwards. +func TestSuspendActor_WorkerRowGone_StillDetaches(t *testing.T) { + ns := namespaceForTest("ns-suspend-worker-gone") + plugin := &attachFailVolumePlugin{} + tc := setupTestWithVolumePlugins(t, ns, map[string]volume.VolumePluginControlPlane{ + "substrate.io/mock": plugin, + }) + defer tc.cleanup() + + const name = "worker-gone-actor" + workerName := createSingleVolumeActor(t, tc, ns, name) + resumeToRunningOnNode1(t, tc, plugin, name) + + // Remove the worker row directly, bypassing the DeleteWorker workflow, so + // the actor stays RUNNING with an assignment to a worker that no longer + // exists. + if _, err := tc.persistence.DeleteWorker(context.Background(), workerName, store.DeletePreconditions{}); err != nil { + t.Fatalf("DeleteWorker(%s) from store: %v", workerName, err) + } + + suspended, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, + }) + if err != nil { + t.Fatalf("SuspendActor failed: %v", err) + } + if got := suspended.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + t.Errorf("state = %v, want SUSPENDED", got) + } + assertDetachedNodes(t, plugin, "after suspend", "node1") + assertAssignedNode(t, tc, name, "after suspend", "") +} + +// TestRevertActor_Crashed_DetachesAssignedNode verifies that reverting an +// actor that crashed because its worker went away unpublishes its volume from +// the node recorded in assigned_node, clears assigned_node, and leaves the +// actor SUSPENDED. +func TestRevertActor_Crashed_DetachesAssignedNode(t *testing.T) { + ns := namespaceForTest("ns-revert-crashed-detach") + plugin := &attachFailVolumePlugin{} + tc := setupTestWithVolumePlugins(t, ns, map[string]volume.VolumePluginControlPlane{ + "substrate.io/mock": plugin, + }) + defer tc.cleanup() + + const name = "crashed-revert-actor" + createSingleVolumeActor(t, tc, ns, name) + resumeToRunningOnNode1(t, tc, plugin, name) + + deleteWorkerPod(t, tc, ns, "worker-1") + assertCrashedKeepingAssignedNode(t, tc, name, "node1") + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.TerminateCalled = false + tc.fakeAtelet.Lock.Unlock() + + reverted, err := tc.client.RevertActor(context.Background(), &ateapipb.RevertActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + }) + if err != nil { + t.Fatalf("RevertActor failed: %v", err) + } + if got := reverted.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + t.Errorf("state = %v, want SUSPENDED", got) + } + assertDetachedNodes(t, plugin, "after revert", "node1") + assertAssignedNode(t, tc, name, "after revert", "") + tc.fakeAtelet.Lock.Lock() + if !tc.fakeAtelet.TerminateCalled { + t.Errorf("expected Terminate call to clean up assigned node") + } else if gotUID := tc.fakeAtelet.TerminateRequest.GetWorkerPodUid(); gotUID != "" { + t.Errorf("TerminateRequest.WorkerPodUid = %q, want empty", gotUID) + } + tc.fakeAtelet.Lock.Unlock() +} + +// TestRevertActor_WorkerNoLongerHosts_Detaches verifies that reverting a +// RUNNING actor whose worker no longer hosts it skips the ateom workload +// termination (WorkerPodUid == "") but still unpublishes the volume from the +// node recorded on the actor and clears assigned_node. +func TestRevertActor_WorkerNoLongerHosts_Detaches(t *testing.T) { + ns := namespaceForTest("ns-revert-unhosted-detach") + plugin := &attachFailVolumePlugin{} + tc := setupTestWithVolumePlugins(t, ns, map[string]volume.VolumePluginControlPlane{ + "substrate.io/mock": plugin, + }) + defer tc.cleanup() + + const name = "unhosted-revert-actor" + workerName := createSingleVolumeActor(t, tc, ns, name) + resumeToRunningOnNode1(t, tc, plugin, name) + + actor, err := tc.persistence.GetActor(context.Background(), resources.ActorRef{Atespace: testAtespace, Name: name}) + if err != nil { + t.Fatalf("GetActor(%s) from store: %v", name, err) + } + // Drop the worker's assignment row only; the actor still names the worker. + if _, err := tc.persistence.ReleaseActorFromWorker(context.Background(), workerName, actor.GetMetadata().GetUid()); err != nil { + t.Fatalf("ReleaseActorFromWorker failed: %v", err) + } + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.TerminateCalled = false + tc.fakeAtelet.Lock.Unlock() + + reverted, err := tc.client.RevertActor(context.Background(), &ateapipb.RevertActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + }) + if err != nil { + t.Fatalf("RevertActor failed: %v", err) + } + if got := reverted.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + t.Errorf("state = %v, want SUSPENDED", got) + } + assertDetachedNodes(t, plugin, "after revert", "node1") + assertAssignedNode(t, tc, name, "after revert", "") + tc.fakeAtelet.Lock.Lock() + if !tc.fakeAtelet.TerminateCalled { + t.Errorf("expected Terminate call to clean up assigned node") + } else if gotUID := tc.fakeAtelet.TerminateRequest.GetWorkerPodUid(); gotUID != "" { + t.Errorf("TerminateRequest.WorkerPodUid = %q, want empty", gotUID) + } + tc.fakeAtelet.Lock.Unlock() +} diff --git a/cmd/ateapi/internal/controlapi/functionaltest/atespace_test.go b/cmd/ateapi/internal/controlapi/functionaltest/atespace_test.go index b682b50cd6..58a0636376 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/atespace_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/atespace_test.go @@ -152,6 +152,9 @@ func TestDeleteAtespace_Empty_Success(t *testing.T) { if got := deleted.GetMetadata().GetName(); got != "team-a" { t.Errorf("deleted atespace name = %q, want team-a", got) } + if deleted.GetMetadata().GetDeleteTime() == nil { + t.Errorf("the deleted atespace carries no delete_time") + } _, err = tc.client.GetAtespace(context.Background(), &ateapipb.GetAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}}) assertGrpcError(t, err, codes.NotFound, "Atespace team-a not found") diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index 7aef251900..b175b509bb 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -75,10 +75,10 @@ const ( var ( ignoreUID = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "uid") ignoreVersion = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "version") - ignoreTimestamps = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "create_time", "update_time") + ignoreTimestamps = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "create_time", "update_time", "delete_time") // ignoreServerMetadata skips the ResourceMetadata fields the store assigns. - ignoreServerMetadata = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "uid", "create_time", "update_time") + ignoreServerMetadata = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "uid", "create_time", "update_time", "delete_time") ) type testContext struct { @@ -532,8 +532,9 @@ func newDurableSnapshot(gen int32, owner ateapipb.SnapshotOwner, fidelity ateapi } } -// newPausedSnapshot is the snapshot a pause leaves on node locality: its -// durable copy, under the actor's own prefix, is PENDING until atelet uploads it. +// newPausedSnapshot is the snapshot an asynchronous suspend leaves on node +// locality: its durable copy, under the actor's own prefix, is PENDING until +// atelet uploads it. func newPausedSnapshot(t *testing.T, gen int32, fidelity ateapipb.SnapshotFidelity, templateUID, actorUID, uuid, locality string) *ateapipb.Snapshot { t.Helper() uri, err := resources.NewActorSnapshotURI(testStorageLocation, testAtespace, actorUID, uuid) @@ -692,7 +693,6 @@ func createWorkerPod(t *testing.T, tc *testContext, ns string, name string, node WorkerPodUid: string(createdPod.UID), Ips: []string{"127.0.0.1"}, NodeName: nodeName, - SandboxClass: string(pool.Spec.DefaultSandboxClass()), Labels: pool.GetLabels(), // Capacity is not settable here: a Worker gets it from its own // worker's report, which the reportWorkerCapacity below stands in diff --git a/cmd/ateapi/internal/controlapi/functionaltest/egress_policy_test.go b/cmd/ateapi/internal/controlapi/functionaltest/egress_policy_test.go index 470a53b499..1917594e52 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/egress_policy_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/egress_policy_test.go @@ -195,9 +195,12 @@ func TestDeleteActorEgressPolicy(t *testing.T) { if err != nil { t.Fatalf("DeleteActorEgressPolicy failed: %v", err) } - if diff := cmp.Diff(created, deleted, protocmp.Transform()); diff != "" { + if diff := cmp.Diff(created, deleted, protocmp.Transform(), protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "delete_time")); diff != "" { t.Errorf("deleted policy mismatch (-created +deleted):\n%s", diff) } + if deleted.GetMetadata().GetDeleteTime() == nil { + t.Errorf("the deleted policy carries no delete_time") + } _, err = tc.client.DeleteActorEgressPolicy(context.Background(), &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName}) assertGrpcError(t, err, codes.NotFound, "EgressPolicy not found") } diff --git a/cmd/ateapi/internal/controlapi/functionaltest/main_test.go b/cmd/ateapi/internal/controlapi/functionaltest/main_test.go index 939b736507..a40b5674d0 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/main_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/main_test.go @@ -122,9 +122,10 @@ type FakeAteletServer struct { RunRequest *ateletpb.RunRequest FailRun error - CheckpointCalled bool - CheckpointRequest *ateletpb.CheckpointRequest - FailCheckpoint error + CheckpointCalled bool + CheckpointRequest *ateletpb.CheckpointRequest + FailCheckpoint error + KeepCheckpointLocal bool // checkpointed is each actor's last checkpointed snapshot, by UID. checkpointed map[string]*ateletpb.Snapshot @@ -184,6 +185,7 @@ func (f *FakeAteletServer) Reset() { f.CheckpointRequest = nil f.FailCheckpoint = nil f.checkpointed = nil + f.KeepCheckpointLocal = false f.RestoreCalled = false f.RestoreRequest = nil @@ -282,6 +284,14 @@ func (f *FakeAteletServer) Checkpoint(ctx context.Context, req *ateletpb.Checkpo return nil, err } } + if f.KeepCheckpointLocal { + report := &ateletpb.SnapshotReport{Snapshot: proto.CloneOf(req.GetSnapshot())} + state := &ateletpb.ActorSnapshotState{LatestLocal: report} + if req.GetUploadConfig().GetMode() == ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE { + state.LatestDurable = proto.CloneOf(report) + } + return &ateletpb.CheckpointResponse{State: state}, nil + } return &ateletpb.CheckpointResponse{}, nil } diff --git a/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go b/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go index 9249c50888..05f9736e4d 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go @@ -151,9 +151,10 @@ func TestCreateTag_ReusedTagName(t *testing.T) { } } -// TestCreateTag_FromPausedActor verifies that tagging a paused actor has the -// atelet on its node upload the node-local snapshot first, and tags that -// snapshot. The actor keeps the snapshot on its node. +// TestCreateTag_FromPausedActor verifies that tagging a suspended actor whose +// local snapshot is not uploaded yet has the atelet on its node upload the +// node-local snapshot first, and tags that snapshot. The actor keeps the +// snapshot on its node. func TestCreateTag_FromPausedActor(t *testing.T) { ns := namespaceForTest("ns-tag-paused") tc := setupTest(t, ns) @@ -209,7 +210,7 @@ func TestCreateTag_FromPausedActor(t *testing.T) { if err != nil { t.Fatalf("GetActor failed: %v", err) } - // The paused snapshot is now durable and stays on its node. + // The node-local snapshot is now durable and stays on its node. if got := durableSnapshot(stored.GetStatus()); got.GetUuid() != pausedSnap.GetUuid() { t.Errorf("latest durable snapshot = %v, want the paused snapshot %q", got, pausedSnap.GetUuid()) } @@ -361,6 +362,36 @@ func TestUpdateTag_NotFound(t *testing.T) { assertGrpcError(t, err, codes.NotFound, "Tag test-atespace/does-not-exist not found") } +func TestDeleteTag(t *testing.T) { + ns := namespaceForTest("ns-delete-tag") + tc := setupTest(t, ns) + defer tc.cleanup() + ctx := context.Background() + createTemplate(t, tc, ns) + + created := seedTag(t, tc, "actor-1", "v1") + tagRef := resources.TagRefFromTag(created) + snapshotURI, err := resources.ParseSnapshotURI(tagSnapshotURI(created)) + if err != nil { + t.Fatalf("ParseSnapshotURI: %v", err) + } + tc.objectStore.PutSnapshot(t, snapshotURI, "manifest.json") + + deleted, err := tc.client.DeleteTag(ctx, &ateapipb.DeleteTagRequest{Tag: tagRef.ToObjectRef()}) + if err != nil { + t.Fatalf("DeleteTag failed: %v", err) + } + if diff := cmp.Diff(created, deleted, protocmp.Transform(), ignoreVersion, ignoreTimestamps); diff != "" { + t.Errorf("DeleteTag response mismatch (-created +deleted):\n%s", diff) + } + if deleted.GetMetadata().GetDeleteTime() == nil { + t.Errorf("the deleted tag carries no delete_time") + } + assertSnapshotCollected(t, tc, snapshotURI.String()) + _, err = tc.client.GetTag(ctx, &ateapipb.GetTagRequest{Tag: tagRef.ToObjectRef()}) + assertGrpcError(t, err, codes.NotFound, "Tag "+tagRef.String()+" not found") +} + func TestDeleteTag_NotFound(t *testing.T) { ns := namespaceForTest("ns-delete-tag-missing") tc := setupTest(t, ns) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go b/cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go index 9ac8e83c88..0acfe318f5 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go @@ -26,6 +26,7 @@ import ( "go.ate.dev/substrate/cmd/ateapi/internal/store" "go.ate.dev/substrate/internal/hardware" "go.ate.dev/substrate/pkg/proto/ateapipb" + "google.golang.org/protobuf/proto" ) // TestResumeActor_ConcurrentOntoOneWorker is the regression test for a refusal @@ -153,13 +154,13 @@ func setWorkerActorCapacity(t *testing.T, tc *testContext, pool string, actors i } } -// defaultWorkerRuntime returns the default SandboxRuntime that +// defaultWorkerRuntime returns the default ActorSandboxRuntime that // reportWorkerCapacity records on a test worker of the given sandbox class. -func defaultWorkerRuntime(class string) *ateapipb.SandboxRuntime { +func defaultWorkerRuntime(class string) *ateapipb.ActorSandboxRuntime { if class == "" { class = "gvisor" } - return &ateapipb.SandboxRuntime{ + return &ateapipb.ActorSandboxRuntime{ SandboxClass: class, Version: &ateapipb.VersionedSandboxCompat{ SchemaVersion: hardware.SchemaVersionV1, @@ -171,6 +172,19 @@ func defaultWorkerRuntime(class string) *ateapipb.SandboxRuntime { } } +// workerSandboxClasses is the sandbox_classes a worker reports to run def as +// its default runtime and restore restorable beyond it, all of def's class. +// The arguments are the ActorSandboxRuntimes a test also expects on snapshots and +// assignments, so both sides of a match derive from one value; the compats +// are copied so the stored row never aliases them. +func workerSandboxClasses(def *ateapipb.ActorSandboxRuntime, restorable ...*ateapipb.ActorSandboxRuntime) []*ateapipb.WorkerSandboxClass { + class := &ateapipb.WorkerSandboxClass{SandboxClass: def.GetSandboxClass(), DefaultRuntime: &ateapipb.WorkerSandboxRuntime{Version: proto.CloneOf(def.GetVersion())}} + for _, r := range restorable { + class.RestorableRuntimes = append(class.RestorableRuntimes, &ateapipb.WorkerSandboxRuntime{Version: proto.CloneOf(r.GetVersion())}) + } + return []*ateapipb.WorkerSandboxClass{class} +} + // reportWorkerCapacity stands in for the worker's capacity report, which atelet // forwards to WorkerService in a real cluster, with compute for actors of the // size test templates declare. It waits for the worker cache, which placement @@ -187,8 +201,8 @@ func reportWorkerCapacity(t *testing.T, tc *testContext, name string, actors int Actors: actors, CpuMillicores: int64(actors) * 1000, MemoryBytes: int64(actors) * 512 << 20, } - if toUpdate.Status.DefaultRuntime == nil { - toUpdate.Status.DefaultRuntime = defaultWorkerRuntime(toUpdate.GetSandboxClass()) + if len(toUpdate.Status.SandboxClasses) == 0 { + toUpdate.Status.SandboxClasses = workerSandboxClasses(defaultWorkerRuntime("gvisor")) } return nil }); err != nil { diff --git a/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go b/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go index 87573a4ea7..e51a7a0513 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go @@ -49,7 +49,6 @@ func newTestWorker(ns string) *ateapipb.Worker { WorkerPodUid: testWorkerName, NodeName: "node1", Ips: []string{"10.1.2.3"}, - SandboxClass: "gvisor", } } @@ -164,12 +163,11 @@ func TestListWorkers(t *testing.T) { WorkerPodUid: podUID, NodeName: "node1", Ips: []string{"127.0.0.1"}, - SandboxClass: "gvisor", Labels: map[string]string{"foo": "bar"}, Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1, CpuMillicores: 1000, MemoryBytes: 512 << 20}, - DefaultRuntime: defaultWorkerRuntime("gvisor"), + SandboxClasses: workerSandboxClasses(defaultWorkerRuntime("gvisor")), }, }, } @@ -316,9 +314,9 @@ func TestMarkWorkerAvailability(t *testing.T) { } // TestDeleteWorker deregisters a Worker and checks it is gone. The delete -// drains the Worker on its way out, so the record it returns is that one rather -// than the one Create stored. Deregistering is not silently idempotent, so a -// second attempt reports NOT_FOUND. +// marks and drains the Worker in one write on its way out, so the record it +// returns is one write past the one Create stored. Deregistering is not +// silently idempotent, so a second attempt reports NOT_FOUND. func TestDeleteWorker(t *testing.T) { ns := namespaceForTest("ns-worker-delete") tc := setupTest(t, ns) @@ -337,6 +335,9 @@ func TestDeleteWorker(t *testing.T) { if diff := cmp.Diff(want, deleted, protocmp.Transform(), ignoreServerMetadata); diff != "" { t.Errorf("DeleteWorker response mismatch (-want +got):\n%s", diff) } + if deleted.GetMetadata().GetDeleteTime() == nil { + t.Error("DeleteWorker returned a worker without delete_time") + } listed, err := tc.client.ListWorkers(ctx, &ateapipb.ListWorkersRequest{}) if err != nil { diff --git a/cmd/ateapi/internal/controlapi/hosting.go b/cmd/ateapi/internal/controlapi/hosting.go index b155c9a7f7..d93afee96f 100644 --- a/cmd/ateapi/internal/controlapi/hosting.go +++ b/cmd/ateapi/internal/controlapi/hosting.go @@ -27,7 +27,7 @@ import ( // HostingNode is the node that holds the actor, and "" when no node does. The // actor's state decides which status field to trust, since both can outlive -// the hosting they record: pause, suspend, crash and revert release the +// the hosting they record: suspend, crash and revert release the // worker (delete its row in the store's worker assignments) before they clear // status.worker_assignment, and a crash keeps status.assigned_node as a record // of the last node. The rules: diff --git a/cmd/ateapi/internal/controlapi/hosting_test.go b/cmd/ateapi/internal/controlapi/hosting_test.go index 54449ad009..a963441391 100644 --- a/cmd/ateapi/internal/controlapi/hosting_test.go +++ b/cmd/ateapi/internal/controlapi/hosting_test.go @@ -36,13 +36,13 @@ import ( // doing so until the workflow is retried. The worker's node must not count as // hosting the actor in that window, nor once the worker hosts another actor. func TestHostingNodeAfterWorkerRelease(t *testing.T) { - // The worker is the one wireTestAssignment names, so a pause or suspend + // The worker is the one wireTestAssignment names, so an async or sync suspend // reaches newWireCaptureWorkflow's fake atelet on its node. wa := wireTestAssignment() node := wa.GetNodeName() // wantFinalNode is HostingNode once the retry commits, by workflow name. wantFinalNode := map[string]string{ - // Paused: the node holding the local snapshot. + // Async suspend: the node holding the local snapshot. "pause": node, } for _, tc := range ReleaseWindowWorkflows() { @@ -59,8 +59,11 @@ func TestHostingNodeAfterWorkerRelease(t *testing.T) { WorkerPod: wa.GetWorkerPod(), WorkerPodUid: wa.GetWorkerPodUid(), NodeName: node, - SandboxClass: "gvisor", - Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}}, + Status: &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + SandboxClasses: []*ateapipb.WorkerSandboxClass{{SandboxClass: "gvisor"}}, + }, }); err != nil { t.Fatalf("CreateWorker: %v", err) } @@ -165,11 +168,10 @@ func TestHostingNodeAfterResumeOntoNewWorker(t *testing.T) { WorkerPod: "stale-pod", WorkerPodUid: testWorkerUID("stale-pod"), NodeName: "node1", - SandboxClass: "microvm", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, - DefaultRuntime: &ateapipb.SandboxRuntime{SandboxClass: "microvm", Version: testSandboxRuntime().GetVersion()}, + SandboxClasses: []*ateapipb.WorkerSandboxClass{{SandboxClass: "microvm", DefaultRuntime: &ateapipb.WorkerSandboxRuntime{Version: testSandboxRuntime().GetVersion()}}}, }, } free := &ateapipb.Worker{ @@ -179,12 +181,11 @@ func TestHostingNodeAfterResumeOntoNewWorker(t *testing.T) { WorkerPod: "free-pod", WorkerPodUid: testWorkerUID("free-pod"), NodeName: "node2", - SandboxClass: "gvisor", Labels: map[string]string{"pool": "b"}, Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, - DefaultRuntime: testSandboxRuntime(), + SandboxClasses: workerSandboxClasses(testSandboxRuntime()), }, } for _, w := range []*ateapipb.Worker{stale, free} { diff --git a/cmd/ateapi/internal/controlapi/metrics.go b/cmd/ateapi/internal/controlapi/metrics.go index f523743627..647045720d 100644 --- a/cmd/ateapi/internal/controlapi/metrics.go +++ b/cmd/ateapi/internal/controlapi/metrics.go @@ -21,6 +21,7 @@ import ( "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/hardware" atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel/attribute" @@ -98,11 +99,11 @@ func RegisterWorkerCount(meter metric.Meter, workers func() ([]*ateapipb.Worker, } for _, w := range ws { state := workerState(w) - // CreateWorker accepts an empty class, as the field is optional. - // Report it as unknown, not as the pool's class: the scheduler puts - // no actor on a worker that offers no class, thus the pool's series - // must not count it as free capacity. - class := ateattr.NormalizeSandboxClass(w.GetSandboxClass()) + // A Worker has no sandbox class until its worker registers with + // WorkerService. Report it as unknown, not as the pool's class: the + // scheduler puts no actor on a worker that offers no class, thus + // the pool's series must not count it as free capacity. + class := ateattr.NormalizeSandboxClass(hardware.SandboxClass(w.GetStatus())) tally[key{w.GetWorkerNamespace(), w.GetWorkerPool(), state, class}]++ } for k, n := range tally { @@ -201,13 +202,13 @@ func (i *Instruments) recordLifecycleOp(ctx context.Context, op string, start ti i.lifecycleOpDuration.Record(ctx, time.Since(start).Seconds(), metric.WithAttributes(attrs...)) } -// lifecycleOpAttrs builds the resume/suspend/pause/revert dimensions from +// lifecycleOpAttrs builds the resume/suspend/revert dimensions from // workflow state. Nil-safe, and omits the template, pool, sandbox-class, // snapshot-kind and snapshot-fidelity labels until they are known so a failure // before the load/assign/restore steps never emits an empty-string series. -// snapshotKind is empty for suspend/pause, which do not restore; snapshotScope -// applies to all three and separates a full restore from a data-only one of the -// same kind. +// snapshotKind is empty for suspend, which does not restore; snapshotScope +// applies to resume and suspend and separates a full restore from a data-only +// one of the same kind. // The pool keys are set together or not at all; see ateattr.WorkerPoolAttributes. func lifecycleOpAttrs(actor *ateapipb.Actor, snapshotKind, snapshotScope string) []attribute.KeyValue { var attrs []attribute.KeyValue diff --git a/cmd/ateapi/internal/controlapi/metrics_test.go b/cmd/ateapi/internal/controlapi/metrics_test.go index a2f271b574..3a68f8668e 100644 --- a/cmd/ateapi/internal/controlapi/metrics_test.go +++ b/cmd/ateapi/internal/controlapi/metrics_test.go @@ -76,16 +76,19 @@ func mustMetric(t *testing.T, reader *sdkmetric.ManualReader, name string) metri // worker builds an ACTIVE worker holding allocated of capacity actor slots. func worker(namespace, pool, class string, allocated, capacity int32) *ateapipb.Worker { - return &ateapipb.Worker{ + w := &ateapipb.Worker{ WorkerNamespace: namespace, WorkerPool: pool, - SandboxClass: class, Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: capacity}, Allocated: &ateapipb.WorkerResources{Actors: allocated}, }, } + if class != "" { + w.Status.SandboxClasses = []*ateapipb.WorkerSandboxClass{{SandboxClass: class}} + } + return w } type series struct{ namespace, pool, state, class string } @@ -245,7 +248,7 @@ func TestLifecycleOpDurationShape(t *testing.T) { WorkerAssignment: &ateapipb.WorkerAssignment{ WorkerNamespace: "ate-workers", WorkerPool: "pool-a", - SandboxRuntime: &ateapipb.SandboxRuntime{SandboxClass: "gvisor"}, + SandboxRuntime: &ateapipb.ActorSandboxRuntime{SandboxClass: "gvisor"}, }, }, } @@ -445,11 +448,11 @@ func TestWorkerCountSeedsZeroForKnownPools(t *testing.T) { } } -// TestWorkerCountEmptyClassWorker covers the workers that CreateWorker accepted -// with no sandbox class. They report as unknown, not as an empty-string class, -// and not as capacity of the pool they name: the scheduler puts no actor on -// them, thus the pool's own series stays at 0 and an idle==0 alert on it still -// fires. +// TestWorkerCountEmptyClassWorker covers the workers that have not yet +// reported default_runtime via RegisterWorker. They report as unknown, not as +// an empty-string class, and not as capacity of the pool they name: the +// scheduler puts no actor on them, thus the pool's own series stays at 0 and +// an idle==0 alert on it still fires. func TestWorkerCountEmptyClassWorker(t *testing.T) { const ( gvisor = string(atev1alpha1.SandboxClassGvisor) diff --git a/cmd/ateapi/internal/controlapi/mint_after_release_test.go b/cmd/ateapi/internal/controlapi/mint_after_release_test.go index 6ca42106cd..a33c8a59ba 100644 --- a/cmd/ateapi/internal/controlapi/mint_after_release_test.go +++ b/cmd/ateapi/internal/controlapi/mint_after_release_test.go @@ -45,7 +45,7 @@ import ( // window, nor once it hosts another actor or the status is finalized, and // another worker on the node never gets one. func TestMintWorkerActorCertificateAfterWorkerRelease(t *testing.T) { - // The worker is the one WireTestAssignment names, so a pause or suspend + // The worker is the one WireTestAssignment names, so an async or sync suspend // reaches the workflow's fake atelet on its node. wa := controlapi.WireTestAssignment() node := wa.GetNodeName() @@ -70,8 +70,11 @@ func TestMintWorkerActorCertificateAfterWorkerRelease(t *testing.T) { WorkerPod: w.pod, WorkerPodUid: w.podUID, NodeName: node, - SandboxClass: "gvisor", - Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}}, + Status: &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + SandboxClasses: []*ateapipb.WorkerSandboxClass{{SandboxClass: "gvisor"}}, + }, }); err != nil { t.Fatalf("CreateWorker(%s): %v", w.name, err) } @@ -165,7 +168,7 @@ func TestMintWorkerActorCertificateAfterWorkerRelease(t *testing.T) { if actor.GetStatus().GetState() != wf.WantFinalState { t.Errorf("actor state once finalized = %v, want %v", actor.GetStatus().GetState(), wf.WantFinalState) } - // No worker runs the actor once finalized, paused or not. + // No worker runs the actor once finalized, with or without a local snapshot. if got != codes.NotFound { t.Errorf("certificate mint once finalized: code %v, want NotFound", got) } diff --git a/cmd/ateapi/internal/controlapi/snapshot_helpers.go b/cmd/ateapi/internal/controlapi/snapshot_helpers.go index 6aa9977b98..45c027d7ed 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_helpers.go +++ b/cmd/ateapi/internal/controlapi/snapshot_helpers.go @@ -60,21 +60,14 @@ func findLatestDurableSnapshot(status *ateapipb.ActorStatus, storageStatus ateap }) } -// findLatestLocalSnapshot returns the highest-generation Snapshot on status -// that holds a completed local checkpoint (Locality != ""), or nil if none -// exists. +// findLatestLocalSnapshot returns the highest-generation snapshot with a local copy. func findLatestLocalSnapshot(status *ateapipb.ActorStatus) *ateapipb.Snapshot { return latestSnapshot(status, func(snap *ateapipb.Snapshot) bool { return snap.GetLocality() != "" }) } -// latestSnapshotIsLocal reports whether the snapshot at the actor's latest -// generation is node-local: the shape a pause leaves behind, which atelet's -// upload of the snapshot doesn't change. A paused actor is SUSPENDED with this -// shape, and resumes only on the node named by Locality. A RUNNING actor -// resumed from pause keeps the local snapshot too, so callers that need "at -// rest" check the state as well. +// latestSnapshotIsLocal reports whether the latest generation has a local copy. func latestSnapshotIsLocal(status *ateapipb.ActorStatus) bool { return snapshotAtLatestGeneration(status).GetLocality() != "" } @@ -98,13 +91,14 @@ func clearLocalSnapshots(status *ateapipb.ActorStatus) { // beforehand. // // Dropping PENDING durable snapshots is safe because: -// - On pause finalization, the new snapshot is kept with its PENDING copy. -// An earlier pause's PENDING copy is dropped: whatever atelet uploaded of -// it stays under the actor's prefix until DeleteActor collects it. +// - On asynchronous suspend finalization, the new snapshot is kept with its +// PENDING copy. An earlier asynchronous suspend's PENDING copy is dropped: +// whatever atelet uploaded of it stays under the actor's prefix until +// DeleteActor collects it. // - On revert finalization, ensureInProgressSnapshotDiscarded has already // deleted the objects of every PENDING durable snapshot. -// - On suspend finalization, the latest snapshot has already been marked -// COMPLETED. +// - On synchronous suspend finalization, the latest snapshot has already been +// marked COMPLETED. // // status.LastAssignedGeneration is not decreased so that we can keep track of // stale snapshots that need to be cleaned up and discarded generation numbers diff --git a/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go b/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go index 207559c4d0..2f466f6329 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go +++ b/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go @@ -74,7 +74,7 @@ func TestFindLatestDurableAndLocalSnapshots(t *testing.T) { t.Errorf("findLatestDurableSnapshot(COMPLETED) = %v, want gen 2 (gs://b/snap-2)", gotSnap) } - // A paused snapshot's durable copy is PENDING until atelet uploads it. + // An async suspend snapshot's durable copy is PENDING until atelet uploads it. gotSnap = findLatestDurableSnapshot(status, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) if gotSnap != s4 { t.Errorf("findLatestDurableSnapshot(PENDING) = %v, want gen 4", gotSnap) @@ -207,10 +207,9 @@ func TestSnapshotPruningHelpers(t *testing.T) { }) } -// TestLatestSnapshotIsLocal pins the shape a paused actor has: the latest -// snapshot is node-local, whether or not atelet has uploaded it. A latest -// snapshot with only a durable copy, or a local one not captured yet, does not -// count. +// TestLatestSnapshotIsLocal pins when the actor's latest snapshot is +// node-local, whether or not atelet has uploaded it. A latest snapshot with +// only a durable copy, or a local one not captured yet, does not count. func TestLatestSnapshotIsLocal(t *testing.T) { local := func(gen int32) *ateapipb.Snapshot { return newPausedSnapshot(t, gen, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local", "node-1") diff --git a/cmd/ateapi/internal/controlapi/snapshot_report.go b/cmd/ateapi/internal/controlapi/snapshot_report.go index 5f57ff26f4..4b5ef70f28 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_report.go +++ b/cmd/ateapi/internal/controlapi/snapshot_report.go @@ -164,8 +164,8 @@ func applyAteletResponseState(ctx context.Context, status *ateapipb.ActorStatus, // applyLatestDurable marks the reported durable snapshot COMPLETED, and records // its fidelity. Its uuid and upload URI must match the recorded ones, and its -// fidelity must be set. The snapshot keeps its locality, so a paused Actor -// stays on its node. +// fidelity must be set. The snapshot keeps its locality, so an Actor with a +// node-local snapshot stays on its node. // // A late report changes nothing and is logged at Info. A report is late when // its generation is older than the latest COMPLETED one, or is no longer in diff --git a/cmd/ateapi/internal/controlapi/snapshot_report_test.go b/cmd/ateapi/internal/controlapi/snapshot_report_test.go index 6921978722..e73398cf6c 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_report_test.go +++ b/cmd/ateapi/internal/controlapi/snapshot_report_test.go @@ -564,8 +564,8 @@ func TestApplyAteletResponseState(t *testing.T) { otherActorUID = "1a7e4c83-6d20-4f95-b3c8-9e0a2f6d4b17" ) - // previousPause is a pausing actor's status, holding the snapshot of the - // pause before, whose upload is pending. + // previousPause is a suspending actor's status, holding the snapshot of the + // async suspend before, whose upload is pending. previousPause := func() *ateapipb.ActorStatus { return &ateapipb.ActorStatus{State: suspending, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ withDurable(t, 1, reportNode, pending), diff --git a/cmd/ateapi/internal/controlapi/tag.go b/cmd/ateapi/internal/controlapi/tag.go index 4bff1c88f3..28d09dffe7 100644 --- a/cmd/ateapi/internal/controlapi/tag.go +++ b/cmd/ateapi/internal/controlapi/tag.go @@ -190,6 +190,10 @@ func (s *RPCService) DeleteTag(ctx context.Context, req *ateapipb.DeleteTagReque return s.actorWorkflow.DeleteTag(ctx, resources.TagRefFromObjectRef(req.GetTag()), toDeletePreconditions(req.GetOptions())) } +func (s *ServiceImpl) MarkTagForDeletion(ctx context.Context, tagRef resources.TagRef, precondition store.DeletePreconditions, mutate func(*ateapipb.Tag) error) (*ateapipb.Tag, error) { + return s.store.MarkTagForDeletion(ctx, tagRef, precondition, mutate) +} + func (s *ServiceImpl) DeleteTag(ctx context.Context, tagRef resources.TagRef, precondition store.DeletePreconditions) (*ateapipb.Tag, error) { // TODO: implement this return s.store.DeleteTag(ctx, tagRef, precondition) diff --git a/cmd/ateapi/internal/controlapi/tag_test.go b/cmd/ateapi/internal/controlapi/tag_test.go index 347dc3f65f..705842b836 100644 --- a/cmd/ateapi/internal/controlapi/tag_test.go +++ b/cmd/ateapi/internal/controlapi/tag_test.go @@ -324,7 +324,11 @@ func TestUpdateTag_DeleteRecreateRace(t *testing.T) { racing := &conflictInjectingStore{ Interface: persistence, inject: func() { - if _, err := persistence.DeleteTag(ctx, resources.TagRef{Atespace: testAtespace, Name: tagName}, store.DeletePreconditions{}); err != nil { + tagRef := resources.TagRef{Atespace: testAtespace, Name: tagName} + if _, err := persistence.MarkTagForDeletion(ctx, tagRef, store.DeletePreconditions{}, nil); err != nil { + t.Fatalf("Racing writer: MarkTagForDeletion: %v", err) + } + if _, err := persistence.DeleteTag(ctx, tagRef, store.DeletePreconditions{}); err != nil { t.Fatalf("Racing writer: DeleteTag: %v", err) } recreatedTag = storetest.MustCreateTag(t, ctx, persistence, newTestTag(t, tagName, actorTwo)) diff --git a/cmd/ateapi/internal/controlapi/template_reconciler.go b/cmd/ateapi/internal/controlapi/template_reconciler.go index 350f6954ef..891128b0dd 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler.go @@ -27,6 +27,7 @@ import ( "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/workqueue" @@ -46,16 +47,19 @@ const ( ) const ( - reasonGoldenTagConflict = "GoldenTagConflict" - reasonGoldenActorInvalid = "GoldenActorInvalid" - reasonGoldenActorCrashed = "GoldenActorCrashed" - reasonUnexpectedState = "GoldenActorUnexpectedState" + reasonGoldenTagConflict = "GoldenTagConflict" + reasonGoldenEgressPolicyConflict = "GoldenEgressPolicyConflict" + reasonGoldenActorInvalid = "GoldenActorInvalid" + reasonGoldenActorCrashed = "GoldenActorCrashed" + reasonUnexpectedState = "GoldenActorUnexpectedState" ) // maxGoldenErrorMessageLen is the maxLength of // GoldenSnapshotStatus.error_message. const maxGoldenErrorMessageLen = 4096 +var errGoldenEgressPolicyConflict = errors.New("golden actor egress policy does not match the template") + // templateReconcilerStore enumerates the exact storage methods needed by // ActorTemplateReconciler and nothing more. type templateReconcilerStore interface { @@ -74,6 +78,8 @@ type goldenActorControl interface { DeleteActor(ctx context.Context, req *ateapipb.DeleteActorRequest) (*ateapipb.Actor, error) CreateActor(ctx context.Context, req *ateapipb.CreateActorRequest) (*ateapipb.Actor, error) GetActor(ctx context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error) + CreateActorEgressPolicy(ctx context.Context, req *ateapipb.CreateActorEgressPolicyRequest) (*ateapipb.EgressPolicy, error) + GetActorEgressPolicy(ctx context.Context, req *ateapipb.GetActorEgressPolicyRequest) (*ateapipb.EgressPolicy, error) ResumeActor(ctx context.Context, req *ateapipb.ResumeActorRequest) (*ateapipb.ResumeActorResponse, error) SuspendActor(ctx context.Context, req *ateapipb.SuspendActorRequest) (*ateapipb.SuspendActorResponse, error) } @@ -191,6 +197,8 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource } return 0, err } + // TODO: Skip a template with delete_time set. DeleteActorTemplate may + // already have deleted the golden actor and tag, and this pass rebuilds them. if tmpl.GetSnapshotConfig().GetGoldenSnapshotConfig().GetMode() == ateapipb.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED { return 0, nil @@ -294,6 +302,12 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource // without being recorded. return 0, r.tagGoldenActor(ctx, tmpl, goldenActorRef) } + if err := r.ensureGoldenEgressPolicy(ctx, tmpl, goldenActorRef); err != nil { + if errors.Is(err, errGoldenEgressPolicyConflict) { + return 0, r.fail(ctx, tmpl, reasonGoldenEgressPolicyConflict, err.Error()) + } + return 0, fmt.Errorf("while ensuring golden egress policy: %w", err) + } if _, err := r.control.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: goldenActorRef}); err != nil { // A crash during resume is observed as CRASHED on the retry. return 0, fmt.Errorf("while resuming golden actor: %w", err) @@ -446,3 +460,33 @@ func (r *ActorTemplateReconciler) ensureActorExists(ctx context.Context, tmpl *a } return actor, nil } + +// ensureGoldenEgressPolicy also runs when an earlier pass created the actor but +// failed before creating its policy or resuming it. +func (r *ActorTemplateReconciler) ensureGoldenEgressPolicy(ctx context.Context, tmpl *ateapipb.ActorTemplate, actorRef *ateapipb.ObjectRef) error { + want := tmpl.GetGoldenEgressPolicy() + if want == nil { + return nil + } + _, err := r.control.CreateActorEgressPolicy(ctx, &ateapipb.CreateActorEgressPolicyRequest{ + Actor: actorRef, + EgressPolicy: &ateapipb.EgressPolicy{ + Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.GetAtespace(), Name: resources.SingletonName}, + Rules: proto.CloneOf(want).GetRules(), + }, + }) + if err == nil { + return nil + } + if apierror.Code(err) != codes.AlreadyExists { + return err + } + existing, err := r.control.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: actorRef, Name: resources.SingletonName}) + if err != nil { + return err + } + if !proto.Equal(&ateapipb.EgressPolicyTemplate{Rules: existing.GetRules()}, want) { + return errGoldenEgressPolicyConflict + } + return nil +} diff --git a/cmd/ateapi/internal/controlapi/template_reconciler_test.go b/cmd/ateapi/internal/controlapi/template_reconciler_test.go index 8a53f51c89..4786dab365 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler_test.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler_test.go @@ -27,7 +27,9 @@ import ( "go.ate.dev/substrate/cmd/ateapi/internal/apivalidation" "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/preview" "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" @@ -238,6 +240,14 @@ func (c *fakeGoldenControl) ResumeActor(_ context.Context, req *ateapipb.ResumeA return &ateapipb.ResumeActorResponse{}, nil } +func (c *fakeGoldenControl) CreateActorEgressPolicy(context.Context, *ateapipb.CreateActorEgressPolicyRequest) (*ateapipb.EgressPolicy, error) { + return nil, errors.New("unexpected golden egress policy creation") +} + +func (c *fakeGoldenControl) GetActorEgressPolicy(context.Context, *ateapipb.GetActorEgressPolicyRequest) (*ateapipb.EgressPolicy, error) { + return nil, errors.New("unexpected golden egress policy lookup") +} + func (c *fakeGoldenControl) SuspendActor(_ context.Context, req *ateapipb.SuspendActorRequest) (*ateapipb.SuspendActorResponse, error) { c.mu.Lock() defer c.mu.Unlock() @@ -638,6 +648,228 @@ func TestReconcileOne(t *testing.T) { } } +type goldenEgressControl struct { + *RPCService + resume func(context.Context, *ateapipb.ResumeActorRequest) (*ateapipb.ResumeActorResponse, error) + createPolicy func(context.Context, *ateapipb.CreateActorEgressPolicyRequest) (*ateapipb.EgressPolicy, error) + getPolicy func(context.Context, *ateapipb.GetActorEgressPolicyRequest) (*ateapipb.EgressPolicy, error) +} + +func (c *goldenEgressControl) ResumeActor(ctx context.Context, req *ateapipb.ResumeActorRequest) (*ateapipb.ResumeActorResponse, error) { + return c.resume(ctx, req) +} + +func (c *goldenEgressControl) CreateActorEgressPolicy(ctx context.Context, req *ateapipb.CreateActorEgressPolicyRequest) (*ateapipb.EgressPolicy, error) { + if c.createPolicy != nil { + return c.createPolicy(ctx, req) + } + return c.RPCService.CreateActorEgressPolicy(ctx, req) +} + +func (c *goldenEgressControl) GetActorEgressPolicy(ctx context.Context, req *ateapipb.GetActorEgressPolicyRequest) (*ateapipb.EgressPolicy, error) { + if c.getPolicy != nil { + return c.getPolicy(ctx, req) + } + return c.RPCService.GetActorEgressPolicy(ctx, req) +} + +func TestReconcileOne_GoldenEgressPolicyBeforeResume(t *testing.T) { + preview.InitForTest(t, "GoldenEgressPolicy") + for _, tt := range []struct { + name string + policy *ateapipb.EgressPolicyTemplate + }{ + {name: "absent"}, + {name: "configured", policy: &ateapipb.EgressPolicyTemplate{Rules: []*ateapipb.EgressRule{{Https: &ateapipb.HttpsRule{Hostnames: []string{"api.example.com"}}}}}}, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := t.Context() + persistence := newTestPersistence(t) + impl := newServiceImpl(persistence, nil) + impl.clusterSnapshotLocation = "gs://cluster-bucket/" + svc := &RPCService{impl: impl} + storetest.MustCreateAtespace(t, ctx, persistence, "templates") + tmpl, err := svc.CreateActorTemplate(ctx, &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { + tmpl.Metadata = &ateapipb.ResourceMetadata{Atespace: "templates", Name: "tmpl"} + tmpl.GoldenEgressPolicy = tt.policy + })}) + if err != nil { + t.Fatal(err) + } + stop := errors.New("stop before running workload") + control := &goldenEgressControl{RPCService: svc, resume: func(ctx context.Context, req *ateapipb.ResumeActorRequest) (*ateapipb.ResumeActorResponse, error) { + ref := req.GetActor() + if ref.GetAtespace() != tmpl.GetMetadata().GetAtespace() || ref.GetName() != tmpl.GetMetadata().GetUid() { + t.Fatalf("unexpected golden actor: %v", ref) + } + policy, err := svc.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: ref, Name: resources.SingletonName}) + if tt.policy == nil { + if apierror.Code(err) != codes.NotFound { + t.Fatalf("golden without configured policy: %v, want NotFound", err) + } + return nil, stop + } + if err != nil { + t.Fatalf("golden policy must exist before resume: %v", err) + } + want := &ateapipb.EgressPolicy{Metadata: policy.Metadata, Rules: tmpl.GetGoldenEgressPolicy().GetRules()} + if policy.GetMetadata().GetAtespace() != ref.GetAtespace() || policy.GetMetadata().GetName() != "default" || !proto.Equal(policy, want) { + t.Fatalf("golden policy = %v, want rules %v in %s", policy, want.Rules, ref.GetAtespace()) + } + if md := policy.GetMetadata(); md.GetUid() == "" || md.GetVersion() != 1 || md.GetCreateTime() == nil || md.GetUpdateTime() == nil { + t.Errorf("unexpected policy metadata: %v", md) + } + return nil, stop + }} + r := newTestTemplateReconciler(persistence, control) + defer r.queue.ShutDown() + // Turning the gate off must not drop an existing template's policy. + preview.InitForTest(t) + for range 2 { + if _, err := r.reconcileOne(ctx, resources.ActorTemplateRefFromActorTemplate(tmpl)); !errors.Is(err, stop) { + t.Fatalf("reconcile = %v, want to reach resume with the policy present", err) + } + } + }) + } +} + +func TestReconcileOne_GoldenEgressPolicyRecovery(t *testing.T) { + preview.InitForTest(t, "GoldenEgressPolicy") + for _, scenario := range []string{"create fails", "create response lost", "lookup fails", "conflicting policy"} { + t.Run(scenario, func(t *testing.T) { + ctx := t.Context() + persistence := newTestPersistence(t) + impl := newServiceImpl(persistence, nil) + impl.clusterSnapshotLocation = "gs://cluster-bucket/" + svc := &RPCService{impl: impl} + storetest.MustCreateAtespace(t, ctx, persistence, "templates") + tmpl, err := svc.CreateActorTemplate(ctx, &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { + tmpl.Metadata = &ateapipb.ResourceMetadata{Atespace: "templates", Name: "tmpl"} + tmpl.GoldenEgressPolicy = &ateapipb.EgressPolicyTemplate{Rules: []*ateapipb.EgressRule{ + {Https: &ateapipb.HttpsRule{Hostnames: []string{"api.example.com"}}}, + }} + })}) + if err != nil { + t.Fatal(err) + } + ref := &ateapipb.ObjectRef{Atespace: tmpl.GetMetadata().GetAtespace(), Name: tmpl.GetMetadata().GetUid()} + transient := errors.New("policy service unavailable") + stop := errors.New("stop before running workload") + resumeCalls := 0 + control := &goldenEgressControl{RPCService: svc, resume: func(context.Context, *ateapipb.ResumeActorRequest) (*ateapipb.ResumeActorResponse, error) { + resumeCalls++ + return nil, stop + }} + control.createPolicy = func(ctx context.Context, req *ateapipb.CreateActorEgressPolicyRequest) (*ateapipb.EgressPolicy, error) { + if scenario == "create fails" { + return nil, transient + } + if scenario == "conflicting policy" { + req.EgressPolicy.Rules[0].Https.Hostnames = []string{"other.example.com"} + } + if _, err := svc.CreateActorEgressPolicy(ctx, req); err != nil { + t.Fatal(err) + } + if scenario == "create response lost" { + return nil, transient + } + return nil, apierror.AlreadyExists("policy exists") + } + if scenario == "lookup fails" { + control.getPolicy = func(context.Context, *ateapipb.GetActorEgressPolicyRequest) (*ateapipb.EgressPolicy, error) { + return nil, transient + } + } + var actorUID string + var previousPolicy *ateapipb.EgressPolicy + for pass := range 2 { + // A fresh reconciler must recover from persisted state alone. + r := newTestTemplateReconciler(persistence, control) + after, err := r.reconcileOne(ctx, resources.ActorTemplateRefFromActorTemplate(tmpl)) + r.queue.ShutDown() + switch { + case scenario == "conflicting policy": + if err != nil || after != 0 { + t.Fatalf("reconcile = (%v, %v), want terminal failure without retry", after, err) + } + case pass == 0: + if !errors.Is(err, transient) { + t.Fatalf("reconcile = %v, want policy service error", err) + } + default: + if !errors.Is(err, stop) { + t.Fatalf("reconcile = %v, want to reach resume", err) + } + } + updated, err := persistence.GetActorTemplate(ctx, resources.ActorTemplateRefFromActorTemplate(tmpl)) + if err != nil { + t.Fatal(err) + } + wantError := "" + if scenario == "conflicting policy" { + wantError = "GoldenEgressPolicyConflict: golden actor egress policy does not match the template" + } + snapshotStatus := updated.GetStatus().GetGoldenSnapshotStatus() + if snapshotStatus.GetErrorMessage() != wantError || snapshotStatus.GetGoldenTag() != nil { + t.Fatalf("unexpected golden snapshot status: %v", snapshotStatus) + } + wantResumes := 0 + if pass == 1 && scenario != "conflicting policy" { + wantResumes = 1 + } + if resumeCalls != wantResumes { + t.Fatalf("resume calls = %d, want %d", resumeCalls, wantResumes) + } + actor, err := svc.GetActor(ctx, &ateapipb.GetActorRequest{Actor: ref}) + if err != nil { + t.Fatal(err) + } + if actorUID != "" && actor.GetMetadata().GetUid() != actorUID { + t.Fatal("retry recreated the golden actor") + } + actorUID = actor.GetMetadata().GetUid() + if actor.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + t.Fatalf("actor state = %v, want SUSPENDED", actor.GetStatus().GetState()) + } + policy, err := svc.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: ref, Name: resources.SingletonName}) + if pass == 0 && scenario == "create fails" { + if apierror.Code(err) != codes.NotFound { + t.Fatalf("policy lookup = %v, want NotFound", err) + } + } else { + if err != nil { + t.Fatal(err) + } + if previousPolicy != nil && !proto.Equal(policy, previousPolicy) { + t.Fatal("reconcile changed an existing policy") + } + wantHost := "api.example.com" + if scenario == "conflicting policy" { + wantHost = "other.example.com" + } + if got := policy.GetRules()[0].GetHttps().GetHostnames(); len(got) != 1 || got[0] != wantHost { + t.Fatalf("policy hostnames = %v, want %s", got, wantHost) + } + previousPolicy = policy + } + control.createPolicy = nil + control.getPolicy = nil + if scenario == "conflicting policy" { + control.createPolicy = func(context.Context, *ateapipb.CreateActorEgressPolicyRequest) (*ateapipb.EgressPolicy, error) { + t.Fatal("retried policy creation after terminal failure") + return nil, nil + } + control.getPolicy = func(context.Context, *ateapipb.GetActorEgressPolicyRequest) (*ateapipb.EgressPolicy, error) { + t.Fatal("retried policy lookup after terminal failure") + return nil, nil + } + } + } + }) + } +} + // TestReconcileOne_GoldenActorRequests pins the shape of the control-plane // requests the happy path issues: the golden actor is named after the // template UID so recreated templates with the same name never collide, and diff --git a/cmd/ateapi/internal/controlapi/validate.go b/cmd/ateapi/internal/controlapi/validate.go index c83b3fdac3..50959b934a 100644 --- a/cmd/ateapi/internal/controlapi/validate.go +++ b/cmd/ateapi/internal/controlapi/validate.go @@ -14,15 +14,7 @@ package controlapi -import ( - "go.ate.dev/substrate/internal/apierror" - "go.ate.dev/substrate/pkg/proto/ateapipb" - "k8s.io/apimachinery/pkg/util/validation/field" -) - -func toGRPCInternalError(errs field.ErrorList) error { - return apierror.Internal("%v", errs.ToAggregate()) -} +import "go.ate.dev/substrate/pkg/proto/ateapipb" // scrubResourceMetadataForCreate removes fields that should not be set by the // user when creating a resource. @@ -34,6 +26,7 @@ func scrubResourceMetadataForCreate(in *ateapipb.ResourceMetadata) { in.Version = 0 // will be set later in.CreateTime = nil // will be set later in.UpdateTime = nil // will be set later + in.DeleteTime = nil // will be set later } // scrubResourceMetadataForUpdate removes fields that should not be set by the @@ -45,4 +38,5 @@ func scrubResourceMetadataForUpdate(in *ateapipb.ResourceMetadata) { // in.Uid and in.Version are preconditions, so we don't scrub them. in.CreateTime = nil // will be set later in.UpdateTime = nil // will be set later + in.DeleteTime = nil // will be set later } diff --git a/cmd/ateapi/internal/controlapi/worker.go b/cmd/ateapi/internal/controlapi/worker.go index c7778cd82f..9d0c6ba68e 100644 --- a/cmd/ateapi/internal/controlapi/worker.go +++ b/cmd/ateapi/internal/controlapi/worker.go @@ -138,7 +138,7 @@ func (s *ServiceImpl) CreateWorker(ctx context.Context, inWorker *ateapipb.Worke // Verify that the result is properly valid before storing it. if errs := apivalidation.ValidateWorkerUpdate(ctx, field.NewPath("worker"), outWorker, inWorker, true); len(errs) > 0 { - return nil, toGRPCInternalError(errs) + return nil, resources.ToInternalError(errs) } // Save the data in the storage layer. @@ -207,7 +207,7 @@ func (s *ServiceImpl) UpdateWorker(ctx context.Context, name string, preconditio // Validate the final value before storing it. if errs := apivalidation.ValidateWorkerUpdate(ctx, field.NewPath("worker"), newVal, oldVal, true); len(errs) > 0 { - return toGRPCInternalError(errs) + return resources.ToInternalError(errs) } return nil @@ -246,6 +246,10 @@ func (s *RPCService) DeleteWorker(ctx context.Context, req *ateapipb.DeleteWorke return s.workerWorkflow.DeleteWorker(ctx, req.GetWorker().GetName(), toDeletePreconditions(req.GetOptions())) } +func (s *ServiceImpl) MarkWorkerForDeletion(ctx context.Context, name string, precondition store.DeletePreconditions, mutate func(*ateapipb.Worker) error) (*ateapipb.Worker, error) { + return s.store.MarkWorkerForDeletion(ctx, name, precondition, mutate) +} + func (s *ServiceImpl) DeleteWorker(ctx context.Context, name string, precondition store.DeletePreconditions) (*ateapipb.Worker, error) { return s.store.DeleteWorker(ctx, name, precondition) } diff --git a/cmd/ateapi/internal/controlapi/worker_test.go b/cmd/ateapi/internal/controlapi/worker_test.go index 8ec1a6dcd3..1cd167e5ab 100644 --- a/cmd/ateapi/internal/controlapi/worker_test.go +++ b/cmd/ateapi/internal/controlapi/worker_test.go @@ -47,7 +47,6 @@ func validWorker(name string, mods ...func(*ateapipb.Worker)) *ateapipb.Worker { WorkerPodUid: name, NodeName: "node-1", Ips: []string{"10.1.2.3"}, - SandboxClass: "gvisor", } for _, m := range mods { m(w) @@ -83,7 +82,12 @@ func seedAPIWorker(t *testing.T, ctx context.Context, persistence store.Interfac t.Helper() worker = proto.Clone(worker).(*ateapipb.Worker) if worker.GetStatus() == nil { - worker.Status = &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE} + // A seeded Worker stands in for one that registered: ACTIVE, with the + // gvisor default runtime its class is read from. + worker.Status = &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + SandboxClasses: workerSandboxClasses(testSandboxRuntime()), + } } created, err := persistence.CreateWorker(ctx, worker) if err != nil { @@ -460,7 +464,6 @@ func TestUpdateWorker_Errors(t *testing.T) { {"ips family added", func(w *ateapipb.Worker) { w.Ips = append(w.Ips, "fd00::1") }, codes.InvalidArgument}, {"worker_pod changed", func(w *ateapipb.Worker) { w.WorkerPod = "worker-pod-2" }, codes.InvalidArgument}, {"node_name changed", func(w *ateapipb.Worker) { w.NodeName = "node-2" }, codes.InvalidArgument}, - {"sandbox_class changed", func(w *ateapipb.Worker) { w.SandboxClass = "microvm" }, codes.InvalidArgument}, // capacity is deliberately absent here: it may change (the pool's actor // ceiling moves, a pod can be resized, a Worker may report its own). // TestUpdateWorker_CapacityChanges covers that. @@ -468,7 +471,6 @@ func TestUpdateWorker_Errors(t *testing.T) { // And immutable fields dropped, which a replacement update reads as a // request to clear them. Rejected rather than silently applied. {"ips omitted", func(w *ateapipb.Worker) { w.Ips = nil }, codes.InvalidArgument}, - {"sandbox_class omitted", func(w *ateapipb.Worker) { w.SandboxClass = "" }, codes.InvalidArgument}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -518,11 +520,15 @@ func TestDeleteWorker(t *testing.T) { if err != nil { t.Fatalf("DeleteWorker() failed: %v", err) } - // Delete drains before it sweeps, so it removes one revision past what was - // seeded. + // Delete admits and drains in one write before it sweeps, so it removes one + // revision past what was seeded. + if got.GetMetadata().GetDeleteTime() == nil { + t.Error("DeleteWorker() returned a worker without delete_time") + } want := proto.Clone(seeded).(*ateapipb.Worker) want.Metadata.Version = seeded.GetMetadata().GetVersion() + 1 want.Metadata.UpdateTime = got.GetMetadata().GetUpdateTime() + want.Metadata.DeleteTime = got.GetMetadata().GetDeleteTime() want.Status.State = ateapipb.WorkerState_WORKER_STATE_DRAINING if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { t.Errorf("DeleteWorker() returned something other than the worker it removed (-want +got):\n%s", diff) diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index c7b8cbb3c1..abec2402a9 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -174,6 +174,7 @@ func (w *ActorWorkflow) copySnapshot(ctx context.Context, src, dst resources.Sto type actorWorkflowStore interface { GetActor(ctx context.Context, actorRef resources.ActorRef) (*ateapipb.Actor, error) UpdateActor(ctx context.Context, actorRef resources.ActorRef, precondition store.Precondition, mutate func(toUpdate *ateapipb.Actor) error) (*ateapipb.Actor, error) + MarkActorForDeletion(ctx context.Context, actorRef resources.ActorRef, precondition store.DeletePreconditions, mutate func(toUpdate *ateapipb.Actor) error) (*ateapipb.Actor, error) DeleteActor(ctx context.Context, actorRef resources.ActorRef, precondition store.DeletePreconditions) (*ateapipb.Actor, error) GetWorker(ctx context.Context, name string) (*ateapipb.Worker, error) UpdateWorker(ctx context.Context, name string, precondition store.Precondition, mutate func(toUpdate *ateapipb.Worker) error) (*ateapipb.Worker, error) @@ -187,8 +188,10 @@ type actorWorkflowStore interface { CreateTag(ctx context.Context, tag *ateapipb.Tag) (*ateapipb.Tag, error) GetTag(ctx context.Context, tagRef resources.TagRef) (*ateapipb.Tag, error) UpdateTag(ctx context.Context, tagRef resources.TagRef, precondition store.Precondition, mutate func(toUpdate *ateapipb.Tag) error) (*ateapipb.Tag, error) + MarkTagForDeletion(ctx context.Context, tagRef resources.TagRef, precondition store.DeletePreconditions, mutate func(toUpdate *ateapipb.Tag) error) (*ateapipb.Tag, error) DeleteTag(ctx context.Context, tagRef resources.TagRef, precondition store.DeletePreconditions) (*ateapipb.Tag, error) GetActorTemplate(ctx context.Context, templateRef resources.ActorTemplateRef) (*ateapipb.ActorTemplate, error) + MarkActorTemplateForDeletion(ctx context.Context, templateRef resources.ActorTemplateRef, precondition store.DeletePreconditions, mutate func(toUpdate *ateapipb.ActorTemplate) error) (*ateapipb.ActorTemplate, error) DeleteActorTemplate(ctx context.Context, templateRef resources.ActorTemplateRef, precondition store.DeletePreconditions) (*ateapipb.ActorTemplate, error) AcquireLease(ctx context.Context, key string) (*store.Lease, error) } @@ -213,6 +216,7 @@ func NewWorkerWorkflow(store workerWorkflowStore) *WorkerWorkflow { type workerWorkflowStore interface { GetWorker(ctx context.Context, name string) (*ateapipb.Worker, error) UpdateWorker(ctx context.Context, name string, precondition store.Precondition, mutate func(toUpdate *ateapipb.Worker) error) (*ateapipb.Worker, error) + MarkWorkerForDeletion(ctx context.Context, name string, precondition store.DeletePreconditions, mutate func(toUpdate *ateapipb.Worker) error) (*ateapipb.Worker, error) DeleteWorker(ctx context.Context, name string, precondition store.DeletePreconditions) (*ateapipb.Worker, error) ListWorkerAssignments(ctx context.Context, workerName string, opts store.ListOptions) (store.ListResponse[*ateapipb.ActorAssignment], error) ReleaseActorFromWorker(ctx context.Context, workerName string, actorUID string) (*ateapipb.Worker, error) diff --git a/cmd/ateapi/internal/controlapi/workflow_delete.go b/cmd/ateapi/internal/controlapi/workflow_delete.go index 8317791a82..2dc3e09d37 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete.go @@ -40,22 +40,11 @@ func (w *ActorWorkflow) DeleteActor(ctx context.Context, actorRef resources.Acto } defer lease.Close() - actor, err := w.loadActorForDelete(ctx, actorRef) + actor, err := w.ensureMarkedDeleting(ctx, actorRef, precondition) if err != nil { return nil, err } - if err := precondition.Check(actor.GetMetadata()); err != nil { - if errors.Is(err, store.ErrUIDConflict) { - return nil, apierror.Aborted("Actor %s does not have uid %s", actorRef, precondition.UID) - } - return nil, apierror.Aborted("concurrent update conflict, please retry") - } - - if actor, err = w.ensureMarkedDeleting(ctx, actorRef, actor); err != nil { - return nil, err - } - // DeleteActor will attempt best-effort cleanup across all steps, collecting any errors. // If any step fails, errors are returned so the caller can retry, and the actor record // is retained in the store in the DELETING state. @@ -107,21 +96,6 @@ func (w *ActorWorkflow) DeleteActor(ctx context.Context, actorRef resources.Acto return w.finalizeDeleted(ctx, actor) } -// loadActorForDelete fetches the current actor record. -func (w *ActorWorkflow) loadActorForDelete(ctx context.Context, actorRef resources.ActorRef) (_ *ateapipb.Actor, err error) { - ctx, done := stepSpan(ctx, "LoadActorForDelete") - defer func() { err = done(err) }() - - actor, err := w.store.GetActor(ctx, actorRef) - if err != nil { - if errors.Is(err, store.ErrNotFound) { - return nil, apierror.NotFound("Actor %s not found", actorRef) - } - return nil, fmt.Errorf("while fetching actor: %w", err) - } - return actor, nil -} - // ensureAteletTerminated calls atelet to terminate the workload for opName. func (w *ActorWorkflow) ensureAteletTerminated(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate, opName string) (err error) { ctx, done := stepSpan(ctx, "CallAteletTerminate") @@ -154,6 +128,19 @@ func (w *ActorWorkflow) ensureAteletTerminated(ctx context.Context, actorRef res } conn, err := w.dialer.DialForAteletOnNode(nodeName) + if errors.Is(err, ErrNoAteletOnNode) { + gone, nodeErr := w.dialer.NodeGone(ctx, nodeName) + if nodeErr != nil { + return fmt.Errorf("while connecting to atelet on node %q: %w (and while checking the node: %w)", nodeName, err, nodeErr) + } + if gone { + // A deleted Node object counts as the node being gone for good. If the + // machine still runs, whatever atelet holds for the actor stays there. + slog.WarnContext(ctx, "assigned node no longer exists, skipping atelet terminate request", + slog.String("node", nodeName), slog.Any("actor", actorRef)) + return nil + } + } if err != nil { return fmt.Errorf("while connecting to atelet on node %q: %w", nodeName, err) } @@ -289,7 +276,7 @@ func (w *ActorWorkflow) ensureWorkerReleased(ctx context.Context, actorRef resou } if latestActor.GetStatus().GetWorkerAssignment() != nil { - _, _, err := releaseWorker(ctx, w.store, latestActor) + _, err := releaseWorker(ctx, w.store, latestActor) if err != nil { return nil, err } @@ -318,27 +305,39 @@ func (w *ActorWorkflow) ensureWorkerReleased(ctx context.Context, actorRef resou return latestActor, nil } -// ensureMarkedDeleting transitions the actor to DELETING and persists the -// change, returning the stored copy. Skips when a previous attempt already -// marked the actor. -func (w *ActorWorkflow) ensureMarkedDeleting(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor) (updated *ateapipb.Actor, err error) { +var errAlreadyMarked = errors.New("already marked for deletion") + +// ensureMarkedDeleting sets the actor's delete_time, checking precondition, +// and moves it to DELETING, returning the stored copy. Skips when a previous +// attempt already marked the actor. +func (w *ActorWorkflow) ensureMarkedDeleting(ctx context.Context, actorRef resources.ActorRef, precondition store.DeletePreconditions) (updated *ateapipb.Actor, err error) { ctx, done := stepSpan(ctx, "MarkDeleting") defer func() { err = done(err) }() - if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_DELETING { - markSkipped(ctx, "actor already DELETING") - return actor, nil - } - - storedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { + var alreadyDeleting *ateapipb.Actor + storedActor, err := w.store.MarkActorForDeletion(ctx, actorRef, precondition, func(toUpdate *ateapipb.Actor) error { + if toUpdate.GetMetadata().GetDeleteTime() != nil { + alreadyDeleting = toUpdate + return errAlreadyMarked + } toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_DELETING return nil }) + if errors.Is(err, errAlreadyMarked) { + markSkipped(ctx, "actor already marked for deletion") + return alreadyDeleting, nil + } if err != nil { + if errors.Is(err, store.ErrUIDConflict) { + return nil, apierror.Aborted("Actor %s does not have uid %s", actorRef, precondition.UID) + } if errors.Is(err, store.ErrVersionConflict) { return nil, apierror.Aborted("concurrent update conflict, please retry") } - return nil, fmt.Errorf("while setting actor state to DELETING: %w", err) + if errors.Is(err, store.ErrNotFound) { + return nil, apierror.NotFound("Actor %s not found", actorRef) + } + return nil, fmt.Errorf("while marking actor for deletion: %w", err) } logActorStateChanged(ctx, storedActor, ateattr.OperationDelete) return storedActor, nil @@ -438,7 +437,10 @@ func (w *ActorWorkflow) finalizeDeleted(ctx context.Context, actor *ateapipb.Act defer func() { err = done(err) }() actorRef := resources.ActorRefFromActor(actor) - precondition := store.DeletePreconditions{UID: actor.GetMetadata().GetUid(), Version: actor.GetMetadata().GetVersion()} + // Only the uid guards the delete. The mark already committed this + // incarnation to deletion, and a later write such as an UpdateActor RPC, + // which takes no actor lease, must not block removing the row. + precondition := store.DeletePreconditions{UID: actor.GetMetadata().GetUid()} deleted, err := w.store.DeleteActor(ctx, actorRef, precondition) if err != nil { if errors.Is(err, store.ErrNotFound) { diff --git a/cmd/ateapi/internal/controlapi/workflow_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_delete_test.go index 863049b616..917b56254f 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete_test.go @@ -17,6 +17,7 @@ package controlapi import ( "context" "errors" + "slices" "strings" "testing" "time" @@ -24,11 +25,14 @@ import ( "go.ate.dev/substrate/cmd/ateapi/internal/store" "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/installdefaults" "go.ate.dev/substrate/internal/objectstore/objectstoretest" "go.ate.dev/substrate/internal/objectstoreplugin/objectstoreplugintest" "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/volume" "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func TestDeleteActorWorkflow_ExecutionPaths(t *testing.T) { @@ -36,6 +40,8 @@ func TestDeleteActorWorkflow_ExecutionPaths(t *testing.T) { name string seedState ateapipb.ActorState missingTmpl bool + // markActorForDeletion marks the actor before the call, as a failed delete would. + markActorForDeletion bool }{ { name: "delete suspended actor succeeds", @@ -46,8 +52,9 @@ func TestDeleteActorWorkflow_ExecutionPaths(t *testing.T) { seedState: ateapipb.ActorState_ACTOR_STATE_CRASHED, }, { - name: "delete deleting actor succeeds", - seedState: ateapipb.ActorState_ACTOR_STATE_DELETING, + name: "delete deleting actor succeeds", + seedState: ateapipb.ActorState_ACTOR_STATE_DELETING, + markActorForDeletion: true, }, { name: "delete running actor succeeds", @@ -78,6 +85,11 @@ func TestDeleteActorWorkflow_ExecutionPaths(t *testing.T) { tmplName = "missing-tmpl" } seedWorkflowActor(t, ctx, st, actorRef, "ns", tmplName, tc.seedState) + if tc.markActorForDeletion { + if _, err := st.MarkActorForDeletion(ctx, actorRef, store.DeletePreconditions{}, nil); err != nil { + t.Fatalf("MarkActorForDeletion: %v", err) + } + } deleted, err := w.DeleteActor(ctx, actorRef, store.DeletePreconditions{}) if err != nil { @@ -93,8 +105,178 @@ func TestDeleteActorWorkflow_ExecutionPaths(t *testing.T) { } } +// A deleted Node object counts as the node being gone for good, so revert and +// delete do not wait for its atelet. A node that is still there may get its +// atelet back, and a failed lookup proves nothing, so both stay retryable. +func TestEnsureAteletTerminated_NodeGone(t *testing.T) { + actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} + actor := &ateapipb.Actor{Status: &ateapipb.ActorStatus{AssignedNode: "node1"}} + for _, tc := range []struct { + name string + nodes []string + lookupErr error + wantErr bool + }{ + {name: "node gone", nodes: []string{"node2"}}, + {name: "node still there", nodes: []string{"node1"}, wantErr: true}, + {name: "node lookup fails", lookupErr: errors.New("connection refused"), wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + w := newTestActorWorkflow(t, newTestPersistence(t), "team-a", "tmpl") + w.dialer = NewAteletDialer(newTestAteletIndexer(t), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "", + WithNodeGetter(nodeGetter(tc.lookupErr, tc.nodes...))) + err := w.ensureAteletTerminated(context.Background(), actorRef, actor, nil, "revert") + if tc.wantErr != (err != nil) { + t.Fatalf("ensureAteletTerminated = %v, want error %v", err, tc.wantErr) + } + if tc.wantErr && !errors.Is(err, ErrNoAteletOnNode) { + t.Fatalf("ensureAteletTerminated = %v, want ErrNoAteletOnNode", err) + } + if tc.lookupErr != nil && !strings.Contains(err.Error(), tc.lookupErr.Error()) { + t.Fatalf("ensureAteletTerminated = %v, want the node lookup error in it", err) + } + }) + } +} + +// detachDeletePlugin is a volume plugin for a whole delete: it records +// detaches like mockDetachVolumePlugin and accepts the volume delete. +type detachDeletePlugin struct{ mockDetachVolumePlugin } + +func (*detachDeletePlugin) DeleteVolume(context.Context, string) error { return nil } + +// TestRevertAndDelete_AssignedNodeGone runs the whole revert and delete +// workflows for a paused actor with a PENDING upload and a crashed actor whose +// assigned node has no atelet (#2470). If the Node object is gone, both finish, +// and revert clears assigned_node. If it still exists, both fail with +// ErrNoAteletOnNode and leave the actor REVERTING or DELETING for a retry. +// A volume detach on the gone node still runs: OK and NotFound from the plugin +// count as detached, and any other error fails the step as before. +func TestRevertAndDelete_AssignedNodeGone(t *testing.T) { + memory := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + completed := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + for _, tc := range []struct { + name string + state ateapipb.ActorState + paused bool + nodeKept bool + volume bool + detachErr error + wantErr error // ErrNoAteletOnNode for a kept node + // wantDetachErr expects the plugin's detach error back. + wantDetachErr bool + }{ + {name: "paused, node gone", state: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, paused: true}, + {name: "crashed, node gone", state: ateapipb.ActorState_ACTOR_STATE_CRASHED}, + {name: "paused, node kept", state: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, paused: true, nodeKept: true, wantErr: ErrNoAteletOnNode}, + {name: "crashed, node kept", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, nodeKept: true, wantErr: ErrNoAteletOnNode}, + {name: "node gone, volume detach OK", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, volume: true}, + {name: "node gone, volume detach NotFound", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, volume: true, detachErr: status.Error(codes.NotFound, "no such node")}, + {name: "node gone, volume detach fails", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, volume: true, detachErr: status.Error(codes.Internal, "instance not found"), wantDetachErr: true}, + } { + for _, verb := range []string{"revert", "delete"} { + t.Run(tc.name+"/"+verb, func(t *testing.T) { + ctx := context.Background() + st, cleanup := storetest.SetupTestStore(t) + defer cleanup() + w := newTestActorWorkflow(t, st, "ns", "tmpl1") + nodes := []string{"node-2"} + if tc.nodeKept { + nodes = append(nodes, "node-1") + } + w.dialer = NewAteletDialer(newTestAteletIndexer(t), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "", + WithNodeGetter(nodeGetter(nil, nodes...))) + plugin := &detachDeletePlugin{mockDetachVolumePlugin{detachErrs: map[string]error{"storage-vol-1": tc.detachErr}}} + w.pluginRegistry = &mockPluginRegistry{plugins: map[string]volume.VolumePluginControlPlane{"mock": plugin}} + tmpl := &ateapipb.ActorTemplate{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "ns", Name: "tmpl-vol"}, + SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://snapshots"}, + Volumes: []*ateapipb.Volume{{Name: "vol1"}}, + Containers: []*ateapipb.Container{{VolumeMounts: []*ateapipb.VolumeMount{{Name: "vol1"}}}}, + } + if _, err := st.CreateActorTemplate(ctx, tmpl); err != nil { + t.Fatalf("create ActorTemplate: %v", err) + } + + actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} + seedWorkflowActor(t, ctx, st, actorRef, "ns", "tmpl-vol", tc.state) + actor, err := st.GetActor(ctx, actorRef) + if err != nil { + t.Fatalf("get seeded actor: %v", err) + } + mustUpdateActorStatus(t, ctx, st, actor, func(s *ateapipb.ActorStatus) { + s.AssignedNode = "node-1" + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", "kept", + mustActorSnapshotURI(t, tmpl, actor, "kept").String(), completed)} + if tc.paused { + local := newPausedSnapshot(t, 2, memory, "", "local", "node-1") + local.DurableSnapshot.Object.SnapshotUri = mustActorSnapshotURI(t, tmpl, actor, "local").String() + s.LastAssignedGeneration = 2 + s.Snapshots = append(s.Snapshots, local) + } + if tc.volume { + s.ExternalVolumes = &ateapipb.ExternalVolumes{Volumes: []*ateapipb.ExternalVolumeStatus{ + {Name: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}}} + } + }) + + wantState := ateapipb.ActorState_ACTOR_STATE_REVERTING + if verb == "revert" { + _, err = w.RevertActor(ctx, actorRef) + } else { + wantState = ateapipb.ActorState_ACTOR_STATE_DELETING + _, err = w.DeleteActor(ctx, actorRef, store.DeletePreconditions{}) + } + switch { + case tc.wantDetachErr: + if err == nil || !strings.Contains(err.Error(), "instance not found") { + t.Fatalf("%s = %v, want the detach error", verb, err) + } + case tc.wantErr != nil: + if !errors.Is(err, tc.wantErr) { + t.Fatalf("%s = %v, want %v", verb, err, tc.wantErr) + } + case err != nil: + t.Fatalf("%s: %v", verb, err) + } + if tc.volume { + if want := []detachCall{{VolumeID: "storage-vol-1", Node: "node-1"}}; !slices.Equal(plugin.detachCalls, want) { + t.Errorf("detach calls = %v, want %v", plugin.detachCalls, want) + } + } + + got, getErr := st.GetActor(ctx, actorRef) + switch { + case tc.wantErr != nil || tc.wantDetachErr: + if getErr != nil { + t.Fatalf("get actor after failed %s: %v", verb, getErr) + } + if s := got.GetStatus(); s.GetState() != wantState || s.GetAssignedNode() != "node-1" { + t.Errorf("after failed %s: state %v, assigned node %q, want %v on node-1", verb, s.GetState(), s.GetAssignedNode(), wantState) + } + case verb == "revert": + if getErr != nil { + t.Fatalf("get actor after revert: %v", getErr) + } + if s := got.GetStatus(); s.GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED || s.GetAssignedNode() != "" { + t.Errorf("after revert: state %v, assigned node %q, want SUSPENDED with no node", s.GetState(), s.GetAssignedNode()) + } + if snaps := got.GetStatus().GetSnapshots(); len(snaps) != 1 || snaps[0].GetGeneration() != 1 || snaps[0].GetDurableSnapshot().GetStatus() != completed { + t.Errorf("after revert: snapshots %v, want only the completed durable snapshot at generation 1", snaps) + } + default: + if !errors.Is(getErr, store.ErrNotFound) { + t.Errorf("get actor after delete = %v, want ErrNotFound", getErr) + } + } + }) + } + } +} + // TestEnsureMarkedDeleting_AllStates checks that every actor state can be -// marked DELETING. +// marked DELETING, and that a second call leaves the marked actor unwritten. func TestEnsureMarkedDeleting_AllStates(t *testing.T) { for _, seedState := range allActorStates { t.Run(seedState.String(), func(t *testing.T) { @@ -110,13 +292,24 @@ func TestEnsureMarkedDeleting_AllStates(t *testing.T) { t.Fatalf("get seeded actor: %v", err) } - updated, err := w.ensureMarkedDeleting(ctx, actorRef, actor) + updated, err := w.ensureMarkedDeleting(ctx, actorRef, store.DeletePreconditions{Version: actor.GetMetadata().GetVersion()}) if err != nil { t.Fatalf("ensureMarkedDeleting failed: %v", err) } if got := updated.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_DELETING { t.Errorf("ensureMarkedDeleting returned actor in %v, want DELETING", got) } + if updated.GetMetadata().GetDeleteTime() == nil { + t.Error("ensureMarkedDeleting left delete_time unset") + } + + again, err := w.ensureMarkedDeleting(ctx, actorRef, store.DeletePreconditions{}) + if err != nil { + t.Fatalf("second ensureMarkedDeleting failed: %v", err) + } + if got, want := again.GetMetadata().GetVersion(), updated.GetMetadata().GetVersion(); got != want { + t.Errorf("second ensureMarkedDeleting moved the version to %d, want %d", got, want) + } }) } } @@ -441,3 +634,75 @@ func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) { }) } } + +// TestDeleteActorWorkflow_RetryKeepsAdmission fails a guarded delete partway +// and retries it. The first attempt spends the version guard and stamps +// delete_time. Retries keep that delete_time and write nothing until the delete +// finishes. +func TestDeleteActorWorkflow_RetryKeepsAdmission(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + objects := objectstoretest.New() + w := NewActorWorkflow(persistence, nil, nil, nil, nil, "", nil, time.Minute, objectstoreplugintest.ControlClient(objects)) + + actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} + actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: "team-a", Name: "gone-tmpl"}, + Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_CRASHED}, + }) + inFlight := mustActorSnapshotURI(t, &ateapipb.ActorTemplate{ + SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: testStorageLocation}, + }, actor, "abandoned") + objects.PutSnapshot(t, inFlight, "manifest.json") + observed := mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.Name(), inFlight.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + } + }) + // The guards a client sends after reading the actor. + guards := store.DeletePreconditions{UID: observed.GetMetadata().GetUid(), Version: observed.GetMetadata().GetVersion()} + + objects.OnDelete = func(string, string) error { return errObjectStore } + if _, err := w.DeleteActor(ctx, actorRef, guards); !isObjectStoreErr(err) { + t.Fatalf("DeleteActor = %v, want it to report %v", err, errObjectStore) + } + admitted, err := persistence.GetActor(ctx, actorRef) + if err != nil { + t.Fatalf("GetActor after the failure: %v", err) + } + if admitted.GetMetadata().GetDeleteTime() == nil { + t.Fatal("a failed delete left delete_time unset") + } + if got := admitted.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_DELETING { + t.Fatalf("state after the failure = %v, want DELETING", got) + } + + if _, err := w.DeleteActor(ctx, actorRef, store.DeletePreconditions{UID: guards.UID}); !isObjectStoreErr(err) { + t.Fatalf("retried DeleteActor = %v, want it to report %v", err, errObjectStore) + } + retried, err := persistence.GetActor(ctx, actorRef) + if err != nil { + t.Fatalf("GetActor after the retried failure: %v", err) + } + if got, want := retried.GetMetadata().GetVersion(), admitted.GetMetadata().GetVersion(); got != want { + t.Errorf("a retried delete moved the version to %d, want %d", got, want) + } + + objects.OnDelete = nil + // The first attempt spent the version guard. The uid guard still holds. + if _, err := w.DeleteActor(ctx, actorRef, guards); apierror.Code(err) != codes.Aborted { + t.Fatalf("retry with the original version = %v, want ABORTED", err) + } + deleted, err := w.DeleteActor(ctx, actorRef, store.DeletePreconditions{UID: guards.UID}) + if err != nil { + t.Fatalf("retry with the uid alone: %v", err) + } + if !deleted.GetMetadata().GetDeleteTime().AsTime().Equal(admitted.GetMetadata().GetDeleteTime().AsTime()) { + t.Errorf("the retry re-stamped delete_time: %v, want %v", deleted.GetMetadata().GetDeleteTime().AsTime(), admitted.GetMetadata().GetDeleteTime().AsTime()) + } + if _, err := w.DeleteActor(ctx, actorRef, store.DeletePreconditions{}); apierror.Code(err) != codes.NotFound { + t.Errorf("DeleteActor after the row is gone = %v, want NOT_FOUND", err) + } +} diff --git a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go index f30691fc57..b4312d3191 100644 --- a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go @@ -304,6 +304,7 @@ func TestReconcileAssignments_ReleasesByActorState(t *testing.T) { {name: "running elsewhere is released", state: ateapipb.ActorState_ACTOR_STATE_RUNNING, pointsAt: otherWorker, wantState: ateapipb.ActorState_ACTOR_STATE_RUNNING, wantRelease: true}, {name: "suspended elsewhere is released", state: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, pointsAt: otherWorker, wantState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, wantRelease: true}, {name: "resuming crashes", state: ateapipb.ActorState_ACTOR_STATE_RESUMING, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantRelease: true}, + {name: "deleting stays deleting", state: ateapipb.ActorState_ACTOR_STATE_DELETING, wantState: ateapipb.ActorState_ACTOR_STATE_DELETING, wantRelease: true}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -573,12 +574,12 @@ func TestAssignWorkerAttempt_StampsWorkerEpoch(t *testing.T) { WorkerPool: "pool", WorkerPod: "pod-1", WorkerPodUid: name, - SandboxClass: "gvisor", + NodeName: "node-1", Epoch: 1, Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, - DefaultRuntime: testSandboxRuntime(), + SandboxClasses: workerSandboxClasses(testSandboxRuntime()), }, }) if err != nil { diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index 1a96d5c32c..65f8e1f57b 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -37,25 +37,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" ) -// resumeSnapshotSource is the boot source resolved once by loadActorForResume -// and passed by value to the restore step — never mutated after resolution. -type resumeSnapshotSource struct { - SnapshotUUID string - Generation int32 - // SnapshotURI is the storage location of the durable snapshot to restore - // from: the actor's latest snapshot, including a tag borrowed at creation. - // Zero means cold boot from the spec (unless the actor holds a local - // snapshot, which takes precedence at restore). - SnapshotURI resources.SnapshotURI - Fidelity ateapipb.SnapshotFidelity - // TemplateReplaced is true when the external snapshot's recorded template - // UID differs from the actor's current template. - TemplateReplaced bool - // RequiredRuntime is the runtime resolved from the snapshot's memory state - // (via requiredSnapshotRuntime), or nil when memory state is not restored. - RequiredRuntime *ateapipb.SandboxRuntime -} - // restoreTelemetry labels the restore operation for the resume lifecycle // metric. WireFidelity describes the restore requested, not the stored // snapshot's scope: a MEMORY snapshot restored under a replaced template goes @@ -106,8 +87,7 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, actorRef resources.Acto } defer lease.Close() - var src resumeSnapshotSource - actor, actorTemplate, src, err = w.loadActorForResume(leaseCtx, actorRef) + actor, actorTemplate, err = w.loadActorForResume(leaseCtx, actorRef) if err != nil { return nil, false, err } @@ -128,10 +108,6 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, actorRef resources.Acto return nil, false, err } actor = detached - // Resolve the restore source again, from the uploaded snapshot. - if src, err = resolveResumeSource(actorRef, actor, actorTemplate); err != nil { - return nil, false, err - } assigned, worker, err = w.ensureWorkerAssigned(leaseCtx, actorRef, actor, actorTemplate) } if err != nil { @@ -142,7 +118,7 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, actorRef resources.Acto if err != nil { return nil, false, err } - if tele, err = w.ensureAteletRestored(leaseCtx, actorRef, actor, actorTemplate, volumePublishContexts, src); err != nil { + if tele, err = w.ensureAteletRestored(leaseCtx, actorRef, actor, actorTemplate, volumePublishContexts); err != nil { return nil, false, err } var running *ateapipb.Actor @@ -153,61 +129,45 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, actorRef resources.Acto return actor, true, nil } -// loadActorForResume fetches the current actor record and its template, and -// resolves the boot source for the pending restore. -func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resources.ActorRef) (_ *ateapipb.Actor, _ *ateapipb.ActorTemplate, _ resumeSnapshotSource, err error) { +// loadActorForResume fetches the current actor record and its template. +func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resources.ActorRef) (_ *ateapipb.Actor, _ *ateapipb.ActorTemplate, err error) { ctx, done := stepSpan(ctx, "LoadActorForResume") defer func() { err = done(err) }() - var src resumeSnapshotSource actor, err := w.store.GetActor(ctx, actorRef) if err != nil { if errors.Is(err, store.ErrNotFound) { - return nil, nil, src, apierror.NotFound("Actor %s not found", actorRef) + return nil, nil, apierror.NotFound("Actor %s not found", actorRef) } - return nil, nil, src, fmt.Errorf("while getting actor from DB: %w", err) + return nil, nil, fmt.Errorf("while getting actor from DB: %w", err) } // If the actor is already running, there is no pending restore to prepare - // for. Short-circuit immediately to avoid unnecessary store reads for snapshots - // and template resolution on the hot resume path. + // for. Short-circuit immediately to avoid an unnecessary template read on + // the hot resume path. if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_RUNNING { - return actor, nil, src, nil + return actor, nil, nil } actorTemplate, err := resolveActorTemplate(ctx, w.store, actor) if err != nil { - return nil, nil, src, err - } - if src, err = resolveResumeSource(actorRef, actor, actorTemplate); err != nil { - return nil, nil, src, err + return nil, nil, err } - return actor, actorTemplate, src, nil + return actor, actorTemplate, nil } -// resolveResumeSource resolves the boot source for the actor's pending restore. -func resolveResumeSource(actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (resumeSnapshotSource, error) { - var src resumeSnapshotSource - var err error - src.RequiredRuntime = requiredSnapshotRuntime(actor, actorTemplate) - if snap := findLatestDurableSnapshot(actor.GetStatus(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); snap != nil { - st := snap.GetDurableSnapshot() - if src.SnapshotURI, err = resources.ParseSnapshotURI(st.GetObject().GetSnapshotUri()); err != nil { - return src, apierror.DataLoss("Actor %s external snapshot: %v", actorRef, err) - } - src.SnapshotUUID = snap.GetUuid() - src.Generation = snap.GetGeneration() - src.Fidelity = snap.GetFidelity() - src.TemplateReplaced = templateReplaced(snap, actorTemplate) - } - // A local restore also names the local snapshot's upload, so a URI that - // does not parse fails here too, before a worker is claimed. - if local := findLatestLocalSnapshot(actor.GetStatus()); local != nil { - if _, err = resources.ParseSnapshotURI(local.GetDurableSnapshot().GetObject().GetSnapshotUri()); err != nil { - return src, apierror.DataLoss("Actor %s local snapshot: %v", actorRef, err) - } - } - return src, nil +// checkSnapshotURI fails with DataLoss when the URI of snapshotToResume does +// not parse. A local restore also names the snapshot's upload, so its URI +// must parse too. +func checkSnapshotURI(actorRef resources.ActorRef, actor *ateapipb.Actor) error { + snap := snapshotToResume(actor.GetStatus()) + if snap == nil { + return nil + } + if _, err := resources.ParseSnapshotURI(snap.GetDurableSnapshot().GetObject().GetSnapshotUri()); err != nil { + return apierror.DataLoss("Actor %s snapshot %s: %v", actorRef, snap.GetUuid(), err) + } + return nil } // templateReplaced reports whether snap was captured under a different @@ -217,6 +177,18 @@ func templateReplaced(snap *ateapipb.Snapshot, tmpl *ateapipb.ActorTemplate) boo return capturedUnder != "" && capturedUnder != tmpl.GetMetadata().GetUid() } +// snapshotSharing is what a restore declares about a snapshot with the +// given owner. A tag's snapshot is immutable once published and read by +// every actor created from it, so atelet may serve its files from the +// node-local cache; the actor's own is replaced by its next suspend, so +// never, and so is one whose owner is not recorded. +func snapshotSharing(owner ateapipb.SnapshotOwner) ateletpb.SnapshotSharing { + if owner == ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG { + return ateletpb.SnapshotSharing_SNAPSHOT_SHARING_SHARED + } + return ateletpb.SnapshotSharing_SNAPSHOT_SHARING_PRIVATE +} + // ensureVolumesCreated provisions any actor volumes that have no storage // volume ID yet, persisting the resulting volume state (even when creation // partially failed, so progress is not lost) and returning the stored copy. @@ -261,18 +233,9 @@ func (w *ActorWorkflow) ensureVolumesCreated(ctx context.Context, actorRef resou return storedActor, nil } -// ensureWorkerAssigned leaves the actor RESUMING with a validated, live, -// owned, and eligible worker. -// -// A RESUMING actor was assigned by a previous attempt: its persisted -// assignment is revalidated (worker still exists, not draining, still owned -// by this actor UID, still eligible for the actor's constraints) and reused; -// a stale assignment crashes the actor. A SUSPENDED actor goes through -// scheduling (pinned to its assigned node when it holds a node-local -// snapshot), claiming the worker and persisting RESUMING with the -// assignment. A version conflict there is retried under a bounded backoff -// only after re-reading the actor and revalidating it can still be resumed — -// the conflicting writer may have crashed, drained, or deleted it. +// ensureWorkerAssigned assigns a worker and leaves the actor RESUMING. +// Re-entry validates the recorded assignment before reuse. +// Version conflicts re-read the actor before retrying. func (w *ActorWorkflow) ensureWorkerAssigned(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (_ *ateapipb.Actor, _ *ateapipb.Worker, err error) { ctx, done := stepSpan(ctx, "AssignWorker") defer func() { err = done(err) }() @@ -468,7 +431,7 @@ func (w *ActorWorkflow) workerHoldingStaleClaim(ctx context.Context, actor *atea } // The Actor's allocation makes HasRoom unsuitable for an existing claim. - if w.appliesWithFallback(worker, constraints, relaxable) { + if worker.GetNodeName() != "" && w.appliesWithFallback(worker, constraints, relaxable) { return worker, nil } @@ -493,6 +456,10 @@ func schedulerRecordable(err error) bool { // actor is returned along with the conflict so the caller retries with clean // inputs; any other status aborts the resume. func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (_ *ateapipb.Actor, _ *ateapipb.Worker, err error) { + // Checked before a worker is claimed. + if err := checkSnapshotURI(actorRef, actor); err != nil { + return nil, nil, err + } start := time.Now() outcome := ateattr.SchedulerOutcomeError poolNamespace := "" @@ -535,6 +502,10 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour constraints.RequiredRuntime = hardware.Requirement{} } + if assignedWorker.GetNodeName() == "" { + return nil, nil, apierror.FailedPrecondition("worker %s has no node name", assignedWorker.GetMetadata().GetName()) + } + assignment := &ateapipb.ActorAssignment{ Actor: &ateapipb.ObjectRef{ Atespace: actor.GetMetadata().GetAtespace(), @@ -599,7 +570,7 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour } poolNamespace = bound.GetWorkerNamespace() pool = bound.GetWorkerPool() - class = bound.GetSandboxClass() + class = hardware.SandboxClass(bound.GetStatus()) outcome = ateattr.SchedulerOutcomeAssigned logActorStateChanged(ctx, storedActor, ateattr.OperationResume) return storedActor, bound, nil @@ -631,13 +602,13 @@ func workerAssignmentFrom(w *ateapipb.Worker, snapRuntime hardware.Requirement) } } -// toAteletSandboxRuntime converts the control plane's SandboxRuntime to +// toAteletSandboxRuntime converts the control plane's ActorSandboxRuntime to // atelet's, which it mirrors field for field. -func toAteletSandboxRuntime(in *ateapipb.SandboxRuntime) *ateletpb.SandboxRuntime { +func toAteletSandboxRuntime(in *ateapipb.ActorSandboxRuntime) *ateletpb.ActorSandboxRuntime { if in == nil { return nil } - out := &ateletpb.SandboxRuntime{SandboxClass: in.GetSandboxClass()} + out := &ateletpb.ActorSandboxRuntime{SandboxClass: in.GetSandboxClass()} if v := in.GetVersion(); v != nil { out.Version = &ateletpb.VersionedSandboxCompat{SchemaVersion: v.GetSchemaVersion()} for _, a := range v.GetAttributes() { @@ -648,15 +619,14 @@ func toAteletSandboxRuntime(in *ateapipb.SandboxRuntime) *ateletpb.SandboxRuntim } func schedulingConstraints(actor *ateapipb.Actor, tmpl *ateapipb.ActorTemplate) scheduling.Constraints { + snap := snapshotToResume(actor.GetStatus()) var requiredNodes []string - // A suspended Actor with a local snapshot stays pinned to its node until - // atelet detaches it. When the node is full, ResumeActor detaches it. if node := actor.GetStatus().GetAssignedNode(); node != "" { requiredNodes = []string{node} } c := scheduling.Constraints{ - SandboxClass: snapshotSandboxClass(actor), - RequiredRuntime: hardware.NewRequirement(requiredSnapshotRuntime(actor, tmpl)), + SandboxClass: snap.GetSandboxRuntime().GetSandboxClass(), + RequiredRuntime: hardware.NewRequirement(requiredSnapshotRuntime(snap, tmpl)), ActorSelector: labels.SelectorFromSet(labels.Set(actor.GetWorkerSelector().GetMatchLabels())), RequiredNodes: requiredNodes, Limits: tmpl.GetResources().GetLimits(), @@ -667,60 +637,49 @@ func schedulingConstraints(actor *ateapipb.Actor, tmpl *ateapipb.ActorTemplate) return c } -// snapshotSandboxClass returns the sandbox class of the snapshot ResumeActor -// will restore from, the actor's local snapshot or else its latest durable -// one (a golden tag borrowed at creation included), or "" when the actor has -// none and cold-boots. Unlike requiredSnapshotRuntime it does not depend on -// fidelity: a worker of another class cannot read the snapshot's files even -// for a volumes-only restore. +// snapshotToResume returns the snapshot ResumeActor restores from: the latest +// one that is local or has a completed durable copy (a tag borrowed at +// creation included), or nil when the actor has none and cold-boots. +func snapshotToResume(status *ateapipb.ActorStatus) *ateapipb.Snapshot { + return latestSnapshot(status, func(s *ateapipb.Snapshot) bool { + return s.GetLocality() != "" || s.GetDurableSnapshot().GetStatus() == ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + }) +} + +// snapshotSandboxClass returns the sandbox class of snapshotToResume, or "" +// when the actor has none and cold-boots. Unlike requiredSnapshotRuntime it +// does not depend on fidelity: a worker of another class cannot read the +// snapshot's files even for a volumes-only restore. func snapshotSandboxClass(actor *ateapipb.Actor) string { - if localSnap := findLatestLocalSnapshot(actor.GetStatus()); localSnap != nil { - return localSnap.GetSandboxRuntime().GetSandboxClass() - } - if snap := findLatestDurableSnapshot(actor.GetStatus(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); snap != nil { - return snap.GetSandboxRuntime().GetSandboxClass() - } - return "" + return snapshotToResume(actor.GetStatus()).GetSandboxRuntime().GetSandboxClass() } -// requiredSnapshotRuntime returns the SandboxRuntime stamped on the snapshot -// that ResumeActor will restore memory state from, or nil when the resume does -// not restore memory state (cold boot, ROOTFS or VOLUMES fidelity, or replaced -// template). -func requiredSnapshotRuntime(actor *ateapipb.Actor, tmpl *ateapipb.ActorTemplate) *ateapipb.SandboxRuntime { - if localSnap := findLatestLocalSnapshot(actor.GetStatus()); localSnap != nil { - if localSnap.GetFidelity() != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { - return nil - } - if templateReplaced(localSnap, tmpl) { - return nil - } - return localSnap.GetSandboxRuntime() - } - if snap := findLatestDurableSnapshot(actor.GetStatus(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); snap != nil { - if snap.GetFidelity() != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { - return nil - } - if templateReplaced(snap, tmpl) { - return nil - } - return snap.GetSandboxRuntime() +// requiredSnapshotRuntime returns the ActorSandboxRuntime stamped on snap, the +// snapshotToResume, when the resume restores its memory state, or nil when it +// does not (cold boot, ROOTFS or VOLUMES fidelity, or replaced template). +func requiredSnapshotRuntime(snap *ateapipb.Snapshot, tmpl *ateapipb.ActorTemplate) *ateapipb.ActorSandboxRuntime { + if snap.GetFidelity() != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY || templateReplaced(snap, tmpl) { + return nil } - return nil + return snap.GetSandboxRuntime() } -// canRelaxRuntime reports whether a resume may drop its RequiredRuntime -// constraint and fall back to restoring without memory state when no -// runtime-compatible worker has capacity. A local pause checkpoint is not -// relaxed: the resume stays pinned to its node, the local restore has no -// fidelity downgrade, and nothing invalidates the checkpoint files if the actor -// boots elsewhere. The pin lifts once atelet uploads the checkpoint and -// detaches it from the node. +// canRelaxRuntime reports whether scheduling may drop its runtime requirement. func canRelaxRuntime(actor *ateapipb.Actor, constraints scheduling.Constraints) bool { if constraints.RequiredRuntime.Runtime() == nil { return false } - return findLatestLocalSnapshot(actor.GetStatus()) == nil + status := actor.GetStatus() + // A pinned actor detaches first, so a compatible worker elsewhere can keep + // its memory state. + if status.GetState() == ateapipb.ActorState_ACTOR_STATE_SUSPENDED && status.GetAssignedNode() != "" { + return false + } + snap := snapshotToResume(status) + if snap.GetLocality() == "" { + return true + } + return snap.GetDurableSnapshot().GetStatus() == ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED } // appliesWithFallback reports whether worker satisfies constraints, or — when @@ -736,8 +695,9 @@ func (w *ActorWorkflow) appliesWithFallback(worker *ateapipb.Worker, constraints // ensureVolumesAttached attaches the actor's mounted external volumes to the // assigned worker's node and returns each driver's attachment metadata by -// volume name, for the Restore call that follows. Attachment is idempotent, so -// a re-entered workflow safely runs it again. +// volume name, for the Restore call that follows. Every volume must be +// provisioned with a storage volume ID; otherwise nothing is attached. +// Attachment is idempotent, so a re-entered workflow safely runs it again. // TODO replace re-execution with a proper check on the volumes' attach state. func (w *ActorWorkflow) ensureVolumesAttached(ctx context.Context, actor *ateapipb.Actor, worker *ateapipb.Worker, actorTemplate *ateapipb.ActorTemplate) (_ map[string]map[string]string, err error) { ctx, done := stepSpan(ctx, "AttachVolumes") @@ -753,8 +713,14 @@ func (w *ActorWorkflow) ensureVolumesAttached(ctx context.Context, actor *ateapi } ref := &ateapipb.ObjectRef{Atespace: actor.GetMetadata().GetAtespace(), Name: actor.GetMetadata().GetName()} + volumes := getMountedActorVolumes(ctx, ref, actor.GetStatus().GetExternalVolumes().GetVolumes(), actorTemplate) + // Check every volume before attaching any, so a bad one cannot leave the + // ones before it published. + if err := checkVolumesAttachable(volumes); err != nil { + return nil, err + } volumePublishContexts := make(map[string]map[string]string) - for _, vol := range getMountedActorVolumes(ctx, ref, actor.GetStatus().GetExternalVolumes().GetVolumes(), actorTemplate) { + for _, vol := range volumes { slog.InfoContext(ctx, "Attaching volume to node", slog.String("volume_id", vol.GetStorageVolumeId()), slog.String("node", node)) plugin, err := w.pluginRegistry.GetPlugin(ctx, vol.GetVolumeType()) if err != nil { @@ -771,17 +737,33 @@ func (w *ActorWorkflow) ensureVolumesAttached(ctx context.Context, actor *ateapi return volumePublishContexts, nil } +// checkVolumesAttachable returns FailedPrecondition unless every volume has a +// storage volume ID. +func checkVolumesAttachable(volumes []*ateapipb.ExternalVolumeStatus) error { + for _, vol := range volumes { + if vol.GetStorageVolumeId() == "" { + return apierror.FailedPrecondition("volume %q has no storage volume ID to attach", vol.GetName()) + } + } + return nil +} + // ensureAteletRestored brings the workload up on the assigned worker: -// restoring the actor's local snapshot when one exists, else its external -// snapshot, else cold-booting from the template spec. This is the -// atelet reentrancy seam (#372): the request is keyed by the actor UID and +// restoring snapshotToResume, or cold-booting from the template spec when the +// actor has no snapshot. This is the atelet reentrancy seam (#372): the +// request is keyed by the actor UID and // the worker pod UID, so a re-entered workflow re-sends the same semantic // request; once atelet's Restore/Run are idempotent on those keys this step // becomes fully reentrant with no changes here. -func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate, volumePublishContexts map[string]map[string]string, src resumeSnapshotSource) (tele restoreTelemetry, err error) { +func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate, volumePublishContexts map[string]map[string]string) (tele restoreTelemetry, err error) { ctx, done := stepSpan(ctx, "CallAteletRestore") defer func() { err = done(err) }() + // A re-entered resume skips assignWorkerAttempt, which checks this too. + if err := checkSnapshotURI(actorRef, actor); err != nil { + return tele, err + } + assignment := actor.GetStatus().GetWorkerAssignment() ateletConn, err := w.dialer.DialForAteletOnNode(assignment.GetNodeName()) if err != nil { @@ -799,87 +781,14 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou // to the actor (replacing the worker-pod downward-API approach). actorResources := toAteletResources(actorTemplate.GetResources()) + snap := snapshotToResume(actor.GetStatus()) + // The assignment is what the next checkpoint is stamped from, so it is also // what decides how this restore runs. sandboxRuntime := toAteletSandboxRuntime(assignment.GetSandboxRuntime()) - runtimeIncompatible := src.RequiredRuntime != nil && !hardware.Matches(assignment.GetSandboxRuntime(), src.RequiredRuntime) - if localSnap := findLatestLocalSnapshot(actor.GetStatus()); localSnap != nil { - slog.InfoContext(ctx, "Actor has snapshot; Restoring from snapshot") - tele.SnapshotKind = ateattr.SnapshotKindLocal - fidelity := fidelityToAtelet(localSnap.GetFidelity()) - if templateReplaced(localSnap, actorTemplate) { - fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES - } - req := &ateletpb.RestoreRequest{ - WorkerPodUid: assignment.GetWorkerPodUid(), - Atespace: actor.GetMetadata().GetAtespace(), - ActorName: actor.GetMetadata().GetName(), - ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), - ActorTemplateName: actor.GetActorTemplate().GetName(), - Spec: workloadSpec, - Snapshot: &ateletpb.Snapshot{ - SnapshotUuid: localSnap.GetUuid(), - Generation: localSnap.GetGeneration(), - Fidelity: fidelity, - // Named even while the upload is PENDING: atelet may have - // uploaded the snapshot and detached the actor before its - // report of the upload lands. atelet reads it only when it - // holds no usable local copy, and the manifest, uploaded - // last, tells a complete upload from a partial one. - Object: &ateletpb.ObjectStorage{SnapshotUri: localSnap.GetDurableSnapshot().GetObject().GetSnapshotUri()}, - }, - SandboxRuntime: sandboxRuntime, - ActorUid: actor.GetMetadata().Uid, - EgressGateway: egressGateway, - Resources: actorResources, - } - tele.WireFidelity = ateattr.SnapshotFidelityValue(fidelity) - - if _, err = client.Restore(ctx, req); err != nil { - return tele, handleAteletError(ctx, w.store, actorRef, ateattr.OperationResume, "Restore", false, err) - } - return tele, nil - } else if !src.SnapshotURI.IsZero() { - slog.InfoContext(ctx, "Actor has durable snapshot; Restoring from snapshot") - tele.SnapshotKind = ateattr.SnapshotKindLatest - scope := fidelityToAtelet(src.Fidelity) - // Exclusive by construction: requiredSnapshotRuntime returns nil when - // the template changed, so runtimeIncompatible is false when - // src.TemplateReplaced is true. - switch { - case src.TemplateReplaced: - scope = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES - case runtimeIncompatible: - logRuntimeFallback(ctx, actorRef, assignment, src.RequiredRuntime) - scope = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS - } - tele.WireFidelity = ateattr.SnapshotFidelityValue(scope) - req := &ateletpb.RestoreRequest{ - WorkerPodUid: assignment.GetWorkerPodUid(), - Atespace: actor.GetMetadata().GetAtespace(), - ActorName: actor.GetMetadata().GetName(), - ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), - ActorTemplateName: actor.GetActorTemplate().GetName(), - Spec: workloadSpec, - Snapshot: &ateletpb.Snapshot{ - SnapshotUuid: src.SnapshotUUID, - Generation: src.Generation, - Fidelity: scope, - Object: &ateletpb.ObjectStorage{ - SnapshotUri: src.SnapshotURI.String(), - }, - }, - SandboxRuntime: sandboxRuntime, - ActorUid: actor.GetMetadata().Uid, - EgressGateway: egressGateway, - Resources: actorResources, - } - if _, err = client.Restore(ctx, req); err != nil { - return tele, handleAteletError(ctx, w.store, actorRef, ateattr.OperationResume, "Restore", false, err) - } - return tele, nil - } else { + // New actor without any existing snapshots to restore from. + if snap == nil { slog.InfoContext(ctx, "Actor has no snapshot; Booting from ActorTemplate spec") tele.SnapshotKind = ateattr.SnapshotKindBoot @@ -900,12 +809,64 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou } return tele, nil } + + slog.InfoContext(ctx, "Actor has snapshot; Restoring from snapshot", slog.String("snapshot_uuid", snap.GetUuid())) + tele.SnapshotKind = ateattr.SnapshotKindLatest + if snap.GetLocality() != "" { + tele.SnapshotKind = ateattr.SnapshotKindLocal + } + + // Exclusive by construction: requiredSnapshotRuntime returns nil when the + // template changed, so runtimeIncompatible is false when templateReplaced + // is true. + requiredRuntime := requiredSnapshotRuntime(snap, actorTemplate) + runtimeIncompatible := requiredRuntime != nil && !hardware.Matches(assignment.GetSandboxRuntime(), requiredRuntime) + + fidelity := fidelityToAtelet(snap.GetFidelity()) + switch { + case templateReplaced(snap, actorTemplate): + fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + case runtimeIncompatible: + logRuntimeFallback(ctx, actorRef, assignment, requiredRuntime) + fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + } + tele.WireFidelity = ateattr.SnapshotFidelityValue(fidelity) + req := &ateletpb.RestoreRequest{ + WorkerPodUid: assignment.GetWorkerPodUid(), + Atespace: actor.GetMetadata().GetAtespace(), + ActorName: actor.GetMetadata().GetName(), + ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), + ActorTemplateName: actor.GetActorTemplate().GetName(), + Spec: workloadSpec, + Snapshot: &ateletpb.Snapshot{ + SnapshotUuid: snap.GetUuid(), + Generation: snap.GetGeneration(), + Fidelity: fidelity, + // A local snapshot's upload is named even while PENDING: atelet + // may have uploaded the snapshot and detached the actor before + // its report of the upload lands. atelet reads it only when it + // holds no usable local copy, and the manifest, uploaded last, + // tells a complete upload from a partial one. + Object: &ateletpb.ObjectStorage{ + SnapshotUri: snap.GetDurableSnapshot().GetObject().GetSnapshotUri(), + Sharing: snapshotSharing(snap.GetOwner()), + }, + }, + SandboxRuntime: sandboxRuntime, + ActorUid: actor.GetMetadata().Uid, + EgressGateway: egressGateway, + Resources: actorResources, + } + if _, err = client.Restore(ctx, req); err != nil { + return tele, handleAteletError(ctx, w.store, actorRef, ateattr.OperationResume, "Restore", false, err) + } + return tele, nil } // logRuntimeFallback records a resume that restores without memory state // because the assigned worker does not run the runtime the snapshot's memory // state was captured under. -func logRuntimeFallback(ctx context.Context, actorRef resources.ActorRef, assignment *ateapipb.WorkerAssignment, required *ateapipb.SandboxRuntime) { +func logRuntimeFallback(ctx context.Context, actorRef resources.ActorRef, assignment *ateapipb.WorkerAssignment, required *ateapipb.ActorSandboxRuntime) { slog.WarnContext(ctx, "Assigned worker cannot restore the snapshot's runtime; restoring without memory state", slog.String("actor", actorRef.String()), slog.String("worker", assignment.GetWorkerPod()), diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_detach.go b/cmd/ateapi/internal/controlapi/workflow_resume_detach.go index e80e8fe167..665ed311d5 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_detach.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_detach.go @@ -23,6 +23,7 @@ import ( "go.ate.dev/substrate/cmd/ateapi/internal/scheduling" "go.ate.dev/substrate/cmd/ateapi/internal/store" "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" "go.ate.dev/substrate/internal/hardware" "go.ate.dev/substrate/internal/proto/ateletpb" "go.ate.dev/substrate/internal/resources" @@ -40,12 +41,9 @@ func shouldDetachFromNode(actor *ateapipb.Actor, assignErr error) bool { actor.GetStatus().GetAssignedNode() != "" } -// detachFromFullNode detaches a SUSPENDED actor from its full node when -// another node has room, so the resume restores it there from the durable -// snapshot, and returns the actor as stored. A worker of another runtime -// counts: it restores the snapshot at ROOTFS, without its process memory, and -// assignment still prefers one that keeps it. With no room anywhere it returns -// assignErr. A failed detach leaves the actor on its node. +// detachFromFullNode releases the node binding so scheduling can retry. +// Unreachable nodes can be forgotten when the latest local snapshot is durable. +// A deleted node crashes the actor when that snapshot is not durable. func (w *ActorWorkflow) detachFromFullNode(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate, assignErr error) (_ *ateapipb.Actor, err error) { ctx, done := stepSpan(ctx, "DetachFromFullNode") defer func() { err = done(err) }() @@ -69,6 +67,22 @@ func (w *ActorWorkflow) detachFromFullNode(ctx context.Context, actorRef resourc // their assignments; the loser is left durable-only. ateletConn, err := w.dialer.DialForAteletOnNode(node) if err != nil { + if canDetachWithoutAtelet(ctx, actor, err) { + return w.recordDetach(ctx, actorRef, actor, node, nil) + } + if errors.Is(err, ErrNoAteletOnNode) && latestLocalNotDurable(actor) { + gone, nodeErr := w.dialer.NodeGone(ctx, node) + if nodeErr != nil { + // The caller went away or the lease was lost: report that. + if ctx.Err() != nil { + return nil, fmt.Errorf("while checking node %q: %w: %w", node, nodeErr, ctx.Err()) + } + return nil, apierror.Unavailable("while getting atelet conn for node %q: %w (and while checking the node: %w)", node, err, nodeErr) + } + if gone { + return w.recordGoneNode(ctx, actorRef, actor, node) + } + } return nil, apierror.Unavailable("while getting atelet conn for node %q: %w", node, err) } resp, err := ateletpb.NewAteletClient(ateletConn).DetachActor(ctx, &ateletpb.DetachActorRequest{ @@ -77,6 +91,9 @@ func (w *ActorWorkflow) detachFromFullNode(ctx context.Context, actorRef resourc ActorUid: actor.GetMetadata().GetUid(), }) if err != nil { + if canDetachWithoutAtelet(ctx, actor, err) { + return w.recordDetach(ctx, actorRef, actor, node, nil) + } return nil, detachActorError(ctx, err) } state := resp.GetState() @@ -84,8 +101,8 @@ func (w *ActorWorkflow) detachFromFullNode(ctx context.Context, actorRef resourc return nil, apierror.Internal("atelet on node %q detached Actor %s but still reports a local snapshot", node, actorRef) } - // A conflict fails the resume with Aborted. The next resume detaches - // again, and atelet returns the same state. + // A conflict fails the resume with Aborted. atelet's report of the detach + // records it, and so does the next resume, which detaches again. stored, err := w.recordDetach(ctx, actorRef, actor, node, state) if err != nil { return nil, err @@ -111,20 +128,20 @@ func (w *ActorWorkflow) loadPinnedActor(ctx context.Context, actorRef resources. return actor, nil } -// recordDetach records the state atelet on node returned from DetachActor on -// the actor as read, and returns it as stored. It fails with -// FailedPrecondition if the latest copy on node was not durable. +// recordDetach forgets local copies after the latest local generation is durable. +// state may be nil when atelet is unreachable. func (w *ActorWorkflow) recordDetach(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, node string, state *ateletpb.ActorSnapshotState) (*ateapipb.Actor, error) { applied := proto.CloneOf(actor.GetStatus()) if err := applyLatestDurable(ctx, applied, snapshotReportFromAtelet(state.GetLatestDurable())); err != nil { return nil, apierror.Internal("atelet on node %q reported a state for Actor %s that conflicts with its record: %w", node, actorRef, err) } - // Under the lease, atelet deleted every copy on node, including any newer - // than the snapshot it reports durable, and reports none when it holds no - // entry for the actor. + // Forget local copies only after the latest recorded generation is durable. if err := applyDetach(applied, node, applied.GetLastAssignedGeneration()); err != nil { return nil, err } + if applied.GetState() == ateapipb.ActorState_ACTOR_STATE_REVERTING && applied.GetWorkerAssignment() == nil { + applied.AssignedNode = "" + } stored, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { // The precondition makes toUpdate the version applied was built from. toUpdate.Status.Snapshots = applied.GetSnapshots() @@ -140,6 +157,52 @@ func (w *ActorWorkflow) recordDetach(ctx context.Context, actorRef resources.Act return stored, nil } +// recordGoneNode crashes an actor whose latest snapshot went with its deleted +// node. It reads the actor again first, so an upload recorded during the node +// lookup detaches it instead, and it crashes only the version it checked. +func (w *ActorWorkflow) recordGoneNode(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, node string) (*ateapipb.Actor, error) { + actor, err := w.loadPinnedActor(ctx, actorRef, actor.GetMetadata().GetUid(), node) + if err != nil { + return nil, err + } + if !latestLocalNotDurable(actor) { + return w.recordDetach(ctx, actorRef, actor, node, nil) + } + if err := crashActorVersion(ctx, w.store, actorRef, actor, ateattr.OperationResume, crashMessageLocalSnapshotNodeGone); err != nil { + if errors.Is(err, store.ErrVersionConflict) { + return nil, apierror.Aborted("Actor %s changed during the resume, please retry", actorRef) + } + return nil, err + } + return nil, apierror.FailedPrecondition("actor %s is CRASHED: %s", actorRef, crashMessageLocalSnapshotNodeGone) +} + +// latestLocalNotDurable reports whether the actor's latest local snapshot has +// no completed durable copy. +func latestLocalNotDurable(actor *ateapipb.Actor) bool { + local := findLatestLocalSnapshot(actor.GetStatus()) + return local != nil && local.GetDurableSnapshot().GetStatus() != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED +} + +func canDetachWithoutAtelet(ctx context.Context, actor *ateapipb.Actor, err error) bool { + if ctx.Err() != nil || actor.GetStatus().GetWorkerAssignment() != nil { + return false + } + // A failed restore can leave a live workload after its assignment is cleared. + if actor.GetStatus().GetCrash() != nil { + return false + } + code := status.Code(err) + if code == codes.Unknown { + code = apierror.Code(err) + } + if !errors.Is(err, ErrNoAteletOnNode) && code != codes.Unavailable && code != codes.DeadlineExceeded { + return false + } + local := findLatestLocalSnapshot(actor.GetStatus()) + return local != nil && local.GetDurableSnapshot().GetStatus() == ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED +} + // detachActorError maps a DetachActor error to the one the resume fails with. func detachActorError(ctx context.Context, err error) error { // The caller went away or the lease was lost: report that, not atelet's error. diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go index e11902b4e0..74b79e4ac1 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go @@ -17,6 +17,7 @@ package controlapi import ( "context" "errors" + "strings" "testing" "time" @@ -114,11 +115,20 @@ func TestDetachFromFullNode(t *testing.T) { // otherNode adds a free worker on node-2 with otherRuntime, or the // test runtime when unset. otherNode bool - otherRuntime *ateapipb.SandboxRuntime + otherRuntime *ateapipb.ActorSandboxRuntime // snapRuntime stamps the MEMORY snapshot. - snapRuntime *ateapipb.SandboxRuntime + snapRuntime *ateapipb.ActorSandboxRuntime // completed records the snapshot's upload COMPLETED. completed bool + cancel bool + noIPs bool + // nodeGone deletes node's Node object. nodeLookupErr fails every + // Node read, and cancelOnNodeLookup cancels the step during one. + // completeOnNodeLookup records the upload COMPLETED during one. + nodeGone bool + nodeLookupErr error + cancelOnNodeLookup bool + completeOnNodeLookup bool // before updates the actor after it was read and before the step // runs, and during while atelet detaches it. before, during func(*ateapipb.Actor) @@ -126,8 +136,11 @@ func TestDetachFromFullNode(t *testing.T) { detachState func(*ateapipb.Snapshot) *ateletpb.ActorSnapshotState detachErr error wantDetached bool + wantCrashed bool wantCode codes.Code - wantCall bool + // wantErrText is text the error must contain. + wantErrText string + wantCall bool }{{ name: "another node has room: detaches", node: "node-1", @@ -196,11 +209,106 @@ func TestDetachFromFullNode(t *testing.T) { detachState: detachedState, wantDetached: true, wantCall: true, + }, { + name: "no atelet on the node, upload completed: detaches", + node: "node-3", + otherNode: true, + completed: true, + wantDetached: true, + }, { + name: "atelet unreachable, upload completed: detaches", + node: "node-1", + otherNode: true, + completed: true, + detachErr: apierror.Unavailable("connection refused"), + wantDetached: true, + wantCall: true, + }, { + name: "atelet deadline exceeded, upload completed: detaches", + node: "node-1", + otherNode: true, + completed: true, + detachErr: context.DeadlineExceeded, + wantDetached: true, + wantCall: true, + }, { + name: "actor updated while unreachable: Aborted, record unchanged", + node: "node-1", + otherNode: true, + completed: true, + during: func(*ateapipb.Actor) {}, + detachErr: apierror.Unavailable("connection refused"), + wantCode: codes.Aborted, + wantCall: true, + }, { + name: "workflow canceled during detach: actor stays on its node", + node: "node-1", + otherNode: true, + completed: true, + cancel: true, + detachErr: apierror.Unavailable("connection refused"), + wantCode: codes.Canceled, + }, { + name: "atelet missing IP: actor stays on its node", + node: "node-1", + otherNode: true, + completed: true, + noIPs: true, + wantCode: codes.Unavailable, }, { name: "no atelet on the node: Unavailable, actor stays on its node", node: "node-3", otherNode: true, wantCode: codes.Unavailable, + }, { + name: "node gone, upload completed: detaches", + node: "node-3", + otherNode: true, + completed: true, + nodeGone: true, + wantDetached: true, + }, { + name: "node gone, upload PENDING: crashes on its node", + node: "node-3", + otherNode: true, + nodeGone: true, + wantCrashed: true, + wantCode: codes.FailedPrecondition, + }, { + name: "node gone, upload completes during the node lookup: detaches", + node: "node-3", + otherNode: true, + nodeGone: true, + completeOnNodeLookup: true, + wantDetached: true, + }, { + // A SUSPENDED actor never carries a crash record, so this state is + // synthetic. It checks that resume crashes only when the latest + // snapshot is not durable. + name: "node gone, crash recorded, upload completed: Unavailable, actor stays on its node", + node: "node-3", + otherNode: true, + completed: true, + nodeGone: true, + before: func(a *ateapipb.Actor) { + a.Status.Crash = &ateapipb.ActorCrash{Message: "restore failed"} + }, + wantCode: codes.Unavailable, + }, { + name: "node lookup fails: Unavailable, actor stays on its node", + node: "node-3", + otherNode: true, + nodeGone: true, + nodeLookupErr: errors.New("connection refused"), + wantCode: codes.Unavailable, + wantErrText: "while checking the node", + }, { + name: "workflow canceled during the node lookup: actor stays on its node", + node: "node-3", + otherNode: true, + nodeGone: true, + cancelOnNodeLookup: true, + wantCode: codes.Canceled, }, { name: "atelet refuses: Aborted, actor stays on its node", node: "node-1", @@ -240,7 +348,8 @@ func TestDetachFromFullNode(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) st := newTestPersistence(t) fillWorker(t, st, seedRuntimeWorker(t, ctx, st, "pod-"+tt.node, tt.node, testSandboxRuntime())) if tt.otherNode { @@ -257,6 +366,39 @@ func TestDetachFromFullNode(t *testing.T) { } actor := seedActorWithSnapshot(t, ctx, st, suspended, tt.node, snap) w, atelet := newDetachTestWorkflow(t, ctx, st) + if tt.noIPs { + w.dialer = NewAteletDialer(newTestAteletIndexer(t, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: installdefaults.SystemNamespace, Name: "atelet-1", UID: "atelet-uid"}, + Spec: corev1.PodSpec{NodeName: tt.node}, + }), installdefaults.SystemNamespace, "", "") + } + nodes := []string{"node-1", "node-2"} + if !tt.nodeGone { + nodes = append(nodes, tt.node) + } + getNode := nodeGetter(tt.nodeLookupErr, nodes...) + if tt.cancelOnNodeLookup { + getNode = func(ctx context.Context, _ string) error { + cancel() + return ctx.Err() + } + } + if tt.completeOnNodeLookup { + lookup := getNode + getNode = func(ctx context.Context, name string) error { + cur, err := st.GetActor(ctx, actorRef) + if err != nil { + t.Errorf("GetActor: %v", err) + } else if _, err := st.UpdateActor(ctx, actorRef, store.PreconditionFrom(cur), func(a *ateapipb.Actor) error { + a.Status.Snapshots[0].DurableSnapshot.Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + return nil + }); err != nil { + t.Errorf("UpdateActor: %v", err) + } + return lookup(ctx, name) + } + } + WithNodeGetter(getNode)(w.dialer) if tt.detachState != nil { atelet.detachState = tt.detachState(snap) } @@ -281,6 +423,9 @@ func TestDetachFromFullNode(t *testing.T) { if tt.during != nil { atelet.onDetach = func() { update(tt.during) } } + if tt.cancel { + atelet.onDetach = cancel + } // The record unchanged by a failed step is the one as last written. want, err := st.GetActor(ctx, actorRef) if err != nil { @@ -291,8 +436,11 @@ func TestDetachFromFullNode(t *testing.T) { if got := apierror.Code(err); got != tt.wantCode { t.Fatalf("detachFromFullNode() = %v, want code %v", err, tt.wantCode) } + if tt.wantErrText != "" && !strings.Contains(err.Error(), tt.wantErrText) { + t.Fatalf("detachFromFullNode() = %v, want it to contain %q", err, tt.wantErrText) + } req := atelet.detachRequest() - if (req != nil) != tt.wantCall { + if !tt.cancel && (req != nil) != tt.wantCall { t.Fatalf("DetachActor request = %v, want a call: %v", req, tt.wantCall) } if req != nil { @@ -302,10 +450,24 @@ func TestDetachFromFullNode(t *testing.T) { } } - after, err := st.GetActor(ctx, actorRef) + after, err := st.GetActor(context.Background(), actorRef) if err != nil { t.Fatalf("GetActor: %v", err) } + if tt.wantCrashed { + // Only the state and crash change. The node and the PENDING + // snapshot stay for revert to clean up. + crashed := proto.CloneOf(want.GetStatus()) + crashed.State = ateapipb.ActorState_ACTOR_STATE_CRASHED + crashed.Crash = after.GetStatus().GetCrash() + if !proto.Equal(after.GetStatus(), crashed) { + t.Errorf("status = %v, want %v", after.GetStatus(), crashed) + } + if got := after.GetStatus().GetCrash().GetMessage(); !strings.Contains(got, crashMessageLocalSnapshotNodeGone) { + t.Errorf("crash message = %q, want %q", got, crashMessageLocalSnapshotNodeGone) + } + return + } if !tt.wantDetached { if !proto.Equal(after.GetStatus(), want.GetStatus()) { t.Errorf("status = %v, want unchanged %v", after.GetStatus(), want.GetStatus()) @@ -364,7 +526,6 @@ func TestResumeActor_DetachesFromFullNode(t *testing.T) { a.Status.LastAssignedGeneration = 1 a.Status.Snapshots = []*ateapipb.Snapshot{proto.CloneOf(snap)} }) - w, atelet := newDetachTestWorkflow(t, ctx, st) addAteletOnNode(t, w, "node-2") atelet.detachState = detachedState(snap) @@ -394,7 +555,12 @@ func TestResumeActor_DetachesFromFullNode(t *testing.T) { SnapshotUuid: snap.GetUuid(), Generation: snap.GetGeneration(), Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, - Object: &ateletpb.ObjectStorage{SnapshotUri: snap.GetDurableSnapshot().GetObject().GetSnapshotUri()}, + // The actor's own snapshot, so private, whether atelet + // downloads it or restores the local copy. + Object: &ateletpb.ObjectStorage{ + SnapshotUri: snap.GetDurableSnapshot().GetObject().GetSnapshotUri(), + Sharing: ateletpb.SnapshotSharing_SNAPSHOT_SHARING_PRIVATE, + }, } if !proto.Equal(restore.GetSnapshot(), want) { t.Errorf("Restore snapshot = %v, want %v", restore.GetSnapshot(), want) @@ -403,6 +569,64 @@ func TestResumeActor_DetachesFromFullNode(t *testing.T) { } } +// TestResumeActor_NodeGoneWithPendingUpload resumes a paused actor whose node +// was deleted before atelet uploaded its latest snapshot. The resume crashes +// the actor, a revert returns it to its older durable snapshot, and the next +// resume restores that snapshot on another node. +func TestResumeActor_NodeGoneWithPendingUpload(t *testing.T) { + ctx := context.Background() + st := newTestPersistence(t) + actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} + + storetest.MustCreateAtespace(t, ctx, st, "ns") + if _, err := st.CreateActorTemplate(ctx, &ateapipb.ActorTemplate{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "ns", Name: "tmpl1"}, + SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: testStorageLocation}, + }); err != nil { + t.Fatalf("create template: %v", err) + } + worker := seedRuntimeWorker(t, ctx, st, "pod-2", "node-2", testSandboxRuntime()) + memory := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + durable := newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", "durable-1", + someActorSnapshotURI(t, testStorageLocation, "team-a", "durable-1"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + paused := newPausedSnapshot(t, 2, memory, "", "pause-snap-2", "node-3") + seedWorkflowActor(t, ctx, st, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDED, func(a *ateapipb.Actor) { + a.Status.AssignedNode = "node-3" + a.Status.LastAssignedGeneration = 2 + a.Status.Snapshots = []*ateapipb.Snapshot{proto.CloneOf(durable), proto.CloneOf(paused)} + }) + w, atelet := newDetachTestWorkflow(t, ctx, st) + addAteletOnNode(t, w, "node-2") + WithNodeGetter(nodeGetter(nil, "node-1", "node-2"))(w.dialer) + + if _, _, err := w.ResumeActor(ctx, actorRef); apierror.Code(err) != codes.FailedPrecondition { + t.Fatalf("first ResumeActor = %v, want FailedPrecondition", err) + } + if got, err := st.GetActor(ctx, actorRef); err != nil || got.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_CRASHED { + t.Fatalf("after first resume: state %v, err %v, want CRASHED", got.GetStatus().GetState(), err) + } + + reverted, err := w.RevertActor(ctx, actorRef) + if err != nil { + t.Fatalf("RevertActor: %v", err) + } + if s := reverted.GetStatus(); s.GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED || s.GetAssignedNode() != "" { + t.Fatalf("after revert: state %v, assigned node %q, want SUSPENDED with no node", s.GetState(), s.GetAssignedNode()) + } + + got, resumed, err := w.ResumeActor(ctx, actorRef) + if err != nil { + t.Fatalf("second ResumeActor: %v", err) + } + if !resumed || got.GetStatus().GetAssignedNode() != "node-2" { + t.Fatalf("second ResumeActor = (node %q, resumed %v), want node-2 and resumed", got.GetStatus().GetAssignedNode(), resumed) + } + restore, _ := atelet.requests() + if restore.GetWorkerPodUid() != worker.GetWorkerPodUid() || restore.GetSnapshot().GetSnapshotUuid() != durable.GetUuid() { + t.Errorf("Restore = (worker %q, snapshot %q), want (%q, %q)", restore.GetWorkerPodUid(), restore.GetSnapshot().GetSnapshotUuid(), worker.GetWorkerPodUid(), durable.GetUuid()) + } +} + // addAteletOnNode routes node's atelet to the same capturing fake as node-1's. func addAteletOnNode(t *testing.T, w *ActorWorkflow, node string) { t.Helper() diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index 15a527569e..4e07ca3ef9 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "net" + "strings" "sync" "testing" "time" @@ -205,11 +206,10 @@ func TestAssignWorkerAttempt_SkipsWorkerAssignedInOtherAtespace(t *testing.T) { WorkerPool: "pool", WorkerPod: "pod-1", WorkerPodUid: testWorkerUID("pod-1"), - SandboxClass: "gvisor", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, - DefaultRuntime: testSandboxRuntime(), + SandboxClasses: workerSandboxClasses(testSandboxRuntime()), }, } if _, err := persistence.CreateWorker(ctx, worker); err != nil { @@ -246,6 +246,68 @@ func TestAssignWorkerAttempt_SkipsWorkerAssignedInOtherAtespace(t *testing.T) { } } +// TestEnsureWorkerAssigned_MalformedSnapshotURIClaimsNoWorker verifies that a +// snapshot URI that does not parse fails the resume before a worker is claimed, +// so the actor stays SUSPENDED and the worker stays free. +func TestEnsureWorkerAssigned_MalformedSnapshotURIClaimsNoWorker(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + + worker := &ateapipb.Worker{ + Metadata: &ateapipb.ResourceMetadata{Name: testWorkerUID("pod-1")}, + WorkerNamespace: "worker-ns", + WorkerPool: "pool", + WorkerPod: "pod-1", + WorkerPodUid: testWorkerUID("pod-1"), + Status: &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + SandboxClasses: workerSandboxClasses(testSandboxRuntime()), + }, + } + if _, err := persistence.CreateWorker(ctx, worker); err != nil { + t.Fatalf("CreateWorker: %v", err) + } + + cacheCtx, cancel := context.WithCancel(ctx) + defer cancel() + wc := workercache.New(persistence, time.Minute) + if err := wc.Start(cacheCtx); err != nil { + t.Fatalf("workercache.Start: %v", err) + } + + actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} + seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDED, func(a *ateapipb.Actor) { + a.Status.LastAssignedGeneration = 1 + a.Status.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snap-1", "not-a-valid-snapshot-uri", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } + }) + actor, err := persistence.GetActor(ctx, actorRef) + if err != nil { + t.Fatalf("GetActor: %v", err) + } + + w := &ActorWorkflow{store: persistence, workerCache: wc, scheduler: scheduling.New(wc)} + if _, _, err := w.ensureWorkerAssigned(ctx, actorRef, actor, &ateapipb.ActorTemplate{}); apierror.Code(err) != codes.DataLoss { + t.Fatalf("ensureWorkerAssigned() error = %v, want DataLoss", err) + } + + if got := firstAssignment(t, persistence, testWorkerUID("pod-1")); got != nil { + t.Errorf("worker assignment = %v, want none", got) + } + stored, err := persistence.GetActor(ctx, actorRef) + if err != nil { + t.Fatalf("GetActor: %v", err) + } + if got := stored.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + t.Errorf("stored state = %v, want SUSPENDED", got) + } + if got := stored.GetStatus().GetWorkerAssignment(); got != nil { + t.Errorf("stored WorkerAssignment = %v, want none", got) + } +} + // TestAssignWorkerAttempt_ReleasesIneligibleStaleWorker verifies that a worker // claimed by a previous failed attempt whose pool is no longer eligible is // released back to the free pool, without failing the resume, while a fresh @@ -267,11 +329,11 @@ func TestAssignWorkerAttempt_ReleasesIneligibleStaleWorker(t *testing.T) { WorkerPool: "pool-a", WorkerPod: "stale-pod", WorkerPodUid: testWorkerUID("stale-pod"), - SandboxClass: "gvisor", + NodeName: "node-1", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, - DefaultRuntime: testSandboxRuntime(), + SandboxClasses: workerSandboxClasses(testSandboxRuntime()), }, } free := &ateapipb.Worker{ @@ -280,12 +342,12 @@ func TestAssignWorkerAttempt_ReleasesIneligibleStaleWorker(t *testing.T) { WorkerPool: "pool-b", WorkerPod: "free-pod", WorkerPodUid: testWorkerUID("free-pod"), - SandboxClass: "gvisor", + NodeName: "node-1", Labels: map[string]string{"pool": "b"}, Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, - DefaultRuntime: testSandboxRuntime(), + SandboxClasses: workerSandboxClasses(testSandboxRuntime()), }, } for _, w := range []*ateapipb.Worker{stale, free} { @@ -341,11 +403,11 @@ func TestAssignWorkerAttempt_RetryAfterConflictPicksFreshWorker(t *testing.T) { WorkerPool: "pool", WorkerPod: "contested-pod", WorkerPodUid: testWorkerUID("contested-pod"), - SandboxClass: "gvisor", + NodeName: "node-1", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, - DefaultRuntime: wantRuntime, + SandboxClasses: workerSandboxClasses(wantRuntime), }, } fallback := &ateapipb.Worker{ @@ -354,11 +416,11 @@ func TestAssignWorkerAttempt_RetryAfterConflictPicksFreshWorker(t *testing.T) { WorkerPool: "pool", WorkerPod: "fallback-pod", WorkerPodUid: testWorkerUID("fallback-pod"), - SandboxClass: "gvisor", + NodeName: "node-1", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, - DefaultRuntime: wantRuntime, + SandboxClasses: workerSandboxClasses(wantRuntime), }, } for _, w := range []*ateapipb.Worker{contested, fallback} { @@ -449,11 +511,11 @@ func seedAssignFixture(t *testing.T, ctx context.Context, persistence store.Inte WorkerPool: "pool", WorkerPod: "pod-1", WorkerPodUid: testWorkerUID("pod-1"), - SandboxClass: "gvisor", + NodeName: "node-1", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, - DefaultRuntime: testSandboxRuntime(), + SandboxClasses: workerSandboxClasses(testSandboxRuntime()), }, }); err != nil { t.Fatalf("CreateWorker: %v", err) @@ -762,17 +824,16 @@ func TestValidateAssignedWorker(t *testing.T) { activeStatus := &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, - DefaultRuntime: testSandboxRuntime(), + SandboxClasses: workerSandboxClasses(testSandboxRuntime()), } - drainingStatus := &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_DRAINING, Capacity: &ateapipb.WorkerResources{Actors: 1}} - unavailableStatus := &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE, Capacity: &ateapipb.WorkerResources{Actors: 1}} + drainingStatus := &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_DRAINING, Capacity: &ateapipb.WorkerResources{Actors: 1}, SandboxClasses: workerSandboxClasses(testSandboxRuntime())} + unavailableStatus := &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE, Capacity: &ateapipb.WorkerResources{Actors: 1}, SandboxClasses: workerSandboxClasses(testSandboxRuntime())} tests := []struct { name string // workerStatus is the stored worker's status, nil for a worker that // is gone. workerStatus *ateapipb.WorkerStatus - sandboxClass string tmplSelector map[string]string assignment *ateapipb.ActorAssignment // wantCode is codes.OK when validateAssignedWorker must return nil. @@ -795,7 +856,6 @@ func TestValidateAssignedWorker(t *testing.T) { { name: "crashes actor and leaves worker untouched when worker is draining", workerStatus: drainingStatus, - sandboxClass: "gvisor", assignment: ownAssignment, wantCode: codes.Aborted, wantActorState: ateapipb.ActorState_ACTOR_STATE_CRASHED, @@ -805,7 +865,6 @@ func TestValidateAssignedWorker(t *testing.T) { { name: "leaves actor and worker untouched when worker is unavailable", workerStatus: unavailableStatus, - sandboxClass: "gvisor", assignment: ownAssignment, wantCode: codes.Unavailable, wantActorState: ateapipb.ActorState_ACTOR_STATE_RESUMING, @@ -814,7 +873,6 @@ func TestValidateAssignedWorker(t *testing.T) { { name: "crashes actor and leaves worker untouched when assigned to another actor", workerStatus: activeStatus, - sandboxClass: "gvisor", assignment: otherAssignment, wantCode: codes.Aborted, wantActorState: ateapipb.ActorState_ACTOR_STATE_CRASHED, @@ -824,7 +882,6 @@ func TestValidateAssignedWorker(t *testing.T) { { name: "crashes actor and leaves worker untouched when assigned to previous incarnation of same actor", workerStatus: activeStatus, - sandboxClass: "gvisor", assignment: staleIncarnationAssignment, wantCode: codes.Aborted, wantActorState: ateapipb.ActorState_ACTOR_STATE_CRASHED, @@ -834,7 +891,6 @@ func TestValidateAssignedWorker(t *testing.T) { { name: "crashes actor and leaves worker untouched when assignment is cleared", workerStatus: activeStatus, - sandboxClass: "gvisor", assignment: nil, wantCode: codes.Aborted, wantActorState: ateapipb.ActorState_ACTOR_STATE_CRASHED, @@ -844,7 +900,6 @@ func TestValidateAssignedWorker(t *testing.T) { { name: "passes for own eligible worker", workerStatus: activeStatus, - sandboxClass: "gvisor", assignment: ownAssignment, wantCode: codes.OK, wantActorState: ateapipb.ActorState_ACTOR_STATE_RESUMING, @@ -853,7 +908,6 @@ func TestValidateAssignedWorker(t *testing.T) { { name: "releases own ineligible worker and crashes actor", workerStatus: activeStatus, - sandboxClass: "gvisor", tmplSelector: map[string]string{"pool": "other"}, assignment: ownAssignment, wantCode: codes.Aborted, @@ -877,7 +931,6 @@ func TestValidateAssignedWorker(t *testing.T) { WorkerPool: "pool", WorkerPod: "pod-1", WorkerPodUid: testWorkerUID("pod-1"), - SandboxClass: tt.sandboxClass, Status: tt.workerStatus, }); err != nil { t.Fatalf("CreateWorker: %v", err) @@ -944,108 +997,6 @@ func TestValidateAssignedWorker(t *testing.T) { } } -// A golden tag becoming ready after creation does not change an actor's source. -func TestLoadActorForResume_DoesNotDefaultGolden(t *testing.T) { - ctx := context.Background() - persistence := newTestPersistence(t) - actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} - seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDED) - storetest.MustCreateAtespace(t, ctx, persistence, "ns") - if _, err := persistence.CreateActorTemplate(ctx, &ateapipb.ActorTemplate{ - Metadata: &ateapipb.ResourceMetadata{Atespace: "ns", Name: "tmpl1"}, - Status: &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ - GoldenTag: &ateapipb.ObjectRef{Atespace: "ns", Name: "golden"}, - }}, - }); err != nil { - t.Fatal(err) - } - w := &ActorWorkflow{store: persistence} - _, _, src, err := w.loadActorForResume(ctx, actorRef) - if err != nil || !src.SnapshotURI.IsZero() { - t.Fatalf("source = %+v, err = %v; want cold boot", src, err) - } -} - -// TestLoadActorForResume_TemplateReplaced covers the detection of a repointed -// actor: when ExternalSnapshot.actor_template_uid differs from the actor's -// current template UID, TemplateReplaced is set so the external restore -// downgrades to data-only. -func TestLoadActorForResume_TemplateReplaced(t *testing.T) { - actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} - - // "current" stands for the created template's own UID. - const current = "current" - - tests := []struct { - name string - snapshotUID string - noSnapshot bool - want bool - }{ - { - name: "snapshot taken under the current template", - snapshotUID: current, - }, - { - name: "snapshot taken under a replaced template", - snapshotUID: "some-other-uid", - want: true, - }, - { - name: "snapshot without a recorded template UID", - snapshotUID: "", - }, - { - name: "no durable snapshot", - noSnapshot: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() - persistence := newTestPersistence(t) - - storetest.MustCreateAtespace(t, ctx, persistence, "ns") - tmpl, err := persistence.CreateActorTemplate(ctx, &ateapipb.ActorTemplate{ - Metadata: &ateapipb.ResourceMetadata{Atespace: "ns", Name: "tmpl1"}, - }) - if err != nil { - t.Fatalf("create template: %v", err) - } - if tmpl.GetMetadata().GetUid() == "" { - t.Fatal("created template has no UID; the matching cases would be vacuous") - } - resolve := func(uid string) string { - if uid == current { - return tmpl.GetMetadata().GetUid() - } - return uid - } - - var seedOpts []func(*ateapipb.Actor) - if !tt.noSnapshot { - seedOpts = append(seedOpts, func(a *ateapipb.Actor) { - a.Status.LastAssignedGeneration = 1 - a.Status.Snapshots = []*ateapipb.Snapshot{ - newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, resolve(tt.snapshotUID), "snap-1", someActorSnapshotURI(t, testStorageLocation, actorRef.Atespace, "snap-1"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - } - }) - } - seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDED, seedOpts...) - - w := &ActorWorkflow{store: persistence} - _, _, src, err := w.loadActorForResume(ctx, actorRef) - if err != nil { - t.Fatalf("loadActorForResume: %v", err) - } - if src.TemplateReplaced != tt.want { - t.Errorf("src.TemplateReplaced = %v, want %v", src.TemplateReplaced, tt.want) - } - }) - } -} - func TestLoadActorForResume_RunningActorShortCircuits(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) @@ -1058,7 +1009,7 @@ func TestLoadActorForResume_RunningActorShortCircuits(t *testing.T) { w := &ActorWorkflow{store: persistence} - actor, tmpl, src, err := w.loadActorForResume(ctx, actorRef) + actor, tmpl, err := w.loadActorForResume(ctx, actorRef) if err != nil { t.Fatalf("loadActorForResume() unexpected error = %v", err) } @@ -1068,9 +1019,6 @@ func TestLoadActorForResume_RunningActorShortCircuits(t *testing.T) { if tmpl != nil { t.Errorf("expected nil template, got %v", tmpl) } - if !src.SnapshotURI.IsZero() { - t.Errorf("expected empty snapshot source, got %+v", src) - } } // capturingAtelet records the last request of each RPC it receives, so a test @@ -1103,7 +1051,9 @@ type capturingAtelet struct { detachState *ateletpb.ActorSnapshotState detachErr error // onDetach, when set, runs before DetachActor returns. - onDetach func() + onDetach func() + terminateErr error + onTerminate func() } func (f *capturingAtelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (*ateletpb.RestoreResponse, error) { @@ -1163,6 +1113,22 @@ func (f *capturingAtelet) DetachActor(ctx context.Context, req *ateletpb.DetachA return &ateletpb.DetachActorResponse{State: proto.CloneOf(f.detachState)}, nil } +func (f *capturingAtelet) Terminate(ctx context.Context, req *ateletpb.TerminateRequest) (*ateletpb.TerminateResponse, error) { + f.mu.Lock() + hook := f.onTerminate + f.mu.Unlock() + if hook != nil { + hook() + } + f.mu.Lock() + err := f.terminateErr + f.mu.Unlock() + if err != nil { + return nil, err + } + return f.UnimplementedAteletServer.Terminate(ctx, req) +} + func (f *capturingAtelet) detachRequest() *ateletpb.DetachActorRequest { f.mu.Lock() defer f.mu.Unlock() @@ -1293,7 +1259,12 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { localSnap := func(name string, fidelity ateapipb.SnapshotFidelity) *ateapipb.Snapshot { return newPausedSnapshot(t, 1, fidelity, "", name, "node-1") } - // pauseUploadURI is where atelet uploads the pause snapshot. + stampedLocalSnap := func(name string, fidelity ateapipb.SnapshotFidelity, rt *ateapipb.ActorSandboxRuntime) *ateapipb.Snapshot { + snap := localSnap(name, fidelity) + snap.SandboxRuntime = rt + return snap + } + // pauseUploadURI is where atelet uploads the local snapshot. pauseUploadURI := someActorSnapshotURI(t, testStorageLocation, "team-a", localSnapshotName) uploadedLocalSnap := func(name string, fidelity ateapipb.SnapshotFidelity) *ateapipb.Snapshot { snap := localSnap(name, fidelity) @@ -1313,20 +1284,22 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // actorSeed is the actor status a row persists before resuming. type actorSeed struct { - // localSnapshot seeds a completed local snapshot (the pause checkpoint); + // localSnapshot seeds a completed local snapshot; // a non-nil value also pins the actor to the snapshot's node. localSnapshot *ateapipb.Snapshot // externalSnapshot seeds a completed durable snapshot. externalSnapshot *ateapipb.Snapshot - // tmplUID seeds the template UID the snapshot's guest state was built - // on, stamped onto externalSnapshot: "current" stands for the created - // template's store-assigned UID (unknown until runtime), "" leaves the - // field unset, anything else mismatches (a repointed actor). + // tmplUID stamps externalSnapshot. "current" means the created template's UID. tmplUID string + // localTmplUID stamps localSnapshot independently. + localTmplUID string // sourceTag seeds Actor.SourceTag when testing explicit tag clones. sourceTag *ateapipb.ObjectRef // assignedRuntime seeds WorkerAssignment.SandboxRuntime. - assignedRuntime *ateapipb.SandboxRuntime + assignedRuntime *ateapipb.ActorSandboxRuntime + // externalNewer gives externalSnapshot a higher generation than + // localSnapshot. By default the local snapshot is newer. + externalNewer bool } // templateSeed is the ActorTemplate configuration a row persists. type templateSeed struct { @@ -1346,11 +1319,14 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { snapshotName string snapshotURI string scope ateletpb.SnapshotFidelity + // kind is the restore's snapshot kind label: local when the + // restored snapshot is on a node, otherwise latest. + kind string } amd64RT := testSandboxRuntime() arm64RT := testRuntime("arm64", "") - wantAteletRuntime := map[*ateapipb.SandboxRuntime]*ateletpb.SandboxRuntime{ + wantAteletRuntime := map[*ateapipb.ActorSandboxRuntime]*ateletpb.ActorSandboxRuntime{ amd64RT: { SandboxClass: "gvisor", Version: &ateletpb.VersionedSandboxCompat{ @@ -1366,7 +1342,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, }, } - stampedExtSnap := func(owner ateapipb.SnapshotOwner, uri string, fidelity ateapipb.SnapshotFidelity, rt *ateapipb.SandboxRuntime) *ateapipb.Snapshot { + stampedExtSnap := func(owner ateapipb.SnapshotOwner, uri string, fidelity ateapipb.SnapshotFidelity, rt *ateapipb.ActorSandboxRuntime) *ateapipb.Snapshot { snap := newDurableSnapshot(1, owner, fidelity, "", durableSnapshotName, uri, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) snap.SandboxRuntime = rt return snap @@ -1389,6 +1365,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { tmpl: templateSeed{golden: extSnap(goldenURI, fullScope)}, want: restoreWant{ snapshotName: durableSnapshotName, + kind: ateattr.SnapshotKindLatest, snapshotURI: goldenURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, @@ -1415,6 +1392,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { actor: actorSeed{externalSnapshot: extSnap(actorURI, fullScope)}, want: restoreWant{ snapshotName: durableSnapshotName, + kind: ateattr.SnapshotKindLatest, snapshotURI: actorURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, @@ -1427,6 +1405,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, want: restoreWant{ snapshotName: durableSnapshotName, + kind: ateattr.SnapshotKindLatest, snapshotURI: actorURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, @@ -1442,6 +1421,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, want: restoreWant{ snapshotName: durableSnapshotName, + kind: ateattr.SnapshotKindLatest, snapshotURI: actorURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, @@ -1451,6 +1431,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { actor: actorSeed{externalSnapshot: extSnap(actorURI, rootfsScope)}, want: restoreWant{ snapshotName: durableSnapshotName, + kind: ateattr.SnapshotKindLatest, snapshotURI: actorURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, }, @@ -1465,6 +1446,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, want: restoreWant{ snapshotName: durableSnapshotName, + kind: ateattr.SnapshotKindLatest, snapshotURI: actorURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, @@ -1474,6 +1456,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { actor: actorSeed{externalSnapshot: extSnap(actorURI, dataScope)}, want: restoreWant{ snapshotName: durableSnapshotName, + kind: ateattr.SnapshotKindLatest, snapshotURI: actorURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, @@ -1486,6 +1469,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { tmpl: templateSeed{golden: extSnap(goldenURI, fullScope)}, want: restoreWant{ snapshotName: durableSnapshotName, + kind: ateattr.SnapshotKindLatest, snapshotURI: actorURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, @@ -1498,6 +1482,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { actor: actorSeed{externalSnapshot: extSnap(actorURI, unspecScope)}, want: restoreWant{ snapshotName: durableSnapshotName, + kind: ateattr.SnapshotKindLatest, snapshotURI: actorURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, }, @@ -1508,8 +1493,8 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { want: restoreWant{code: codes.DataLoss}, }, { - // A pause snapshot's upload is PENDING until atelet reports it, - // but a local restore names it anyway, so atelet can download the + // An async suspend snapshot's upload is PENDING until atelet reports + // it, but a local restore names it anyway, so atelet can download the // snapshot if it detached the actor before the report landed. name: "13 Full pause snapshot restores locally as Full", actor: actorSeed{ @@ -1518,6 +1503,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, want: restoreWant{ snapshotName: localSnapshotName, + kind: ateattr.SnapshotKindLocal, snapshotURI: pauseUploadURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, @@ -1526,10 +1512,11 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { name: "14 local snapshot built on the current template stays Full", actor: actorSeed{ localSnapshot: localSnap(localSnapshotName, fullScope), - tmplUID: "current", + localTmplUID: "current", }, want: restoreWant{ snapshotName: localSnapshotName, + kind: ateattr.SnapshotKindLocal, snapshotURI: pauseUploadURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, @@ -1538,10 +1525,11 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { name: "14a repointed actor's Full local snapshot drops to Data", actor: actorSeed{ localSnapshot: localSnap(localSnapshotName, fullScope), - tmplUID: "mismatch", + localTmplUID: "mismatch", }, want: restoreWant{ snapshotName: localSnapshotName, + kind: ateattr.SnapshotKindLocal, snapshotURI: pauseUploadURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, @@ -1550,10 +1538,11 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { name: "14b repointed actor's Rootfs local snapshot drops to Data", actor: actorSeed{ localSnapshot: localSnap(localSnapshotName, rootfsScope), - tmplUID: "mismatch", + localTmplUID: "mismatch", }, want: restoreWant{ snapshotName: localSnapshotName, + kind: ateattr.SnapshotKindLocal, snapshotURI: pauseUploadURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, @@ -1565,6 +1554,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, want: restoreWant{ snapshotName: localSnapshotName, + kind: ateattr.SnapshotKindLocal, snapshotURI: pauseUploadURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, @@ -1579,6 +1569,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, want: restoreWant{ snapshotName: localSnapshotName, + kind: ateattr.SnapshotKindLocal, snapshotURI: pauseUploadURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, @@ -1593,13 +1584,14 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, want: restoreWant{ snapshotName: localSnapshotName, + kind: ateattr.SnapshotKindLocal, snapshotURI: pauseUploadURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, }, { // An older external snapshot's template mismatch does not affect a - // local pause restore captured under the current template. + // local restore captured under the current template. name: "18 local snapshot ignores an older external snapshot's template mismatch", actor: actorSeed{ localSnapshot: func() *ateapipb.Snapshot { @@ -1609,13 +1601,29 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }(), externalSnapshot: extSnap(actorURI, fullScope), tmplUID: "mismatch", + localTmplUID: "current", }, want: restoreWant{ snapshotName: localSnapshotName, + kind: ateattr.SnapshotKindLocal, snapshotURI: pauseUploadURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, }, + { + name: "18a newer durable snapshot wins over an older local snapshot", + actor: actorSeed{ + localSnapshot: localSnap(localSnapshotName, fullScope), + externalSnapshot: extSnap(actorURI, rootfsScope), + externalNewer: true, + }, + want: restoreWant{ + snapshotName: durableSnapshotName, + kind: ateattr.SnapshotKindLatest, + snapshotURI: actorURI, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, + }, + }, { name: "19 stamped Full durable snapshot on matching worker restores in Full", actor: actorSeed{ @@ -1625,6 +1633,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, want: restoreWant{ snapshotName: durableSnapshotName, + kind: ateattr.SnapshotKindLatest, snapshotURI: actorURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, @@ -1638,6 +1647,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, want: restoreWant{ snapshotName: durableSnapshotName, + kind: ateattr.SnapshotKindLatest, snapshotURI: actorURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, }, @@ -1652,6 +1662,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { tmpl: templateSeed{golden: stampedExtSnap(ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, goldenURI, fullScope, amd64RT)}, want: restoreWant{ snapshotName: durableSnapshotName, + kind: ateattr.SnapshotKindLatest, snapshotURI: goldenURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, }, @@ -1666,6 +1677,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, want: restoreWant{ snapshotName: durableSnapshotName, + kind: ateattr.SnapshotKindLatest, snapshotURI: actorURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, }, @@ -1680,6 +1692,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, want: restoreWant{ snapshotName: durableSnapshotName, + kind: ateattr.SnapshotKindLatest, snapshotURI: actorURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, @@ -1693,27 +1706,57 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, want: restoreWant{ snapshotName: durableSnapshotName, + kind: ateattr.SnapshotKindLatest, snapshotURI: actorURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, }, }, + { + name: "24a stamped local snapshot on incompatible worker drops to Rootfs", + actor: actorSeed{ + localSnapshot: stampedLocalSnap(localSnapshotName, fullScope, amd64RT), + localTmplUID: "current", + assignedRuntime: arm64RT, + }, + want: restoreWant{ + snapshotName: localSnapshotName, + kind: ateattr.SnapshotKindLocal, + snapshotURI: pauseUploadURI, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, + }, + }, + { + name: "24b stamped local snapshot on matching worker restores in Full", + actor: actorSeed{ + localSnapshot: stampedLocalSnap(localSnapshotName, fullScope, amd64RT), + localTmplUID: "current", + assignedRuntime: amd64RT, + }, + want: restoreWant{ + snapshotName: localSnapshotName, + kind: ateattr.SnapshotKindLocal, + snapshotURI: pauseUploadURI, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + }, + }, { // A recorded upload changes nothing: a local restore names the - // pause snapshot's upload whether or not it is COMPLETED. + // local snapshot's upload whether or not it is COMPLETED. name: "25 uploaded pause snapshot restores locally and sends its upload", actor: actorSeed{ localSnapshot: uploadedLocalSnap(localSnapshotName, fullScope), }, want: restoreWant{ snapshotName: localSnapshotName, + kind: ateattr.SnapshotKindLocal, snapshotURI: pauseUploadURI, scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, }, { - // A local restore names the pause snapshot's upload, so a record - // that cannot name it fails before a worker is claimed, as a - // malformed durable snapshot URI does. + // A local restore names the local snapshot's upload, so a record + // that cannot name it fails, as a malformed durable snapshot URI + // does. name: "26 local snapshot with no upload recorded fails with DataLoss", actor: actorSeed{localSnapshot: localSnapWithUpload(localSnapshotName, fullScope, nil)}, want: restoreWant{code: codes.DataLoss}, @@ -1764,7 +1807,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { t.Fatal("created template has no UID; the matching tmplUID case would be vacuous") } - // Paused and suspended actors are both SUSPENDED; a pause + // Sync and async suspended actors are both SUSPENDED; a node-local // checkpoint is what makes the resume local. actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} var extGen, localGen int32 @@ -1772,40 +1815,56 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { a.SourceTag = tt.actor.sourceTag a.Status.WorkerAssignment = wireTestAssignment() a.Status.WorkerAssignment.SandboxRuntime = proto.CloneOf(tt.actor.assignedRuntime) - uid := tt.actor.tmplUID - if uid == "current" { - uid = createdTmpl.GetMetadata().GetUid() + resolve := func(uid string) string { + if uid == "current" { + return createdTmpl.GetMetadata().GetUid() + } + return uid } var gen int32 - if tt.actor.externalSnapshot != nil { + seedExternal := func() { + if tt.actor.externalSnapshot == nil { + return + } gen++ extGen = gen ext := proto.CloneOf(tt.actor.externalSnapshot) ext.Generation = gen if ext.ActorTemplateUid == "" { - ext.ActorTemplateUid = uid + ext.ActorTemplateUid = resolve(tt.actor.tmplUID) } a.Status.Snapshots = append(a.Status.Snapshots, ext) } - if tt.actor.localSnapshot != nil { + seedLocal := func() { + if tt.actor.localSnapshot == nil { + return + } gen++ localGen = gen a.Status.AssignedNode = "node-1" loc := proto.CloneOf(tt.actor.localSnapshot) loc.Generation = gen if loc.ActorTemplateUid == "" { - loc.ActorTemplateUid = uid + loc.ActorTemplateUid = resolve(tt.actor.localTmplUID) } else if loc.ActorTemplateUid == "current" { loc.ActorTemplateUid = createdTmpl.GetMetadata().GetUid() } a.Status.Snapshots = append(a.Status.Snapshots, loc) } + if tt.actor.externalNewer { + seedLocal() + seedExternal() + } else { + seedExternal() + seedLocal() + } a.Status.LastAssignedGeneration = gen }) - actor, loadedTmpl, src, err := w.loadActorForResume(ctx, actorRef) + actor, loadedTmpl, err := w.loadActorForResume(ctx, actorRef) + var tele restoreTelemetry if err == nil { - _, err = w.ensureAteletRestored(ctx, actorRef, actor, loadedTmpl, nil, src) + tele, err = w.ensureAteletRestored(ctx, actorRef, actor, loadedTmpl, nil) } if got := apierror.Code(err); got != tt.want.code { t.Fatalf("apierror.Code(err) = %v, want %v (err: %v)", got, tt.want.code, err) @@ -1825,7 +1884,10 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { t.Fatalf("atelet requests = (restore=%v, run=%v), want exactly one Run", restore, run) } if !proto.Equal(run.GetSandboxRuntime(), wantSandboxRuntime) { - t.Errorf("run SandboxRuntime = %v, want %v", run.GetSandboxRuntime(), wantSandboxRuntime) + t.Errorf("run ActorSandboxRuntime = %v, want %v", run.GetSandboxRuntime(), wantSandboxRuntime) + } + if tele.SnapshotKind != ateattr.SnapshotKindBoot { + t.Errorf("SnapshotKind = %q, want %q", tele.SnapshotKind, ateattr.SnapshotKindBoot) } return } @@ -1833,7 +1895,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { t.Fatalf("atelet requests = (restore=%v, run=%v), want exactly one Restore", restore, run) } if !proto.Equal(restore.GetSandboxRuntime(), wantSandboxRuntime) { - t.Errorf("restore SandboxRuntime = %v, want %v", restore.GetSandboxRuntime(), wantSandboxRuntime) + t.Errorf("restore ActorSandboxRuntime = %v, want %v", restore.GetSandboxRuntime(), wantSandboxRuntime) } if got := restore.GetSnapshot().GetSnapshotUuid(); got != tt.want.snapshotName { t.Errorf("Snapshot.SnapshotUuid = %q, want %q", got, tt.want.snapshotName) @@ -1844,6 +1906,12 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { if got := restore.GetSnapshot().GetFidelity(); got != tt.want.scope { t.Errorf("restore scope = %v, want %v", got, tt.want.scope) } + if want := ateattr.SnapshotFidelityValue(tt.want.scope); tele.WireFidelity != want { + t.Errorf("WireFidelity = %q, want %q, the fidelity sent to atelet", tele.WireFidelity, want) + } + if tele.SnapshotKind != tt.want.kind { + t.Errorf("SnapshotKind = %q, want %q", tele.SnapshotKind, tt.want.kind) + } wantGen := extGen if tt.want.snapshotName == localSnapshotName { wantGen = localGen @@ -2141,11 +2209,11 @@ func contendedWorker(podName string) *ateapipb.Worker { WorkerPool: "pool", WorkerPod: podName, WorkerPodUid: testWorkerUID(podName), - SandboxClass: "gvisor", + NodeName: "node-1", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, - DefaultRuntime: testSandboxRuntime(), + SandboxClasses: workerSandboxClasses(testSandboxRuntime()), }, } } @@ -2417,29 +2485,41 @@ func TestEnsureWorkerAssigned_ContextEndsInsideAttemptRecordsOne(t *testing.T) { } } -// testRuntime is a gvisor SandboxRuntime for arch under compatibility schema +// testRuntime is a gvisor ActorSandboxRuntime for arch under compatibility schema // v1. A non-empty bundleHash adds a sandbox_bundle_hash attribute, so two // runtimes can share an architecture and still differ, as two versions of // one SandboxConfig do. -func testRuntime(arch, bundleHash string) *ateapipb.SandboxRuntime { +func testRuntime(arch, bundleHash string) *ateapipb.ActorSandboxRuntime { attrs := []*ateapipb.AttributeEntry{{Key: "architecture", Value: arch}} if bundleHash != "" { attrs = append(attrs, &ateapipb.AttributeEntry{Key: hardware.AttrSandboxBundleHash, Value: bundleHash}) } - return &ateapipb.SandboxRuntime{ + return &ateapipb.ActorSandboxRuntime{ SandboxClass: "gvisor", Version: &ateapipb.VersionedSandboxCompat{SchemaVersion: "v1", Attributes: attrs}, } } // testSandboxRuntime is the amd64 gvisor runtime most tests report and stamp. -func testSandboxRuntime() *ateapipb.SandboxRuntime { +func testSandboxRuntime() *ateapipb.ActorSandboxRuntime { return testRuntime("amd64", "") } +// workerSandboxClasses is the sandbox_classes a worker reports to run def as +// its default runtime and restore restorable beyond it, all of def's class. +// The arguments are the ActorSandboxRuntimes a test also stamps snapshots and +// assignments with, so both sides of a match derive from one value. +func workerSandboxClasses(def *ateapipb.ActorSandboxRuntime, restorable ...*ateapipb.ActorSandboxRuntime) []*ateapipb.WorkerSandboxClass { + class := &ateapipb.WorkerSandboxClass{SandboxClass: def.GetSandboxClass(), DefaultRuntime: &ateapipb.WorkerSandboxRuntime{Version: def.GetVersion()}} + for _, r := range restorable { + class.RestorableRuntimes = append(class.RestorableRuntimes, &ateapipb.WorkerSandboxRuntime{Version: r.GetVersion()}) + } + return []*ateapipb.WorkerSandboxClass{class} +} + // seedRuntimeWorker stores an ACTIVE gvisor worker on node with room for one // actor, reporting def as its default runtime and restorable beyond it. -func seedRuntimeWorker(t *testing.T, ctx context.Context, st store.Interface, pod, node string, def *ateapipb.SandboxRuntime, restorable ...*ateapipb.SandboxRuntime) *ateapipb.Worker { +func seedRuntimeWorker(t *testing.T, ctx context.Context, st store.Interface, pod, node string, def *ateapipb.ActorSandboxRuntime, restorable ...*ateapipb.ActorSandboxRuntime) *ateapipb.Worker { t.Helper() worker, err := st.CreateWorker(ctx, &ateapipb.Worker{ Metadata: &ateapipb.ResourceMetadata{Name: testWorkerUID(pod)}, @@ -2447,13 +2527,11 @@ func seedRuntimeWorker(t *testing.T, ctx context.Context, st store.Interface, po WorkerPool: "pool", WorkerPod: pod, WorkerPodUid: testWorkerUID(pod), - SandboxClass: "gvisor", NodeName: node, Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - Capacity: &ateapipb.WorkerResources{Actors: 1}, - DefaultRuntime: def, - RestorableRuntimes: restorable, + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + SandboxClasses: workerSandboxClasses(def, restorable...), }, }) if err != nil { @@ -2464,7 +2542,7 @@ func seedRuntimeWorker(t *testing.T, ctx context.Context, st store.Interface, po // durableMemorySnapshot is a completed, actor-owned MEMORY snapshot of // generation 1 under team-a/id1's own prefix, stamped with runtime. -func durableMemorySnapshot(t *testing.T, runtime *ateapipb.SandboxRuntime) *ateapipb.Snapshot { +func durableMemorySnapshot(t *testing.T, runtime *ateapipb.ActorSandboxRuntime) *ateapipb.Snapshot { t.Helper() snap := newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snap-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snap-1"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) snap.SandboxRuntime = runtime @@ -2481,17 +2559,16 @@ func seedMicroVMWorker(t *testing.T, ctx context.Context, st store.Interface, po WorkerPool: "vm-pool", WorkerPod: pod, WorkerPodUid: testWorkerUID(pod), - SandboxClass: "microvm", NodeName: node, Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, - DefaultRuntime: &ateapipb.SandboxRuntime{ + SandboxClasses: []*ateapipb.WorkerSandboxClass{{ SandboxClass: "microvm", - Version: &ateapipb.VersionedSandboxCompat{SchemaVersion: "v1", Attributes: []*ateapipb.AttributeEntry{ + DefaultRuntime: &ateapipb.WorkerSandboxRuntime{Version: &ateapipb.VersionedSandboxCompat{SchemaVersion: "v1", Attributes: []*ateapipb.AttributeEntry{ {Key: "architecture", Value: "amd64"}, - }}, - }, + }}}, + }}, }, }) if err != nil { @@ -2501,8 +2578,8 @@ func seedMicroVMWorker(t *testing.T, ctx context.Context, st store.Interface, po } // seedActorWithSnapshot stores actor team-a/id1 in state holding snap as its -// only snapshot. node is the AssignedNode a PAUSED actor keeps, which pins -// its resume to that node. +// only snapshot. node is the AssignedNode a suspended actor with a local +// snapshot keeps. func seedActorWithSnapshot(t *testing.T, ctx context.Context, st store.Interface, state ateapipb.ActorState, node string, snap *ateapipb.Snapshot) *ateapipb.Actor { t.Helper() return storetest.MustCreateActor(t, ctx, st, &ateapipb.Actor{ @@ -2531,7 +2608,7 @@ func newSchedulingWorkflow(t *testing.T, ctx context.Context, st store.Interface // assertAssignedRuntime checks the runtime recorded on actor's assignment, // which is what its next checkpoint will be stamped with. -func assertAssignedRuntime(t *testing.T, actor *ateapipb.Actor, want *ateapipb.SandboxRuntime) { +func assertAssignedRuntime(t *testing.T, actor *ateapipb.Actor, want *ateapipb.ActorSandboxRuntime) { t.Helper() if diff := cmp.Diff(want, actor.GetStatus().GetWorkerAssignment().GetSandboxRuntime(), protocmp.Transform()); diff != "" { t.Errorf("WorkerAssignment.SandboxRuntime mismatch (-want +got):\n%s", diff) @@ -2643,7 +2720,7 @@ func TestAssignWorkerAttempt_PrefersRuntimeMatchingWorkerAndFallsBack(t *testing ctx := context.Background() st := newTestPersistence(t) seedRuntimeWorker(t, ctx, st, "node1-incompatible", "node-1", arm64RT) - // A paused actor: SUSPENDED, pinned to its node by a local snapshot. + // A suspended actor with a local snapshot on node-1. local := newPausedSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-1", "node-1") local.SandboxRuntime = amd64RT actor := seedActorWithSnapshot(t, ctx, st, suspended, "node-1", local) @@ -2652,7 +2729,7 @@ func TestAssignWorkerAttempt_PrefersRuntimeMatchingWorkerAndFallsBack(t *testing if got := apierror.Code(err); got != codes.ResourceExhausted { t.Fatalf("apierror.Code(err) = %v, want %v (err: %v)", got, codes.ResourceExhausted, err) } - // Nothing was written: the actor waits for its node, still paused, + // Nothing was written: the actor waits for its node, still SUSPENDED, // unassigned, and holding no claim anything would have to release. after, err := st.GetActor(ctx, actorRef) if err != nil { @@ -2694,7 +2771,7 @@ func TestAssignWorkerAttempt_PrefersRuntimeMatchingWorkerAndFallsBack(t *testing // scheduler hands back still does. fresh := seedRuntimeWorker(t, ctx, st, "amd-pod", "node-amd", newDefaultRT) stale := proto.CloneOf(fresh) - stale.Status.RestorableRuntimes = []*ateapipb.SandboxRuntime{oldRestorableRT} + stale.Status.SandboxClasses = workerSandboxClasses(newDefaultRT, oldRestorableRT) actor := seedActorWithSnapshot(t, ctx, st, suspended, "", durableMemorySnapshot(t, oldRestorableRT)) w := &ActorWorkflow{store: st, scheduler: &relaxingTestScheduler{Scheduler: scheduling.New(nil), relaxedWorker: stale}} @@ -2703,8 +2780,8 @@ func TestAssignWorkerAttempt_PrefersRuntimeMatchingWorkerAndFallsBack(t *testing t.Fatalf("assignWorkerAttempt: %v", err) } assertAssignedRuntime(t, stored, newDefaultRT) - if got := bound.GetStatus().GetRestorableRuntimes(); len(got) != 0 { - t.Errorf("returned worker RestorableRuntimes = %v, want the fresh row's none", got) + if diff := cmp.Diff(fresh.GetStatus().GetSandboxClasses(), bound.GetStatus().GetSandboxClasses(), protocmp.Transform()); diff != "" { + t.Errorf("returned worker SandboxClasses mismatch, want the fresh row's (-fresh +got):\n%s", diff) } }) } @@ -2750,14 +2827,58 @@ func TestValidateAssignedWorker_RuntimeFallback(t *testing.T) { t.Errorf("validated worker = %q, want %q", got, "arm-pod") } }) + + t.Run("accepts fallback worker on re-entry for an uploaded pause snapshot", func(t *testing.T) { + ctx := context.Background() + st := newTestPersistence(t) + arm64RT := testRuntime("arm64", "") + seedRuntimeWorker(t, ctx, st, "arm-pod", "node-arm", arm64RT) + seedAssignment(t, st, testWorkerUID("arm-pod"), &ateapipb.ActorAssignment{ + Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "id1"}, + ActorUid: "own-actor-uid", + }) + + // Re-entry must accept a previously allowed ROOTFS assignment. + local := newPausedSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-1", "node-arm") + local.SandboxRuntime = testSandboxRuntime() + local.DurableSnapshot.Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + resumingActor := &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "id1", Uid: "own-actor-uid"}, + Status: &ateapipb.ActorStatus{ + State: ateapipb.ActorState_ACTOR_STATE_RESUMING, + AssignedNode: "node-arm", + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{local}, + WorkerAssignment: &ateapipb.WorkerAssignment{ + Worker: &ateapipb.ObjectRef{Name: testWorkerUID("arm-pod")}, + WorkerNamespace: "worker-ns", + WorkerPool: "pool", + WorkerPod: "arm-pod", + WorkerPodUid: testWorkerUID("arm-pod"), + NodeName: "node-arm", + SandboxRuntime: arm64RT, + }, + }, + } + + w := &ActorWorkflow{store: st, scheduler: scheduling.New(nil)} + worker, err := w.validateAssignedWorker(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, resumingActor, &ateapipb.ActorTemplate{}) + if err != nil { + t.Fatalf("validateAssignedWorker: %v", err) + } + if got := worker.GetWorkerPod(); got != "arm-pod" { + t.Errorf("validated worker = %q, want %q", got, "arm-pod") + } + }) } -// TestRequiredSnapshotRuntime pins which snapshot a resume takes its runtime -// constraint from, and when it takes none. +// TestRequiredSnapshotRuntime pins which snapshot a resume takes its sandbox +// class and runtime constraint from, and when it takes no runtime constraint. func TestRequiredSnapshotRuntime(t *testing.T) { const tmplUID = "tmpl-uid" tmpl := &ateapipb.ActorTemplate{Metadata: &ateapipb.ResourceMetadata{Uid: tmplUID}} localRT := testRuntime("amd64", "paused-under") + localRT.SandboxClass = "microvm" durableRT := testRuntime("amd64", "suspended-under") memory := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY rootfs := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS @@ -2765,13 +2886,13 @@ func TestRequiredSnapshotRuntime(t *testing.T) { completed := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED pending := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING - durable := func(gen int32, fidelity ateapipb.SnapshotFidelity, capturedUnder string, status ateapipb.SnapshotStorageStatus, rt *ateapipb.SandboxRuntime) *ateapipb.Snapshot { + durable := func(gen int32, fidelity ateapipb.SnapshotFidelity, capturedUnder string, status ateapipb.SnapshotStorageStatus, rt *ateapipb.ActorSandboxRuntime) *ateapipb.Snapshot { snap := newDurableSnapshot(gen, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, fidelity, capturedUnder, "snap", "gs://bucket/snap", status) snap.SandboxRuntime = rt return snap } - local := func(gen int32, fidelity ateapipb.SnapshotFidelity, rt *ateapipb.SandboxRuntime) *ateapipb.Snapshot { - snap := newPausedSnapshot(t, gen, fidelity, tmplUID, "local", "node-1") + local := func(gen int32, fidelity ateapipb.SnapshotFidelity, capturedUnder string, rt *ateapipb.ActorSandboxRuntime) *ateapipb.Snapshot { + snap := newPausedSnapshot(t, gen, fidelity, capturedUnder, "local", "node-1") snap.SandboxRuntime = rt return snap } @@ -2779,30 +2900,36 @@ func TestRequiredSnapshotRuntime(t *testing.T) { tests := []struct { name string snapshots []*ateapipb.Snapshot - want *ateapipb.SandboxRuntime + want *ateapipb.ActorSandboxRuntime + wantClass string }{ {name: "no snapshot cold-boots unconstrained"}, { name: "completed MEMORY durable snapshot requires its runtime", snapshots: []*ateapipb.Snapshot{durable(1, memory, tmplUID, completed, durableRT)}, want: durableRT, + wantClass: "gvisor", }, { name: "completed ROOTFS durable snapshot restores no memory state", snapshots: []*ateapipb.Snapshot{durable(1, rootfs, tmplUID, completed, durableRT)}, + wantClass: "gvisor", }, { name: "VOLUMES durable snapshot restores no memory state", snapshots: []*ateapipb.Snapshot{durable(1, volumes, tmplUID, completed, durableRT)}, + wantClass: "gvisor", }, { name: "durable snapshot captured under a replaced template is not restored", snapshots: []*ateapipb.Snapshot{durable(1, memory, "old-tmpl-uid", completed, durableRT)}, + wantClass: "gvisor", }, { name: "durable snapshot with no template uid keeps its runtime", snapshots: []*ateapipb.Snapshot{durable(1, memory, "", completed, durableRT)}, want: durableRT, + wantClass: "gvisor", }, { name: "unstamped durable snapshot imposes no constraint", @@ -2814,51 +2941,225 @@ func TestRequiredSnapshotRuntime(t *testing.T) { }, { name: "local checkpoint requires its runtime", - snapshots: []*ateapipb.Snapshot{local(1, memory, localRT)}, + snapshots: []*ateapipb.Snapshot{local(1, memory, tmplUID, localRT)}, + want: localRT, + wantClass: "microvm", + }, + { + name: "local checkpoint with no template uid keeps its runtime", + snapshots: []*ateapipb.Snapshot{local(1, memory, "", localRT)}, want: localRT, + wantClass: "microvm", }, { - name: "local checkpoint captured under a replaced template imposes no constraint", + name: "local checkpoint captured under a replaced template is not restored", + snapshots: []*ateapipb.Snapshot{local(1, memory, "old-tmpl-uid", localRT)}, + wantClass: "microvm", + }, + { + name: "newer durable snapshot takes precedence over an older local checkpoint", snapshots: []*ateapipb.Snapshot{ - func() *ateapipb.Snapshot { - s := newPausedSnapshot(t, 1, memory, "old-tmpl-uid", "local", "node-1") - s.SandboxRuntime = localRT - return s - }(), + local(1, memory, tmplUID, localRT), + durable(2, memory, tmplUID, completed, durableRT), }, + want: durableRT, + wantClass: "gvisor", }, { - // ensureAteletRestored restores a local checkpoint before any - // durable snapshot, so the constraint follows it regardless of - // which is newer. - name: "local checkpoint takes precedence over a newer durable snapshot", + name: "newer local checkpoint takes precedence over an older durable snapshot", snapshots: []*ateapipb.Snapshot{ - local(1, memory, localRT), - durable(2, memory, tmplUID, completed, durableRT), + local(2, memory, tmplUID, localRT), + durable(1, memory, tmplUID, completed, durableRT), }, - want: localRT, + want: localRT, + wantClass: "microvm", }, { name: "ROOTFS local checkpoint imposes no constraint beside a stamped durable snapshot", snapshots: []*ateapipb.Snapshot{ - local(2, rootfs, localRT), + local(2, rootfs, tmplUID, localRT), durable(1, memory, tmplUID, completed, durableRT), }, + wantClass: "microvm", }, { name: "VOLUMES local checkpoint imposes no constraint beside a stamped durable snapshot", snapshots: []*ateapipb.Snapshot{ - local(2, volumes, localRT), + local(2, volumes, tmplUID, localRT), durable(1, memory, tmplUID, completed, durableRT), }, + wantClass: "microvm", + }, + { + // Restore uses the newer local snapshot's template. + name: "local checkpoint ignores an older durable snapshot's replaced template", + snapshots: []*ateapipb.Snapshot{ + durable(1, memory, "old-tmpl-uid", completed, durableRT), + local(2, memory, tmplUID, localRT), + }, + want: localRT, + wantClass: "microvm", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { actor := &ateapipb.Actor{Status: &ateapipb.ActorStatus{Snapshots: tc.snapshots}} - if diff := cmp.Diff(tc.want, requiredSnapshotRuntime(actor, tmpl), protocmp.Transform()); diff != "" { + if diff := cmp.Diff(tc.want, requiredSnapshotRuntime(snapshotToResume(actor.GetStatus()), tmpl), protocmp.Transform()); diff != "" { t.Errorf("requiredSnapshotRuntime mismatch (-want +got):\n%s", diff) } + if got := snapshotSandboxClass(actor); got != tc.wantClass { + t.Errorf("snapshotSandboxClass = %q, want %q", got, tc.wantClass) + } + }) + } +} + +// mockVolume returns a volume served by the "mock" plugin. +func mockVolume(name, storageID string) *ateapipb.ExternalVolumeStatus { + return &ateapipb.ExternalVolumeStatus{Name: name, StorageVolumeId: storageID, VolumeType: "mock"} +} + +// volumeTemplate returns a template that declares the volumes named in +// declared and mounts those named in mounted. +func volumeTemplate(declared []string, mounted ...string) *ateapipb.ActorTemplate { + tmpl := &ateapipb.ActorTemplate{ + Containers: []*ateapipb.Container{{Name: "main", Image: "img"}}, + } + for _, name := range declared { + tmpl.Volumes = append(tmpl.Volumes, &ateapipb.Volume{Name: name, External: &ateapipb.ExternalVolumeSource{Template: &ateapipb.ExternalVolumeTemplate{StorageClassName: "sc"}}}) + } + for _, name := range mounted { + tmpl.Containers[0].VolumeMounts = append(tmpl.Containers[0].VolumeMounts, &ateapipb.VolumeMount{Name: name, MountPath: "/" + name}) + } + return tmpl +} + +// seedVolumeAssignFixture stores one free gvisor worker on node and a +// SUSPENDED actor holding volumes, and returns the stored actor plus a +// started worker cache. +func seedVolumeAssignFixture(t *testing.T, ctx context.Context, persistence store.Interface, node string, volumes ...*ateapipb.ExternalVolumeStatus) (*ateapipb.Actor, *workercache.Cache) { + t.Helper() + if _, err := persistence.CreateWorker(ctx, &ateapipb.Worker{ + Metadata: &ateapipb.ResourceMetadata{Name: testWorkerUID("pod-1")}, + WorkerNamespace: "worker-ns", + WorkerPool: "pool", + WorkerPod: "pod-1", + WorkerPodUid: testWorkerUID("pod-1"), + NodeName: node, + Status: &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + SandboxClasses: workerSandboxClasses(testSandboxRuntime()), + }, + }); err != nil { + t.Fatalf("CreateWorker: %v", err) + } + storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "id1"}, + Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, ExternalVolumes: &ateapipb.ExternalVolumes{Volumes: volumes}}, + }) + actor, err := persistence.GetActor(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}) + if err != nil { + t.Fatalf("GetActor: %v", err) + } + cacheCtx, cancel := context.WithCancel(ctx) + t.Cleanup(cancel) + wc := workercache.New(persistence, time.Minute) + if err := wc.Start(cacheCtx); err != nil { + t.Fatalf("workercache.Start: %v", err) + } + return actor, wc +} + +// newVolumeAssignWorkflow returns a workflow that schedules from wc and +// serves "mock" volumes with plugin. +func newVolumeAssignWorkflow(st store.Interface, wc *workercache.Cache, plugin volume.VolumePluginControlPlane) *ActorWorkflow { + return &ActorWorkflow{ + store: st, + workerCache: wc, + scheduler: scheduling.New(wc), + pluginRegistry: &mockPluginRegistry{plugins: map[string]volume.VolumePluginControlPlane{"mock": plugin}}, + } +} + +// TestResume_ChecksMountedVolumesBeforeAttach runs the assignment write and +// the attach step together: the assignment write records AssignedNode on the +// actor, and ensureVolumesAttached publishes only mounted volumes to it. A +// selected volume without a storage volume ID fails the attach step with +// FailedPrecondition before any publish call, even when a valid volume +// precedes it. +func TestResume_ChecksMountedVolumesBeforeAttach(t *testing.T) { + preview.InitForTest(t, "ExternalVolumes") + tests := []struct { + name string + // last is the second mounted volume, placed after every other volume. + last *ateapipb.ExternalVolumeStatus + // wantErr is a substring of the expected FailedPrecondition; empty + // means the attach succeeds. + wantErr string + }{{ + name: "all provisioned", + last: mockVolume("b", "id-b"), + }, { + name: "mounted volume without storage ID", + last: mockVolume("b", ""), + wantErr: `volume "b" has no storage volume ID`, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} + actor, wc := seedVolumeAssignFixture(t, ctx, persistence, "node-1", + mockVolume("a", "id-a"), + mockVolume("u", ""), + mockVolume("o", ""), + tt.last, + ) + plugin := &publishContextVolumePlugin{} + w := newVolumeAssignWorkflow(persistence, wc, plugin) + // u is declared but not mounted; o is not declared at all. + tmpl := volumeTemplate([]string{"a", "u", "b"}, "a", "b") + + assigned, worker, err := w.assignWorkerAttempt(ctx, actorRef, actor, tmpl) + if err != nil { + t.Fatalf("assignWorkerAttempt: %v", err) + } + if got := assigned.GetStatus().GetAssignedNode(); got != "node-1" { + t.Errorf("assigned actor AssignedNode = %q, want node-1", got) + } + _, attachErr := w.ensureVolumesAttached(ctx, assigned, worker, tmpl) + + nameByID := make(map[string]string) + for _, vol := range assigned.GetStatus().GetExternalVolumes().GetVolumes() { + nameByID[vol.GetStorageVolumeId()] = vol.GetName() + } + + if tt.wantErr != "" { + if got := apierror.Code(attachErr); got != codes.FailedPrecondition { + t.Fatalf("ensureVolumesAttached error = %v (code %v), want FailedPrecondition", attachErr, got) + } + if !strings.Contains(attachErr.Error(), tt.wantErr) { + t.Errorf("ensureVolumesAttached error = %q, want it to contain %q", attachErr, tt.wantErr) + } + if len(plugin.attachCalls) != 0 { + t.Errorf("attach calls = %v, want none", plugin.attachCalls) + } + return + } + if attachErr != nil { + t.Fatalf("ensureVolumesAttached: %v", attachErr) + } + var published []string + for _, call := range plugin.attachCalls { + if call.Node != "node-1" { + t.Errorf("attach of %q went to node %q, want node-1", call.VolumeID, call.Node) + } + published = append(published, nameByID[call.VolumeID]) + } + if diff := cmp.Diff([]string{"a", "b"}, published); diff != "" { + t.Errorf("published volumes mismatch (-want +got):\n%s", diff) + } }) } } @@ -2868,43 +3169,127 @@ func TestRequiredSnapshotRuntime(t *testing.T) { func TestCanRelaxRuntime(t *testing.T) { rt := testSandboxRuntime() memory := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + completed := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + pending := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + suspended := ateapipb.ActorState_ACTOR_STATE_SUSPENDED + resuming := ateapipb.ActorState_ACTOR_STATE_RESUMING + durable := func(gen int32) *ateapipb.Snapshot { - return newDurableSnapshot(gen, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", "snap", "gs://bucket/snap", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + return newDurableSnapshot(gen, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", "snap", "gs://bucket/snap", completed) } local := func(gen int32) *ateapipb.Snapshot { return newPausedSnapshot(t, gen, memory, "", "local", "node-1") } + localDurable := func(gen int32, storageStatus ateapipb.SnapshotStorageStatus) *ateapipb.Snapshot { + snap := local(gen) + snap.GetDurableSnapshot().Status = storageStatus + return snap + } + localNoDurable := func(gen int32) *ateapipb.Snapshot { + snap := local(gen) + snap.DurableSnapshot = nil + return snap + } tests := []struct { name string + state ateapipb.ActorState + node string snapshots []*ateapipb.Snapshot - required *ateapipb.SandboxRuntime + required *ateapipb.ActorSandboxRuntime want bool }{ { name: "nothing to relax without a runtime constraint", + state: suspended, snapshots: []*ateapipb.Snapshot{durable(1)}, }, { name: "durable snapshot may fall back", + state: suspended, snapshots: []*ateapipb.Snapshot{durable(1)}, required: rt, want: true, }, { name: "local checkpoint stays pinned", + state: suspended, + node: "node-1", snapshots: []*ateapipb.Snapshot{local(1)}, required: rt, }, { name: "local checkpoint stays pinned beside a durable snapshot", + state: suspended, + node: "node-1", snapshots: []*ateapipb.Snapshot{durable(1), local(2)}, required: rt, }, + { + name: "uploaded checkpoint detaches before relaxing a pinned actor", + state: suspended, + node: "node-1", + snapshots: []*ateapipb.Snapshot{localDurable(2, completed)}, + required: rt, + }, + { + name: "unbound uploaded checkpoint may fall back", + state: suspended, + snapshots: []*ateapipb.Snapshot{localDurable(2, completed)}, + required: rt, + want: true, + }, + { + name: "resuming uploaded checkpoint keeps its relaxed assignment", + state: resuming, + node: "node-1", + snapshots: []*ateapipb.Snapshot{localDurable(2, completed)}, + required: rt, + want: true, + }, + { + name: "resuming pending checkpoint cannot relax", + state: resuming, + node: "node-1", + snapshots: []*ateapipb.Snapshot{localDurable(2, pending)}, + required: rt, + }, + { + name: "resuming checkpoint without a durable copy cannot relax", + state: resuming, + node: "node-1", + snapshots: []*ateapipb.Snapshot{localNoDurable(2)}, + required: rt, + }, + { + name: "older durable copy cannot authorize a resuming checkpoint", + state: resuming, + node: "node-1", + snapshots: []*ateapipb.Snapshot{durable(1), localNoDurable(2)}, + required: rt, + }, + { + name: "pinned actor detaches before relaxing a newer durable snapshot", + state: suspended, + node: "node-1", + snapshots: []*ateapipb.Snapshot{local(1), durable(2)}, + required: rt, + }, + { + name: "newer durable snapshot may fall back beside an older pending checkpoint", + state: suspended, + snapshots: []*ateapipb.Snapshot{localDurable(1, pending), durable(2)}, + required: rt, + want: true, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - actor := &ateapipb.Actor{Status: &ateapipb.ActorStatus{Snapshots: tc.snapshots}} + actor := &ateapipb.Actor{Status: &ateapipb.ActorStatus{ + State: tc.state, + AssignedNode: tc.node, + Snapshots: tc.snapshots, + }} if got := canRelaxRuntime(actor, scheduling.Constraints{RequiredRuntime: hardware.NewRequirement(tc.required)}); got != tc.want { t.Errorf("canRelaxRuntime = %v, want %v", got, tc.want) } @@ -2925,18 +3310,18 @@ func TestWorkerAssignmentFromCopiesSPIFFEID(t *testing.T) { func TestToAteletSandboxRuntime(t *testing.T) { tests := []struct { name string - in *ateapipb.SandboxRuntime - want *ateletpb.SandboxRuntime + in *ateapipb.ActorSandboxRuntime + want *ateletpb.ActorSandboxRuntime }{ {name: "nil"}, { name: "class only", - in: &ateapipb.SandboxRuntime{SandboxClass: "gvisor"}, - want: &ateletpb.SandboxRuntime{SandboxClass: "gvisor"}, + in: &ateapipb.ActorSandboxRuntime{SandboxClass: "gvisor"}, + want: &ateletpb.ActorSandboxRuntime{SandboxClass: "gvisor"}, }, { name: "full", - in: &ateapipb.SandboxRuntime{ + in: &ateapipb.ActorSandboxRuntime{ SandboxClass: "gvisor", Version: &ateapipb.VersionedSandboxCompat{ SchemaVersion: "v1", @@ -2946,7 +3331,7 @@ func TestToAteletSandboxRuntime(t *testing.T) { }, }, }, - want: &ateletpb.SandboxRuntime{ + want: &ateletpb.ActorSandboxRuntime{ SandboxClass: "gvisor", Version: &ateletpb.VersionedSandboxCompat{ SchemaVersion: "v1", @@ -2966,3 +3351,51 @@ func TestToAteletSandboxRuntime(t *testing.T) { }) } } + +// TestAssignWorkerAttempt_WorkerWithoutNodeRejectsVolumes verifies a worker +// with no node name is not claimed: the attempt fails with FailedPrecondition +// before the bind, leaving no worker claim and no actor write. +func TestAssignWorkerAttempt_WorkerWithoutNodeRejectsVolumes(t *testing.T) { + tests := []struct { + name string + mounted []string + }{ + {name: "mounted volume", mounted: []string{"vol"}}, + {name: "unmounted volume", mounted: nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} + actor, wc := seedVolumeAssignFixture(t, ctx, persistence, "", + mockVolume("vol", "storage-vol"), + ) + plugin := &publishContextVolumePlugin{} + w := newVolumeAssignWorkflow(persistence, wc, plugin) + tmpl := volumeTemplate([]string{"vol"}, tt.mounted...) + + _, _, err := w.assignWorkerAttempt(ctx, actorRef, actor, tmpl) + stored, gerr := persistence.GetActor(ctx, actorRef) + if gerr != nil { + t.Fatalf("GetActor: %v", gerr) + } + claim := firstAssignment(t, persistence, testWorkerUID("pod-1")) + if got := apierror.Code(err); got != codes.FailedPrecondition { + t.Fatalf("assignWorkerAttempt error = %v (code %v), want FailedPrecondition", err, got) + } + if want := "has no node name"; !strings.Contains(err.Error(), want) { + t.Errorf("assignWorkerAttempt error = %q, want it to contain %q", err, want) + } + if claim != nil { + t.Errorf("worker claim = %v, want none", claim) + } + if diff := cmp.Diff(actor, stored, protocmp.Transform()); diff != "" { + t.Errorf("assignment wrote to the stored actor (-before +after):\n%s", diff) + } + if len(plugin.attachCalls) != 0 { + t.Errorf("attach calls = %v, want none", plugin.attachCalls) + } + }) + } +} diff --git a/cmd/ateapi/internal/controlapi/workflow_revert.go b/cmd/ateapi/internal/controlapi/workflow_revert.go index dcae8cc7c0..55c239fe25 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert.go @@ -32,10 +32,10 @@ import ( ) // revertableStates are the states a revert is accepted from. A SUSPENDED -// actor is also accepted when it is paused, that is, when its latest snapshot -// is node-local: the revert discards the node-local copy and returns the actor -// to its latest durable snapshot, which is that same snapshot once atelet's -// upload of it is recorded. See isRevertable. +// actor is also accepted when its latest snapshot is node-local: the revert +// discards the node-local copy and returns the actor to its latest durable +// snapshot, which is that same snapshot once atelet's upload of it is +// recorded. See isRevertable. var revertableStates = []ateapipb.ActorState{ ateapipb.ActorState_ACTOR_STATE_RUNNING, ateapipb.ActorState_ACTOR_STATE_CRASHED, @@ -186,18 +186,23 @@ func (w *ActorWorkflow) ensureWorkerDiscarded(ctx context.Context, actorRef reso defer func() { err = done(err) }() if actor.GetStatus().GetAssignedNode() != "" { + forgetNode := false if err := w.ensureAteletTerminated(ctx, actorRef, actor, actorTemplate, ateattr.OperationRevert); err != nil { - // A failed terminate leaves the actor REVERTING with its - // assignment, so the next revert terminates it again. If the - // worker's pod goes away, worker deletion crashes the actor, and - // the next revert needs no live worker. - return err + if !canDetachWithoutAtelet(ctx, actor, err) { + return err + } + forgetNode = true } if err := w.ensureVolumesDetached(ctx, actor, actorTemplate, "DetachVolumesForRevert", ateattr.OperationRevert); err != nil { return err } + if forgetNode { + if _, err := w.recordDetach(ctx, actorRef, actor, actor.GetStatus().GetAssignedNode(), nil); err != nil { + return err + } + } if actor.GetStatus().GetWorkerAssignment() != nil { - if _, _, err := releaseWorker(ctx, w.store, actor); err != nil { + if _, err := releaseWorker(ctx, w.store, actor); err != nil { return fmt.Errorf("while releasing worker: %w", err) } } @@ -206,11 +211,8 @@ func (w *ActorWorkflow) ensureWorkerDiscarded(ctx context.Context, actorRef reso return w.releaseAssignmentWithoutBacklink(ctx, actor) } -// ensureInProgressSnapshotDiscarded deletes the objects of every PENDING -// durable snapshot on the actor: the one a suspend was partway through writing -// when the actor was reverted, and any earlier pause's whose upload was never -// recorded. The revert keeps none of them. atelet's upload of them has already -// been canceled when ensureWorkerDiscarded terminated the actor. +// ensureInProgressSnapshotDiscarded removes pending uploads of the discarded execution. +// The actor keeps its latest completed durable snapshot. func (w *ActorWorkflow) ensureInProgressSnapshotDiscarded(ctx context.Context, actor *ateapipb.Actor) (err error) { ctx, done := stepSpan(ctx, "DiscardInProgressSnapshot") defer func() { err = done(err) }() diff --git a/cmd/ateapi/internal/controlapi/workflow_revert_test.go b/cmd/ateapi/internal/controlapi/workflow_revert_test.go index 64de763c38..5405ee2d08 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/cmd/ateapi/internal/store" "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/objectstore/objectstoretest" @@ -26,6 +27,7 @@ import ( "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" + "google.golang.org/protobuf/proto" ) // TestEnsureMarkedReverting_StateMatrix pins which states a revert is accepted @@ -35,7 +37,7 @@ import ( // SUSPENDED is the load-bearing rejection: it is revert's own destination, and // also what a suspend that won the race leaves behind, so accepting it would // report success for the opposite of what was asked. A SUSPENDED actor that -// holds a node-local snapshot is paused, not reverted, and is accepted; see +// holds a node-local snapshot is accepted; see // TestEnsureMarkedReverting_AcceptsPaused. func TestEnsureMarkedReverting_StateMatrix(t *testing.T) { allowed := map[ateapipb.ActorState]bool{ @@ -69,9 +71,9 @@ func TestEnsureMarkedReverting_StateMatrix(t *testing.T) { } } -// TestEnsureMarkedReverting_AcceptsPaused verifies a paused actor (SUSPENDED -// whose latest snapshot is node-local) takes the revert edge: the revert -// discards that snapshot and returns the actor to its durable one. +// TestEnsureMarkedReverting_AcceptsPaused verifies a SUSPENDED actor whose +// latest snapshot is node-local takes the revert edge: the revert discards that +// local copy and returns the actor to its durable snapshot. func TestEnsureMarkedReverting_AcceptsPaused(t *testing.T) { ctx := context.Background() st, cleanup := storetest.SetupTestStore(t) @@ -105,7 +107,7 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { for _, tc := range []struct { name string seedState ateapipb.ActorState - // paused seeds the paused shape: the latest snapshot is node-local + // paused seeds the local-snapshot shape: the latest snapshot is node-local // and the actor is pinned to its node. Only meaningful with // SUSPENDED, which the revert otherwise rejects. paused bool @@ -131,9 +133,10 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { } // The snapshot revert must preserve, plus the node-local state a - // pause left behind, which it must not. A paused actor's latest - // snapshot is the completed local one; the other origins also carry - // the half-written local snapshot an interrupted pause leaves. + // suspend left behind, which it must not. The suspended actor's + // latest snapshot is the completed local one; the other origins + // also carry the half-written local snapshot an interrupted suspend + // leaves. const keptURI = "gs://snapshots/team-a/actors/keep/snapshot" tmpl := &ateapipb.ActorTemplate{SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://snapshots"}} mustUpdateActorStatus(t, ctx, st, actor, func(s *ateapipb.ActorStatus) { @@ -220,11 +223,124 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { } } -// TestRevertActor_PausedSnapshotAlreadyUploaded verifies a revert of a paused -// actor whose snapshot atelet uploaded returns it to that upload, its latest -// durable snapshot, and collects the durable snapshot the upload replaced. The -// actor has no assigned node, so there is nothing to terminate, as in -// TestRevertActor_ReturnsActorToItsSnapshot. +func TestRevertActor_UnreachableNode(t *testing.T) { + for _, tt := range []struct { + name string + node string + err error + pending bool + assigned bool + crashed bool + conflict bool + wantCode codes.Code + }{ + {name: "missing atelet", node: "node-3"}, + {name: "unavailable atelet", node: "node-1", err: apierror.Unavailable("offline")}, + {name: "atelet deadline", node: "node-1", err: context.DeadlineExceeded}, + {name: "pending beside older completed snapshot", node: "node-1", err: apierror.Unavailable("offline"), pending: true, wantCode: codes.Unavailable}, + {name: "worker still assigned", node: "node-1", err: apierror.Unavailable("offline"), assigned: true, wantCode: codes.Unavailable}, + {name: "crashed execution still needs termination", node: "node-1", err: apierror.Unavailable("offline"), crashed: true, wantCode: codes.Unavailable}, + {name: "other RPC failure", node: "node-1", err: apierror.Internal("injected failure"), wantCode: codes.Internal}, + {name: "concurrent update", node: "node-1", err: apierror.Unavailable("offline"), conflict: true, wantCode: codes.Aborted}, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + st, cleanup := storetest.SetupTestStore(t) + t.Cleanup(cleanup) + w := newTestActorWorkflow(t, st, "ns", "tmpl1") + wire, atelet := newWireCaptureWorkflow(t, st) + w.dialer = wire.dialer + ref := resources.ActorRef{Atespace: "team-a", Name: "id1"} + seedWorkflowActor(t, ctx, st, ref, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDED) + actor, err := st.GetActor(ctx, ref) + if err != nil { + t.Fatal(err) + } + actor = mustUpdateActorStatus(t, ctx, st, actor, func(s *ateapipb.ActorStatus) { + s.AssignedNode = tt.node + s.LastAssignedGeneration = 2 + snap := newPausedSnapshot(t, 2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "uploaded", tt.node) + snap.DurableSnapshot.Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + if tt.pending { + snap.DurableSnapshot.Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + } + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, snap.GetFidelity(), "", "older", "gs://snapshots/team-a/actors/older/snapshot", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + snap, + } + if tt.assigned { + s.State = ateapipb.ActorState_ACTOR_STATE_RUNNING + s.WorkerAssignment = wireTestAssignment() + } + if tt.crashed { + s.State = ateapipb.ActorState_ACTOR_STATE_CRASHED + s.Crash = newActorCrash("ResumeActor", "interrupted execution") + } + }) + want := proto.CloneOf(actor.GetStatus().GetSnapshots()[1]) + atelet.terminateErr = tt.err + if tt.conflict { + atelet.onTerminate = func() { + cur, err := st.GetActor(ctx, ref) + if err != nil { + t.Errorf("GetActor: %v", err) + return + } + if _, err := st.UpdateActor(ctx, ref, store.PreconditionFrom(cur), func(a *ateapipb.Actor) error { + a.WorkerSelector = &ateapipb.Selector{MatchLabels: map[string]string{"changed": "true"}} + return nil + }); err != nil { + t.Errorf("UpdateActor: %v", err) + } + } + } + reverted, err := w.RevertActor(ctx, ref) + if got := apierror.Code(err); got != tt.wantCode { + t.Fatalf("RevertActor = %v, want %v", err, tt.wantCode) + } + if tt.wantCode != codes.OK { + stored, err := st.GetActor(ctx, ref) + if err != nil { + t.Fatal(err) + } + got := stored.GetStatus() + if got.GetState() != ateapipb.ActorState_ACTOR_STATE_REVERTING || got.GetAssignedNode() != tt.node { + t.Fatalf("status = %v, want REVERTING on %s", got, tt.node) + } + if !proto.Equal(got.GetSnapshots()[1], want) { + t.Fatalf("snapshot changed after failed revert: %v", got.GetSnapshots()) + } + if !tt.conflict { + return + } + atelet.mu.Lock() + atelet.onTerminate = nil + atelet.mu.Unlock() + reverted, err = w.RevertActor(ctx, ref) + if err != nil { + t.Fatalf("retry after conflict: %v", err) + } + } + got := reverted.GetStatus() + if got.GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED || got.GetAssignedNode() != "" || got.GetWorkerAssignment() != nil { + t.Fatalf("reverted status = %v, want unbound SUSPENDED", got) + } + if len(got.GetSnapshots()) != 1 { + t.Fatalf("snapshots = %v, want only the latest durable snapshot", got.GetSnapshots()) + } + want.Locality = "" + if !proto.Equal(got.GetSnapshots()[0], want) { + t.Errorf("retained snapshot = %v, want %v", got.GetSnapshots()[0], want) + } + }) + } +} + +// TestRevertActor_PausedSnapshotAlreadyUploaded verifies a revert of a +// suspended actor whose local snapshot atelet uploaded returns it to that +// upload, its latest durable snapshot, and collects the durable snapshot the +// upload replaced. The actor has no assigned node, so there is nothing to +// terminate, as in TestRevertActor_ReturnsActorToItsSnapshot. func TestRevertActor_PausedSnapshotAlreadyUploaded(t *testing.T) { memory := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY completed := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED @@ -380,8 +496,8 @@ func TestEnsureInProgressSnapshotDiscarded(t *testing.T) { // foreignInFlight records the in-progress snapshot under another // owner's prefix, which no suspend can produce. foreignInFlight bool - // earlierPause records an older PENDING snapshot: a pause whose upload - // atelet never reported before the actor resumed. + // earlierPause records an older PENDING snapshot: an async suspend whose + // upload atelet never reported before the actor resumed. earlierPause bool wantInFlightDiscarded bool wantEarlierPauseDiscarded bool diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend.go b/cmd/ateapi/internal/controlapi/workflow_suspend.go index e567291a13..f7a281923b 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend.go @@ -35,18 +35,19 @@ import ( ) // suspendMode selects which of the two suspends the suspend workflow runs. -// Both take the same steps; the mode only decides the shape of the snapshot -// they take. +// Both take the same steps; the mode only decides when atelet uploads the +// durable copy of the snapshot they take. type suspendMode int const ( // suspendSync makes the snapshot durable before the workflow returns: - // the actor ends SUSPENDED holding a durable snapshot. + // the actor ends SUSPENDED holding a COMPLETED durable snapshot (plus its + // node-local copy when atelet retains it on the worker's node). suspendSync suspendMode = iota - // suspendAsync leaves the snapshot on the worker's node: the actor ends + // suspendAsync uploads the snapshot in the background: the actor ends // SUSPENDED holding a node-local snapshot whose durable copy is still - // pending, pinned to that node until atelet uploads it in the background - // and the detach report unpins it. + // PENDING, pinned to that node until atelet finishes the upload and the + // detach report unpins it. suspendAsync ) @@ -69,8 +70,8 @@ func (m suspendMode) uploadMode() ateletpb.UploadMode { // SuspendActor executes the workflow to suspend an actor. A running actor is // checkpointed on its worker: a sync suspend uploads the snapshot before -// returning, an async suspend keeps it on that node. A SUSPENDED actor is -// returned as it is, whatever the mode. Idempotent: a re-entered workflow +// returning, an async suspend uploads it in the background. A SUSPENDED actor +// is returned as it is, whatever the mode. Idempotent: a re-entered workflow // fast-forwards past the steps a previous attempt completed, deriving // progress from the persisted actor alone. func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.ActorRef, mode suspendMode) (_ *ateapipb.Actor, err error) { @@ -211,14 +212,13 @@ func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef res // ensureAteletSuspended checkpoints the workload to the actor's persisted // in-progress snapshot location. A sync suspend has atelet upload it before -// returning; an async suspend has atelet keep it on the node. It returns the -// snapshot state atelet reports once the checkpoint is taken, which finalize -// applies. -// This is the atelet reentrancy seam (#372): -// the request is keyed by the actor UID, the worker pod UID, and the -// once-minted snapshot location, so a re-entered workflow re-sends the same -// semantic request; once atelet's Checkpoint is idempotent on those keys this -// step becomes fully reentrant with no changes here. +// returning; an async suspend has atelet upload it in the background. It +// returns the snapshot state atelet reports once the checkpoint is taken, +// which finalize applies. +// A re-entered workflow re-sends the same snapshot, which +// ensureMarkedSuspending minted once, and atelet answers a Checkpoint of a +// snapshot it already committed without checkpointing again, so a retry after +// a lost response succeeds. func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate, mode suspendMode) (wireFidelity string, state *ateletpb.ActorSnapshotState, err error) { ctx, done := stepSpan(ctx, "CallAteletSuspend") defer func() { err = done(err) }() @@ -312,9 +312,10 @@ func (w *ActorWorkflow) ensureVolumesDetached(ctx context.Context, actor *ateapi // ensureSuspendedFinalized releases the actor's worker (only when it is still // owned by this actor), records the in-progress snapshot as the actor's -// durable snapshot (sync) or node-local snapshot with its PENDING durable copy -// (async), and commits SUSPENDED with the assignment cleared in a single -// update, then releases the durable snapshot that update replaced. +// durable snapshot (sync, or async when atelet keeps only the durable copy) or +// node-local snapshot with its PENDING durable copy (async), and commits +// SUSPENDED with the assignment cleared in a single update, then releases the +// durable snapshot that update replaced. // When the latest snapshot cannot be recorded, it commits CRASHED instead and // fails the step. // It re-reads the actor first so an out-of-band transition (e.g. the syncer @@ -363,7 +364,7 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r } // Both modes mark the durable copy PENDING up front. Without it a sync // suspend has nothing to mark COMPLETED, and atelet's background upload of - // a pause's snapshot could never be recorded. + // an async suspend's snapshot could never be recorded. if snapshotAtLatestGeneration(latestActor.GetStatus()).GetDurableSnapshot() == nil { return nil, crashActorAndFail(ctx, w.store, actorRef, ateattr.OperationSuspend, crashMessageDurableSnapshotMissing) } @@ -381,7 +382,7 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r // 1. Free the worker (if it hasn't been freed yet) if latestActor.GetStatus().GetWorkerAssignment() != nil { t = time.Now() - _, released, err := releaseWorker(ctx, w.store, latestActor) + released, err := releaseWorker(ctx, w.store, latestActor) dReleaseWorker = time.Since(t) if err != nil { return nil, err @@ -402,20 +403,19 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r // The precondition makes toUpdate the version applied was built from. toUpdate.Status.Snapshots = applied.GetSnapshots() snap := snapshotAtLatestGeneration(toUpdate.Status) - switch mode { - case suspendSync: + if mode == suspendSync { snap.GetDurableSnapshot().Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED - // Remove all local snapshots since they are superseded by the new durable snapshot. - // TODO: This will change when an async suspend uploads its snapshot - // and both local and durable snapshots are kept. - pruneSnapshots(toUpdate.Status, nil) - toUpdate.Status.AssignedNode = "" - case suspendAsync: - // Record the node locality of the local checkpoint and clear older local checkpoints. - // AssignedNode is kept: a paused actor resumes on that node. + } + durable := snap.GetDurableSnapshot().GetStatus() == ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + // Keep the actor on the node that holds its snapshot. atelet holds an + // async suspend's snapshot until it detaches it, which it does only + // once the snapshot is durable. + if ateletKeptLocal(ateletState, snap) || !durable { snap.Locality = toUpdate.GetStatus().GetAssignedNode() - pruneSnapshots(toUpdate.Status, snap) + } else { + toUpdate.Status.AssignedNode = "" } + pruneSnapshots(toUpdate.Status, snap) toUpdate.Status.WorkerAssignment = nil return nil }) @@ -437,6 +437,16 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r return storedActor, nil } +// ateletKeptLocal reports whether atelet retained this checkpoint. +func ateletKeptLocal(state *ateletpb.ActorSnapshotState, snap *ateapipb.Snapshot) bool { + local := state.GetLatestLocal().GetSnapshot() + uri := snap.GetDurableSnapshot().GetObject().GetSnapshotUri() + return local != nil && uri != "" && + local.GetSnapshotUuid() == snap.GetUuid() && + local.GetGeneration() == snap.GetGeneration() && + local.GetObject().GetSnapshotUri() == uri +} + // releaseDroppedSnapshots releases each external snapshot that before records // as COMPLETED and after doesn't. A failure is only logged, and leaves the // snapshot's objects in storage until the actor is deleted, which removes its diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go index e99bb961c1..3ec08cf586 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go @@ -31,13 +31,13 @@ import ( ) // TestSuspendActorAsync_Finalized_WorkerGone reproduces the scenario where the -// worker pod disappears from the DB during pause finalization, so the node it -// ran on is unknown. +// worker pod disappears from the DB during async suspend finalization, so the +// node it ran on is unknown. // // Current behavior: AssignedNode is left empty, and the actor is crashed -// instead of left paused, since a local snapshot with an unknown node can -// never be safely resumed. The pause fails with FailedPrecondition rather than -// reporting the pause as done. +// instead of left SUSPENDED with a local snapshot, since a local snapshot with +// an unknown node can never be safely resumed. The suspend fails with +// FailedPrecondition rather than reporting the suspend as done. func TestSuspendActorAsync_Finalized_WorkerGone(t *testing.T) { ctx := context.Background() st := newTestPersistence(t) @@ -85,7 +85,7 @@ func TestSuspendActorAsync_Finalized_WorkerGone(t *testing.T) { } // This site records the crash counter, so it must write the record too, or - // a pause-finalize crash is the one kind nothing can attribute to an actor. + // an async-suspend-finalize crash is the one kind nothing can attribute to an actor. if len(*records) != 1 { t.Fatalf("got %d crash records, want 1", len(*records)) } @@ -99,8 +99,8 @@ func TestSuspendActorAsync_Finalized_WorkerGone(t *testing.T) { // TestSuspendActorAsync_Finalized_AlreadyCrashed verifies that if the actor was // crashed out-of-band while it checkpointed, and has no AssignedNode, the -// pause's finalize preserves the existing Crash status and snapshots, emits no -// duplicate crash log record, and still fails. +// async suspend's finalize preserves the existing Crash status and snapshots, +// emits no duplicate crash log record, and still fails. func TestSuspendActorAsync_Finalized_AlreadyCrashed(t *testing.T) { ctx := context.Background() st := newTestPersistence(t) @@ -150,7 +150,7 @@ func TestSuspendActorAsync_Finalized_AlreadyCrashed(t *testing.T) { } // TestSuspendActorAsync_Finalized_KeepsCrashFromWorkerDelete verifies that a crash the -// DeleteWorker workflow commits while the pause's finalize is freeing the +// DeleteWorker workflow commits while the async suspend's finalize is freeing the // worker is kept: the finalize update is pinned to the actor version it // checked, so it fails Aborted instead of committing SUSPENDED over the crash. func TestSuspendActorAsync_Finalized_KeepsCrashFromWorkerDelete(t *testing.T) { @@ -175,7 +175,7 @@ func TestSuspendActorAsync_Finalized_KeepsCrashFromWorkerDelete(t *testing.T) { assertConcurrentCrashKept(t, ctx, st, actorRef, err, "suspend failed: "+crashMessageWorkerPodGone) } -// TestSuspendActorAsync_Finalized_RecordsFidelity verifies pause records the template's +// TestSuspendActorAsync_Finalized_RecordsFidelity verifies async suspend records the template's // preferred_fidelity on the local Snapshot alongside Locality, and keeps its // durable copy PENDING for atelet to upload in the background. func TestSuspendActorAsync_Finalized_RecordsFidelity(t *testing.T) { @@ -203,7 +203,7 @@ func TestSuspendActorAsync_Finalized_RecordsFidelity(t *testing.T) { w.workerCache.AddHandler(func(worker *ateapipb.Worker) { observed = worker }) actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} - wantRuntime := &ateapipb.SandboxRuntime{ + wantRuntime := &ateapipb.ActorSandboxRuntime{ SandboxClass: "gvisor", Version: &ateapipb.VersionedSandboxCompat{ SchemaVersion: "v1", @@ -277,9 +277,9 @@ func TestSuspendActorAsync_Finalized_RecordsFidelity(t *testing.T) { // TestSuspendActorAsync_Finalized_MissingSnapshotClearsAssignment verifies that if the // latest generation snapshot entry, or its PENDING durable copy, is missing -// when the pause finalizes, the actor transitions to CRASHED and clears +// when the async suspend finalizes, the actor transitions to CRASHED and clears // WorkerAssignment after releasing the worker rather than getting stuck -// SUSPENDING, and the pause fails. +// SUSPENDING, and the suspend fails. func TestSuspendActorAsync_Finalized_MissingSnapshotClearsAssignment(t *testing.T) { tests := []struct { name string @@ -414,9 +414,9 @@ func TestSuspendActorAsync_AteletPaused_DialFailureLeavesActorRetryable(t *testi } // TestSuspendActorAsync_CrashesWhenPausingActorMissingWorkerPod verifies that a -// pausing actor with no worker pod recorded is moved to CRASHED by the -// corrupted-assignment check before atelet is called, and the pause fails -// with FailedPrecondition. +// suspending actor with no worker pod recorded is moved to CRASHED by the +// corrupted-assignment check before atelet is called, and the async suspend +// fails with FailedPrecondition. func TestSuspendActorAsync_CrashesWhenPausingActorMissingWorkerPod(t *testing.T) { ctx := context.Background() st, cleanup := storetest.SetupTestStore(t) @@ -442,9 +442,9 @@ func TestSuspendActorAsync_CrashesWhenPausingActorMissingWorkerPod(t *testing.T) } } -// TestSuspendActorAsync_AteletPaused_ForwardsPreferredFidelity verifies that local pause -// checkpoints forward the template's PreferredFidelity to atelet, and label the -// pause's lifecycle-op metric with it. +// TestSuspendActorAsync_AteletPaused_ForwardsPreferredFidelity verifies that async +// suspend checkpoints forward the template's PreferredFidelity to atelet, and +// label the suspend's lifecycle-op metric with it. func TestSuspendActorAsync_AteletPaused_ForwardsPreferredFidelity(t *testing.T) { tests := []struct { preferred ateapipb.SnapshotFidelity @@ -503,7 +503,7 @@ func TestSuspendActorAsync_AteletPaused_ForwardsPreferredFidelity(t *testing.T) if got := req.GetUploadConfig().GetMode(); got != ateletpb.UploadMode_UPLOAD_MODE_DEFERRED { t.Errorf("CheckpointRequest.UploadConfig.Mode = %v, want UPLOAD_MODE_DEFERRED", got) } - // The fidelity sent to atelet labels the pause's lifecycle-op metric. + // The fidelity sent to atelet labels the suspend's lifecycle-op metric. assertLifecycleFidelity(t, reader, ateattr.SnapshotFidelityValue(tc.want)) }) } diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index af9789d3f7..f31ae34279 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -145,7 +145,7 @@ func TestSuspendActor_MarkedSuspending_SnapshotURI(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: "gs://bucket/root/"}) - wantRuntime := &ateapipb.SandboxRuntime{ + wantRuntime := &ateapipb.ActorSandboxRuntime{ SandboxClass: "gvisor", Version: &ateapipb.VersionedSandboxCompat{ SchemaVersion: "v1", @@ -613,10 +613,11 @@ func TestSuspendActor_Finalized_ReleasesReplacedSnapshot(t *testing.T) { } } -// TestSuspendActor_Finalized_PauseReleasesReplacedSnapshot verifies that a -// pause keeping the previous pause's upload collects the external snapshot -// that upload replaced, and keeps the upload. The upload is recorded from the -// pause's Checkpoint state, or before the pause from atelet's report. +// TestSuspendActor_Finalized_PauseReleasesReplacedSnapshot verifies that an +// async suspend keeping the previous async suspend's upload collects the +// external snapshot that upload replaced, and keeps the upload. The upload is +// recorded from the Checkpoint state, or before the suspend from atelet's +// report. func TestSuspendActor_Finalized_PauseReleasesReplacedSnapshot(t *testing.T) { const node = "node-1" memory := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY @@ -624,8 +625,8 @@ func TestSuspendActor_Finalized_PauseReleasesReplacedSnapshot(t *testing.T) { pending := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING for _, tc := range []struct { name string - // reported records the upload before the pause, as atelet's report - // does. Otherwise only the pause's Checkpoint state carries it. + // reported records the upload before the suspend, as atelet's report + // does. Otherwise only the suspend's Checkpoint state carries it. reported bool }{ {name: "recorded from the pause's Checkpoint state"}, @@ -648,8 +649,9 @@ func TestSuspendActor_Finalized_PauseReleasesReplacedSnapshot(t *testing.T) { }, }) - // The actor suspended to old, paused to snap-2 and resumed from - // it, and is now pausing to snap-3. atelet has uploaded snap-2. + // The actor suspended to old, suspended asynchronously to snap-2 and + // resumed from it, and is now suspending asynchronously to snap-3. + // atelet has uploaded snap-2. old := mustActorSnapshotURI(t, template, actor, "old") paused := mustActorSnapshotURI(t, template, actor, "snap-2") pausing := mustActorSnapshotURI(t, template, actor, "snap-3") @@ -1447,6 +1449,9 @@ func TestSuspendActor_AppliesCheckpointState(t *testing.T) { // state builds what Checkpoint returns, from the request it got. state func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState wantSnapshots []snapshotOutcome + // wantNoNode is set when atelet keeps no local copy of an async + // suspend's durable snapshot, so AssignedNode is cleared. + wantNoNode bool // wantCode, when set, is the error the suspend fails with. wantCode codes.Code }{{ @@ -1464,9 +1469,9 @@ func TestSuspendActor_AppliesCheckpointState(t *testing.T) { }, wantSnapshots: []snapshotOutcome{{generation: 1, fidelity: memory, durable: completed}, {generation: 2, fidelity: memory, locality: node, durable: pending}}, }, { - // A RUNNING actor resumed from a pause keeps that pause's snapshot, - // with its PENDING durable copy. atelet uploads it in the background, - // and the next pause records that. + // A RUNNING actor resumed from a local snapshot keeps that async + // suspend's snapshot, with its PENDING durable copy. atelet uploads it + // in the background, and the next async suspend records that. name: "async: the previous pause's upload atelet finished is recorded as COMPLETED", mode: suspendAsync, status: &ateapipb.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{newPausedSnapshot(t, 1, memory, "", "snap-1", node)}}, @@ -1485,6 +1490,36 @@ func TestSuspendActor_AppliesCheckpointState(t *testing.T) { return nil }, wantSnapshots: []snapshotOutcome{{generation: 2, fidelity: memory, locality: node, durable: pending}}, + }, { + // A retried async suspend can get a state where atelet already + // uploaded and detached the snapshot it committed on the first + // attempt. The snapshot is recorded as durable with no node + // locality, and AssignedNode is cleared so the actor can resume + // on any node. + name: "async: a snapshot atelet already detached is recorded without locality", + mode: suspendAsync, + status: &ateapipb.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{durable(1, completed)}}, + state: func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState { + return &ateletpb.ActorSnapshotState{ + LatestDurable: &ateletpb.SnapshotReport{Snapshot: req.GetSnapshot()}, + } + }, + wantSnapshots: []snapshotOutcome{{generation: 2, fidelity: memory, durable: completed}}, + wantNoNode: true, + }, { + // The suspend's first attempt was sync and kept no local copy, and + // the retry is async. An older snapshot's upload is still running. + name: "async: a durable snapshot atelet kept no copy of is recorded without locality", + mode: suspendAsync, + status: &ateapipb.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{durable(1, pending)}}, + state: func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState { + return &ateletpb.ActorSnapshotState{ + Uploading: report(1, uriOf(1), ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY), + LatestDurable: &ateletpb.SnapshotReport{Snapshot: req.GetSnapshot()}, + } + }, + wantSnapshots: []snapshotOutcome{{generation: 2, fidelity: memory, durable: completed}}, + wantNoNode: true, }, { name: "sync: the fidelity atelet reports is recorded", mode: suspendSync, @@ -1564,9 +1599,107 @@ func TestSuspendActor_AppliesCheckpointState(t *testing.T) { if got := stored.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Errorf("state = %v, want SUSPENDED", got) } + wantNode := node + if tt.mode == suspendSync || tt.wantNoNode { + wantNode = "" + } + if got := stored.GetStatus().GetAssignedNode(); got != wantNode { + t.Errorf("assigned_node = %q, want %q", got, wantNode) + } if diff := cmp.Diff(tt.wantSnapshots, describeSnapshots(stored.GetStatus()), cmp.AllowUnexported(snapshotOutcome{})); diff != "" { t.Errorf("snapshots mismatch (-want +got):\n%s", diff) } }) } } + +func TestSuspendActor_Finalized_RecordsLocalCopyWhenKept(t *testing.T) { + const node = "node-1" + memory := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + completed := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + localReport := func(snap *ateletpb.Snapshot) *ateletpb.ActorSnapshotState { + return &ateletpb.ActorSnapshotState{LatestLocal: &ateletpb.SnapshotReport{Snapshot: snap}} + } + variant := func(req *ateletpb.CheckpointRequest, mutate func(*ateletpb.Snapshot)) *ateletpb.Snapshot { + snap := proto.CloneOf(req.GetSnapshot()) + mutate(snap) + return snap + } + tests := []struct { + name string + state func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState + wantLocality string + }{{ + name: "kept", + state: func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState { + return localReport(req.GetSnapshot()) + }, + wantLocality: node, + }, { + name: "not kept", + state: func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState { + return &ateletpb.ActorSnapshotState{LatestDurable: &ateletpb.SnapshotReport{Snapshot: req.GetSnapshot()}} + }, + }, { + name: "uuid mismatch", + state: func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState { + return localReport(variant(req, func(snap *ateletpb.Snapshot) { snap.SnapshotUuid = "another-snapshot" })) + }, + }, { + name: "generation mismatch", + state: func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState { + return localReport(variant(req, func(snap *ateletpb.Snapshot) { snap.Generation = req.GetSnapshot().GetGeneration() - 1 })) + }, + }, { + name: "uri mismatch", + state: func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState { + return localReport(variant(req, func(snap *ateletpb.Snapshot) { + snap.Object = &ateletpb.ObjectStorage{SnapshotUri: "gs://bucket/root/another-snapshot"} + })) + }, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: testStorageLocation, PreferredFidelity: memory}) + actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} + created := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, + ActorTemplate: suspendTestTemplateRef(), + Status: &ateapipb.ActorStatus{ + State: ateapipb.ActorState_ACTOR_STATE_RUNNING, + AssignedNode: node, + WorkerAssignment: wireTestAssignment(), + }, + }) + seedWireWorker(t, ctx, persistence, assignmentOf(created)) + w, atelet := newWireCaptureWorkflow(t, persistence) + atelet.onCheckpoint = func(context.Context) error { + st := tt.state(atelet.checkpointRequest()) + atelet.mu.Lock() + atelet.checkpointState = st + atelet.mu.Unlock() + return nil + } + + if _, err := w.SuspendActor(ctx, actorRef, suspendSync); err != nil { + t.Fatalf("suspend: %v", err) + } + stored := mustReadActor(t, ctx, persistence, actorRef) + if got := stored.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + t.Errorf("state = %v, want SUSPENDED", got) + } + if got := stored.GetStatus().GetAssignedNode(); got != tt.wantLocality { + t.Errorf("AssignedNode = %q, want %q", got, tt.wantLocality) + } + if got := stored.GetStatus().GetWorkerAssignment(); got != nil { + t.Errorf("WorkerAssignment = %v, want cleared", got) + } + want := []snapshotOutcome{{generation: 1, fidelity: memory, locality: tt.wantLocality, durable: completed}} + if diff := cmp.Diff(want, describeSnapshots(stored.GetStatus()), cmp.AllowUnexported(snapshotOutcome{})); diff != "" { + t.Errorf("snapshots mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/cmd/ateapi/internal/controlapi/workflow_tag.go b/cmd/ateapi/internal/controlapi/workflow_tag.go index fa18332620..012092ae06 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag.go @@ -37,8 +37,8 @@ import ( // // The tag is built in 6 phases: // 1. Check that the name is free and the storage location is usable. -// 2. Make the actor's latest snapshot durable, if it is not uploaded yet: a -// paused actor's snapshot may still be uploading. +// 2. Make the actor's latest snapshot durable, if it is not uploaded yet: an +// asynchronously suspended actor's snapshot may still be uploading. // 3. Reserve the tag's name; the store assigns its UID. // 4. Record the snapshot's URI, under the tag's UID, before anything is // copied there. @@ -136,39 +136,48 @@ func (w *ActorWorkflow) DeleteTag(ctx context.Context, tagRef resources.TagRef, } defer lease.Close() - tag, err := w.loadTagForDelete(ctx, tagRef) + tag, err := w.ensureTagMarkedForDeletion(ctx, tagRef, precondition) if err != nil { return nil, err } - // Checked before the snapshot is collected: a stale caller must not - // reach that step. - if err := precondition.Check(tag.GetMetadata()); err != nil { - if errors.Is(err, store.ErrUIDConflict) { - return nil, apierror.Aborted("Tag %s does not have uid %s", tagRef, precondition.UID) - } - return nil, apierror.Aborted("concurrent update conflict, please retry") - } if err := w.ensureTagSnapshotReleased(ctx, tag); err != nil { return nil, err } - return w.finalizeTagDeleted(ctx, tagRef, precondition) + return w.finalizeTagDeleted(ctx, tag) } -// loadTagForDelete fetches the row the delete works from. The row records where -// the snapshot lives, so the work is rediscovered from it rather than rebuilt -// from the source actor, which may be long gone. -func (w *ActorWorkflow) loadTagForDelete(ctx context.Context, tagRef resources.TagRef) (_ *ateapipb.Tag, err error) { - ctx, done := stepSpan(ctx, "LoadTagForDelete") +// ensureTagMarkedForDeletion sets the tag's delete_time, checking precondition, +// and returns the stored copy. Skips when a previous attempt already marked the +// tag. +func (w *ActorWorkflow) ensureTagMarkedForDeletion(ctx context.Context, tagRef resources.TagRef, precondition store.DeletePreconditions) (_ *ateapipb.Tag, err error) { + ctx, done := stepSpan(ctx, "MarkTagForDeletion") defer func() { err = done(err) }() - tag, err := w.store.GetTag(ctx, tagRef) + var alreadyDeleting *ateapipb.Tag + marked, err := w.store.MarkTagForDeletion(ctx, tagRef, precondition, func(toUpdate *ateapipb.Tag) error { + if toUpdate.GetMetadata().GetDeleteTime() != nil { + alreadyDeleting = toUpdate + return errAlreadyMarked + } + return nil + }) + if errors.Is(err, errAlreadyMarked) { + markSkipped(ctx, "tag already marked for deletion") + return alreadyDeleting, nil + } if err != nil { + if errors.Is(err, store.ErrUIDConflict) { + return nil, apierror.Aborted("Tag %s does not have uid %s", tagRef, precondition.UID) + } + if errors.Is(err, store.ErrVersionConflict) { + return nil, apierror.Aborted("concurrent update conflict, please retry") + } if errors.Is(err, store.ErrNotFound) { return nil, apierror.NotFound("Tag %s not found", tagRef) } - return nil, fmt.Errorf("while getting tag %s: %w", tagRef, err) + return nil, fmt.Errorf("while marking tag %s for deletion: %w", tagRef, err) } - return tag, nil + return marked, nil } // ensureTagSnapshotReleased deletes the objects the tag's external snapshot is @@ -201,11 +210,13 @@ func (w *ActorWorkflow) ensureTagSnapshotReleased(ctx context.Context, tag *atea } // finalizeTagDeleted drops the row, once nothing it names is left behind. -func (w *ActorWorkflow) finalizeTagDeleted(ctx context.Context, tagRef resources.TagRef, precondition store.DeletePreconditions) (_ *ateapipb.Tag, err error) { +func (w *ActorWorkflow) finalizeTagDeleted(ctx context.Context, tag *ateapipb.Tag) (_ *ateapipb.Tag, err error) { ctx, done := stepSpan(ctx, "FinalizeTagDeleted") defer func() { err = done(err) }() - tag, err := w.store.DeleteTag(ctx, tagRef, precondition) + tagRef := resources.TagRefFromTag(tag) + precondition := store.DeletePreconditions{UID: tag.GetMetadata().GetUid()} + tag, err = w.store.DeleteTag(ctx, tagRef, precondition) if err != nil { if errors.Is(err, store.ErrNotFound) { return nil, apierror.NotFound("Tag %s not found", tagRef) @@ -269,11 +280,11 @@ func (w *ActorWorkflow) checkTagCreatable(ctx context.Context, tagRef resources. } // ensureTagSourceSnapshotDurable makes the actor's latest snapshot durable when -// it is not uploaded yet, the shape a pause leaves until atelet's background -// upload is recorded. It asks the atelet on the snapshot's node to upload it -// with PromoteSnapshot, which keeps the local copy so the actor still resumes -// on that node, and records the upload on the actor. It returns the actor as -// stored afterwards. +// it is not uploaded yet, the shape an asynchronous suspend leaves until +// atelet's background upload is recorded. It asks the atelet on the snapshot's +// node to upload it with PromoteSnapshot, which keeps the local copy so the +// actor still resumes on that node, and records the upload on the actor. It +// returns the actor as stored afterwards. // // A failed upload does not crash the actor: the tag fails and the actor is // left as it was. diff --git a/cmd/ateapi/internal/controlapi/workflow_tag_test.go b/cmd/ateapi/internal/controlapi/workflow_tag_test.go index 8820c30890..b53da8cd92 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag_test.go @@ -241,8 +241,8 @@ func TestTagActorSnapshot_TagsNewestExternalSnapshot(t *testing.T) { // snapshot is always at generation extGen. lastAssigned int32 extGen int32 - // locality is set on the external snapshot, as when a paused actor's - // local snapshot was uploaded in place. + // locality is set on the external snapshot, as when the snapshot also + // has a local copy on the node. locality string }{ { @@ -287,11 +287,12 @@ func TestTagActorSnapshot_TagsNewestExternalSnapshot(t *testing.T) { } } -// TestTagActorSnapshot_PromotesLocalSnapshot verifies that tagging a paused -// actor whose snapshot is not uploaded yet has the atelet on the snapshot's -// node upload it first, records the upload on the actor, and tags that -// snapshot. A failed upload fails the tag without reserving its name and leaves -// the actor as it was. A create that cannot succeed fails before the upload. +// TestTagActorSnapshot_PromotesLocalSnapshot verifies that tagging a suspended +// actor whose local snapshot is not uploaded yet has the atelet on the +// snapshot's node upload it first, records the upload on the actor, and tags +// that snapshot. A failed upload fails the tag without reserving its name and +// leaves the actor as it was. A create that cannot succeed fails before the +// upload. func TestTagActorSnapshot_PromotesLocalSnapshot(t *testing.T) { // reported is the state atelet answers with once it made snap durable. reported := func(snap *ateapipb.Snapshot) *ateletpb.ActorSnapshotState { @@ -306,22 +307,22 @@ func TestTagActorSnapshot_PromotesLocalSnapshot(t *testing.T) { tests := []struct { name string - // olderDurable seeds a COMPLETED snapshot below the paused one. + // olderDurable seeds a COMPLETED snapshot below the local one. olderDurable bool - // uploaded records the paused snapshot's durable copy COMPLETED. + // uploaded records the local snapshot's durable copy COMPLETED. uploaded bool - // nodeLost clears the paused snapshot's locality, as when its node + // nodeLost clears the local snapshot's locality, as when its node // was lost before the upload finished. nodeLost bool // nameTaken seeds a tag under the name the create asks for. nameTaken bool // badLocation points the template at an unusable storage location. badLocation bool - // noTemplateUID records no template the paused snapshot was built + // noTemplateUID records no template the local snapshot was built // under, a broken row. noTemplateUID bool // promoteState builds the state PromoteSnapshot answers with from the - // paused snapshot. + // local snapshot. promoteState func(paused *ateapipb.Snapshot) *ateletpb.ActorSnapshotState promoteErr error wantPromote bool @@ -566,13 +567,13 @@ func TestTagActorSnapshot_PromotesLocalSnapshot(t *testing.T) { return } - // The paused snapshot is now durable and stays on its node. + // The local snapshot is now durable and stays on its node. wantPaused := proto.CloneOf(paused) wantPaused.DurableSnapshot.Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED if diff := cmp.Diff(wantPaused, findSnapshotByGeneration(stored.GetStatus().GetSnapshots(), 2), protocmp.Transform()); diff != "" { t.Errorf("the actor's latest snapshot mismatch (-want +got):\n%s", diff) } - // The tag copied the paused snapshot, not the older one. + // The tag copied the local snapshot, not the older one. if diff := cmp.Diff([]string{"manifest.json", "memory.zst"}, objects.Snapshot(t, mustParseSnapshotURI(t, tagSnapshot))); diff != "" { t.Errorf("the tag's external snapshot mismatch (-want +got):\n%s", diff) } @@ -867,16 +868,35 @@ func TestDeleteTag_ReleasesExternalSnapshot(t *testing.T) { if _, err := w.DeleteTag(ctx, tagRef, store.DeletePreconditions{}); !isObjectStoreErr(err) { t.Fatalf("DeleteTag = %v, want an error wrapping %v", err, errObjectStore) } - if _, err := persistence.GetTag(ctx, tagRef); err != nil { + admitted, err := persistence.GetTag(ctx, tagRef) + if err != nil { t.Fatalf("GetTag after the failure: %v", err) } + if admitted.GetMetadata().GetDeleteTime() == nil { + t.Fatal("a failed delete left delete_time unset") + } + + if _, err := w.DeleteTag(ctx, tagRef, store.DeletePreconditions{}); !isObjectStoreErr(err) { + t.Fatalf("retried DeleteTag = %v, want an error wrapping %v", err, errObjectStore) + } + retried, err := persistence.GetTag(ctx, tagRef) + if err != nil { + t.Fatalf("GetTag after the retried failure: %v", err) + } + if got, want := retried.GetMetadata().GetVersion(), admitted.GetMetadata().GetVersion(); got != want { + t.Errorf("a retried delete moved the version to %d, want %d", got, want) + } // Simulates a retried deletion. Now, the object deletion succeeds, // so we can remove the row from the DB. objects.OnDelete = nil - if _, err := w.DeleteTag(ctx, tagRef, store.DeletePreconditions{}); err != nil { + deleted, err := w.DeleteTag(ctx, tagRef, store.DeletePreconditions{}) + if err != nil { t.Fatalf("retried DeleteTag: %v", err) } + if !deleted.GetMetadata().GetDeleteTime().AsTime().Equal(admitted.GetMetadata().GetDeleteTime().AsTime()) { + t.Errorf("the retry re-stamped delete_time: %v, want %v", deleted.GetMetadata().GetDeleteTime().AsTime(), admitted.GetMetadata().GetDeleteTime().AsTime()) + } if got := objects.Snapshot(t, uri); len(got) != 0 { t.Errorf("the tag's external snapshot still holds %v, want it collected", got) } diff --git a/cmd/ateapi/internal/controlapi/workflow_template_delete.go b/cmd/ateapi/internal/controlapi/workflow_template_delete.go index 2d5245f779..e8d4e087c5 100644 --- a/cmd/ateapi/internal/controlapi/workflow_template_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_template_delete.go @@ -40,18 +40,10 @@ func (w *ActorWorkflow) DeleteActorTemplate(ctx context.Context, templateRef res } defer lease.Close() - tmpl, err := w.loadTemplateForDelete(ctx, templateRef) + tmpl, err := w.ensureTemplateMarkedForDeletion(ctx, templateRef, precondition) if err != nil { return nil, err } - // Checked before the golden actor and tag go: a stale caller must not - // destroy them. - if err := precondition.Check(tmpl.GetMetadata()); err != nil { - if errors.Is(err, store.ErrUIDConflict) { - return nil, apierror.Aborted("ActorTemplate %s does not have uid %s", templateRef, precondition.UID) - } - return nil, apierror.Aborted("concurrent update conflict, please retry") - } // Both are named after the template's UID, in the template's atespace. goldenName := tmpl.GetMetadata().GetUid() @@ -61,22 +53,41 @@ func (w *ActorWorkflow) DeleteActorTemplate(ctx context.Context, templateRef res if err := w.ensureGoldenTagDeleted(ctx, resources.TagRef{Atespace: templateRef.Atespace, Name: goldenName}); err != nil { return nil, err } - return w.finalizeTemplateDeleted(ctx, templateRef, precondition) + return w.finalizeTemplateDeleted(ctx, tmpl) } -// loadTemplateForDelete fetches the current template record. -func (w *ActorWorkflow) loadTemplateForDelete(ctx context.Context, templateRef resources.ActorTemplateRef) (_ *ateapipb.ActorTemplate, err error) { - ctx, done := stepSpan(ctx, "LoadTemplateForDelete") +// ensureTemplateMarkedForDeletion sets the template's delete_time, checking +// precondition, and returns the stored copy. Skips when a previous attempt +// already marked the template. +func (w *ActorWorkflow) ensureTemplateMarkedForDeletion(ctx context.Context, templateRef resources.ActorTemplateRef, precondition store.DeletePreconditions) (_ *ateapipb.ActorTemplate, err error) { + ctx, done := stepSpan(ctx, "MarkTemplateForDeletion") defer func() { err = done(err) }() - tmpl, err := w.store.GetActorTemplate(ctx, templateRef) + var alreadyDeleting *ateapipb.ActorTemplate + marked, err := w.store.MarkActorTemplateForDeletion(ctx, templateRef, precondition, func(toUpdate *ateapipb.ActorTemplate) error { + if toUpdate.GetMetadata().GetDeleteTime() != nil { + alreadyDeleting = toUpdate + return errAlreadyMarked + } + return nil + }) + if errors.Is(err, errAlreadyMarked) { + markSkipped(ctx, "template already marked for deletion") + return alreadyDeleting, nil + } if err != nil { + if errors.Is(err, store.ErrUIDConflict) { + return nil, apierror.Aborted("ActorTemplate %s does not have uid %s", templateRef, precondition.UID) + } + if errors.Is(err, store.ErrVersionConflict) { + return nil, apierror.Aborted("concurrent update conflict, please retry") + } if errors.Is(err, store.ErrNotFound) { return nil, apierror.NotFound("ActorTemplate %s not found", templateRef) } - return nil, fmt.Errorf("while getting actor template %s: %w", templateRef, err) + return nil, fmt.Errorf("while marking actor template %s for deletion: %w", templateRef, err) } - return tmpl, nil + return marked, nil } // ensureGoldenActorDeleted removes the template's golden actor, whatever state @@ -114,10 +125,12 @@ func (w *ActorWorkflow) ensureGoldenTagDeleted(ctx context.Context, goldenTagRef // finalizeTemplateDeleted removes the template from the store and returns the // deleted record. -func (w *ActorWorkflow) finalizeTemplateDeleted(ctx context.Context, templateRef resources.ActorTemplateRef, precondition store.DeletePreconditions) (_ *ateapipb.ActorTemplate, err error) { +func (w *ActorWorkflow) finalizeTemplateDeleted(ctx context.Context, tmpl *ateapipb.ActorTemplate) (_ *ateapipb.ActorTemplate, err error) { ctx, done := stepSpan(ctx, "FinalizeTemplateDeleted") defer func() { err = done(err) }() + templateRef := resources.ActorTemplateRefFromActorTemplate(tmpl) + precondition := store.DeletePreconditions{UID: tmpl.GetMetadata().GetUid()} deleted, err := w.store.DeleteActorTemplate(ctx, templateRef, precondition) if err != nil { if errors.Is(err, store.ErrNotFound) { diff --git a/cmd/ateapi/internal/controlapi/workflow_test.go b/cmd/ateapi/internal/controlapi/workflow_test.go index fe24540cbd..ef88cf37ff 100644 --- a/cmd/ateapi/internal/controlapi/workflow_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_test.go @@ -21,9 +21,11 @@ import ( "go.ate.dev/substrate/cmd/ateapi/internal/store" "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" "go.ate.dev/substrate/internal/actorevent" + "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/ateattr" "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc/codes" ) // TestActorStateChangeRecords drives each transition that commits a new state @@ -87,10 +89,14 @@ func TestActorStateChangeRecords(t *testing.T) { { name: "delete marks the actor deleting", seedState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - transition: func(t *testing.T, w *ActorWorkflow, ref resources.ActorRef, actor *ateapipb.Actor, _ *ateapipb.ActorTemplate) { - if _, err := w.ensureMarkedDeleting(context.Background(), ref, actor); err != nil { + transition: func(t *testing.T, w *ActorWorkflow, ref resources.ActorRef, _ *ateapipb.Actor, _ *ateapipb.ActorTemplate) { + if _, err := w.ensureMarkedDeleting(context.Background(), ref, store.DeletePreconditions{}); err != nil { t.Fatalf("ensureMarkedDeleting: %v", err) } + // A retry finds the actor already marked and writes no second record. + if _, err := w.ensureMarkedDeleting(context.Background(), ref, store.DeletePreconditions{}); err != nil { + t.Fatalf("retried ensureMarkedDeleting: %v", err) + } }, wantOp: ateattr.OperationDelete, wantState: ateattr.ActorStateDeleting, @@ -168,9 +174,9 @@ func TestActorStateChangeRecords(t *testing.T) { } } -// markOnly runs a suspend or pause on a RUNNING actor only as far as its mark -// step: the actor gets a worker to checkpoint on, but the atelet dial fails, -// so the SUSPENDING mark is the only transition the workflow commits. +// markOnly runs a suspend (sync or async) on a RUNNING actor only as far as its +// mark step: the actor gets a worker to checkpoint on, but the atelet dial +// fails, so the SUSPENDING mark is the only transition the workflow commits. func markOnly(t *testing.T, w *ActorWorkflow, ref resources.ActorRef, actor *ateapipb.Actor, mode suspendMode) { t.Helper() ctx := context.Background() @@ -238,9 +244,9 @@ func TestActorDeletedRecord(t *testing.T) { actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_DELETING) - actor, err := persistence.GetActor(ctx, actorRef) + actor, err := persistence.MarkActorForDeletion(ctx, actorRef, store.DeletePreconditions{}, nil) if err != nil { - t.Fatalf("get actor: %v", err) + t.Fatalf("mark actor: %v", err) } w := &ActorWorkflow{store: persistence} @@ -283,7 +289,7 @@ func TestActorStateChangeRecordSkippedOnConflict(t *testing.T) { } actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} - seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDED) + seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_RUNNING) stale, err := persistence.GetActor(ctx, actorRef) if err != nil { t.Fatalf("get actor: %v", err) @@ -301,8 +307,8 @@ func TestActorStateChangeRecordSkippedOnConflict(t *testing.T) { } w := &ActorWorkflow{store: persistence} - if _, err := w.ensureMarkedDeleting(ctx, actorRef, stale); err == nil { - t.Fatal("ensureMarkedDeleting on a stale actor = nil, want a conflict") + if _, err := w.ensureMarkedDeleting(ctx, actorRef, store.DeletePreconditions{Version: stale.GetMetadata().GetVersion()}); apierror.Code(err) != codes.Aborted { + t.Fatalf("ensureMarkedDeleting with a stale guard = %v, want ABORTED", err) } if len(*records) != 0 { t.Errorf("got %d state records from a losing writer, want 0: %v", len(*records), *records) diff --git a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go index 3520bf9229..51f3477504 100644 --- a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go @@ -158,9 +158,10 @@ func seedWorkflowActor(t *testing.T, ctx context.Context, st store.Interface, ac storetest.MustCreateActor(t, ctx, st, actor) } -// withPausedShape seeds the shape a pause leaves behind: a node-local -// snapshot at the latest generation with a PENDING durable copy, pinned to -// node. Pair it with ACTOR_STATE_SUSPENDED to seed a paused actor. +// withPausedShape seeds the shape an asynchronous suspend leaves behind: a +// node-local snapshot at the latest generation with a PENDING durable copy, +// pinned to node. Pair it with ACTOR_STATE_SUSPENDED to seed an asynchronously +// suspended actor. func withPausedShape(t *testing.T, node string) func(*ateapipb.Actor) { t.Helper() snap := newPausedSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "pause-snap", node) @@ -171,8 +172,9 @@ func withPausedShape(t *testing.T, node string) func(*ateapipb.Actor) { } } -// withUploadedPausedShape seeds a paused actor whose snapshot atelet has since -// uploaded: the paused shape, with its durable copy COMPLETED. +// withUploadedPausedShape seeds an asynchronously suspended actor whose +// snapshot atelet has since uploaded: the local-snapshot shape, with its +// durable copy COMPLETED. func withUploadedPausedShape(t *testing.T, node string) func(*ateapipb.Actor) { t.Helper() paused := withPausedShape(t, node) @@ -190,9 +192,10 @@ func newInProgressSnapshot(t *testing.T, gen int32, fidelity ateapipb.SnapshotFi return newDurableSnapshot(gen, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, fidelity, "", uuid, someActorSnapshotURI(t, testStorageLocation, "team-a", uuid), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) } -// newPausedSnapshot builds the snapshot a pause leaves on node locality at -// generation gen: a node-local checkpoint whose PENDING durable copy atelet -// uploads in the background. With no locality it is the in-progress mark. +// newPausedSnapshot builds the snapshot an asynchronous suspend leaves on node +// locality at generation gen: a node-local checkpoint whose PENDING durable +// copy atelet uploads in the background. With no locality it is the in-progress +// mark. func newPausedSnapshot(t *testing.T, gen int32, fidelity ateapipb.SnapshotFidelity, templateUID, uuid, locality string) *ateapipb.Snapshot { t.Helper() snap := newInProgressSnapshot(t, gen, fidelity, uuid) @@ -253,9 +256,8 @@ func suspendTestTemplateRef() *ateapipb.ObjectRef { return &ateapipb.ObjectRef{Atespace: "ns", Name: "tmpl1"} } -// seedSuspendTemplate stores the ActorTemplate the suspend and pause workflow -// tests run their actors from, with the given snapshot config, and returns it -// as stored. +// seedSuspendTemplate stores the ActorTemplate the suspend workflow tests run +// their actors from, with the given snapshot config, and returns it as stored. func seedSuspendTemplate(t *testing.T, ctx context.Context, st store.Interface, snapshotConfig *ateapipb.SnapshotConfig) *ateapipb.ActorTemplate { t.Helper() ref := suspendTestTemplateRef() @@ -311,8 +313,8 @@ func assignmentOf(actor *ateapipb.Actor) *ateapipb.ActorAssignment { } // newMarkOnlyWorkflow builds an ActorWorkflow whose atelet dial always fails, -// so a suspend or pause stops right after its mark step and leaves the actor -// as that step committed it. +// so a suspend stops right after its mark step and leaves the actor as that +// step committed it. func newMarkOnlyWorkflow(st store.Interface) *ActorWorkflow { return &ActorWorkflow{store: st, dialer: newDanglingDialer(), workflowDeadline: time.Minute} } diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go index 5aeec6555d..7d12051f2c 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go @@ -38,36 +38,11 @@ import ( // belongs to the caller, which knows whether that is the state it wanted. func (w *WorkerWorkflow) DeleteWorker(ctx context.Context, name string, precondition store.DeletePreconditions) (*ateapipb.Worker, error) { ctx = preview.ContextWithOptIn(ctx) - worker, err := w.loadWorkerForDelete(ctx, name) + worker, err := w.ensureMarkedForDeletion(ctx, name, precondition) if err != nil { return nil, err } - // Checked against the Worker the caller observed, before the drain below - // moves the version. - if err := precondition.Check(worker.GetMetadata()); err != nil { - switch { - case errors.Is(err, store.ErrUIDConflict): - return nil, apierror.Aborted("Worker %s does not have uid %s", name, precondition.UID) - case errors.Is(err, store.ErrVersionConflict): - return nil, apierror.Aborted("concurrent update conflict, please retry") - } - return nil, err - } - - // Scheduling only places on ACTIVE Workers, so draining first stops a - // concurrent resume from binding to a page the sweep has already passed. - // The delete would cascade that assignment away and leave the Actor - // pointing at a Worker that is gone. - worker, err = w.ensureDraining(ctx, worker) - if err != nil { - return nil, err - } - - // The drain moved the version, so only the uid guard still means anything: - // a Worker replaced by a new incarnation mid-delete is still refused. - precondition.Version = 0 - // Order matters: the delete is what erases the Actor's pointer at the // Worker, so a failed release has to leave the record in place for the // caller to rediscover and retry. @@ -75,46 +50,49 @@ func (w *WorkerWorkflow) DeleteWorker(ctx context.Context, name string, precondi return nil, err } - return w.finalizeDeleted(ctx, name, precondition) + return w.finalizeDeleted(ctx, worker) } -// loadWorkerForDelete fetches the current worker record. Reading before any of -// the release runs is also what reports an absent Worker as such. -func (w *WorkerWorkflow) loadWorkerForDelete(ctx context.Context, name string) (_ *ateapipb.Worker, err error) { - ctx, done := stepSpan(ctx, "LoadWorkerForDelete") +// ensureMarkedForDeletion sets the Worker's delete_time, checking +// precondition, and drains it, returning the stored copy. Skips when a +// previous attempt already marked the Worker. +func (w *WorkerWorkflow) ensureMarkedForDeletion(ctx context.Context, name string, precondition store.DeletePreconditions) (_ *ateapipb.Worker, err error) { + ctx, done := stepSpan(ctx, "MarkWorkerForDeletion") defer func() { err = done(err) }() - worker, err := w.store.GetWorker(ctx, name) + var alreadyDeleting *ateapipb.Worker + marked, err := w.store.MarkWorkerForDeletion(ctx, name, precondition, func(toUpdate *ateapipb.Worker) error { + if toUpdate.GetMetadata().GetDeleteTime() != nil { + alreadyDeleting = toUpdate + return errAlreadyMarked + } + // Scheduling only places on ACTIVE Workers, so draining first stops a + // concurrent resume from binding to a page the sweep has already passed. + // The delete would cascade that assignment away and leave the Actor + // pointing at a Worker that is gone. + if toUpdate.Status == nil { + toUpdate.Status = &ateapipb.WorkerStatus{} + } + toUpdate.Status.State = ateapipb.WorkerState_WORKER_STATE_DRAINING + return nil + }) + if errors.Is(err, errAlreadyMarked) { + markSkipped(ctx, "worker already marked for deletion") + return alreadyDeleting, nil + } if err != nil { + if errors.Is(err, store.ErrUIDConflict) { + return nil, apierror.Aborted("Worker %s does not have uid %s", name, precondition.UID) + } + if errors.Is(err, store.ErrVersionConflict) { + return nil, apierror.Aborted("concurrent update conflict, please retry") + } if errors.Is(err, store.ErrNotFound) { return nil, apierror.NotFound("Worker %s not found", name) } - return nil, fmt.Errorf("while fetching worker: %w", err) + return nil, fmt.Errorf("while marking worker for deletion: %w", err) } - return worker, nil -} - -// ensureDraining moves the Worker out of the state scheduling will place on, -// and is a no-op for one already draining. -func (w *WorkerWorkflow) ensureDraining(ctx context.Context, worker *ateapipb.Worker) (_ *ateapipb.Worker, err error) { - ctx, done := stepSpan(ctx, "DrainWorkerForDelete") - defer func() { err = done(err) }() - - if worker.GetStatus().GetState() == ateapipb.WorkerState_WORKER_STATE_DRAINING { - return worker, nil - } - drained, err := w.store.UpdateWorker(ctx, worker.GetMetadata().GetName(), store.PreconditionFrom(worker), - func(toUpdate *ateapipb.Worker) error { - if toUpdate.Status == nil { - toUpdate.Status = &ateapipb.WorkerStatus{} - } - toUpdate.Status.State = ateapipb.WorkerState_WORKER_STATE_DRAINING - return nil - }) - if err != nil { - return nil, fmt.Errorf("while draining worker for delete: %w", err) - } - return drained, nil + return marked, nil } // ensureBoundActorsReleased resets every Actor bound to the Worker. @@ -198,10 +176,19 @@ func (w *WorkerWorkflow) releaseBoundActor(ctx context.Context, worker *ateapipb // crashBoundActor moves an Actor that lost its sandbox on the Worker to // ACTOR_STATE_CRASHED and clears its pointers at the Worker. crashMsg is -// recorded in the Actor's status. The Actor's assignment row in the worker is -// left for the caller to release. +// recorded in the Actor's status. An Actor in ACTOR_STATE_DELETING is left +// untouched. The Actor's assignment row in the worker is left for the caller to +// release. // A concurrent write to the Actor fails this as ABORTED. func (w *WorkerWorkflow) crashBoundActor(ctx context.Context, worker *ateapipb.Worker, actorRef resources.ActorRef, actor *ateapipb.Actor, logMsg, crashMsg string) error { + // DELETING is final. A CRASHED actor can be reverted and resumed, and the + // delete may already have collected its volumes and snapshots. The delete's + // retry clears the actor's pointers. + if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_DELETING { + markSkipped(ctx, "actor is DELETING") + return nil + } + name := worker.GetMetadata().GetName() opName := ateattr.OperationUnknown switch actor.GetStatus().GetState() { @@ -211,14 +198,13 @@ func (w *WorkerWorkflow) crashBoundActor(ctx context.Context, worker *ateapipb.W opName = ateattr.OperationSuspend case ateapipb.ActorState_ACTOR_STATE_REVERTING: opName = ateattr.OperationRevert - case ateapipb.ActorState_ACTOR_STATE_DELETING: - opName = ateattr.OperationDelete } wasAlreadyCrashed := actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_CRASHED // Snapshot crash attributes before pod and pool pointers are cleared on actor. - crashAttrs := ateattr.ActorMetricAttributes(actor, worker.GetSandboxClass(), opName) + sandboxClass := actor.GetStatus().GetWorkerAssignment().GetSandboxRuntime().GetSandboxClass() + crashAttrs := ateattr.ActorMetricAttributes(actor, sandboxClass, opName) slog.LogAttrs(ctx, slog.LevelInfo, logMsg, append(ateattr.ActorLogAttrs(resources.ActorAttributionFromActor(actor)), @@ -250,13 +236,16 @@ func (w *WorkerWorkflow) crashBoundActor(ctx context.Context, worker *ateapipb.W } // finalizeDeleted removes the worker from the store and returns the deleted -// record. The request's guards are carried down as delete preconditions, so a -// worker that moved on since the caller read it is reported as a conflict rather -// than removed. -func (w *WorkerWorkflow) finalizeDeleted(ctx context.Context, name string, precondition store.DeletePreconditions) (_ *ateapipb.Worker, err error) { +// record. +func (w *WorkerWorkflow) finalizeDeleted(ctx context.Context, worker *ateapipb.Worker) (_ *ateapipb.Worker, err error) { ctx, done := stepSpan(ctx, "FinalizeDeleted") defer func() { err = done(err) }() + name := worker.GetMetadata().GetName() + // Only the uid guards the delete. Each actor the sweep releases updates the + // Worker's allocation and moves its version. The uid still refuses a Worker + // replaced by a new incarnation. + precondition := store.DeletePreconditions{UID: worker.GetMetadata().GetUid()} deleted, err := w.store.DeleteWorker(ctx, name, precondition) if err != nil { switch { diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go index 9db03ecf27..d408a328c9 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go @@ -61,6 +61,7 @@ func seedAPIActor(t *testing.T, ctx context.Context, persistence store.Interface WorkerPod: "worker-pod-1", WorkerPodUid: apiWorkerName, WorkerPodIps: []string{"10.1.2.3"}, + SandboxRuntime: &ateapipb.ActorSandboxRuntime{SandboxClass: "gvisor"}, }, }, } @@ -147,8 +148,9 @@ func TestDeleteWorkerWorkflow_ReleasesBoundActor(t *testing.T) { // The Actor's state when its pod vanished decides what the release does: one // that had already suspended saved its state cleanly and stays resumable, while -// anything still in flight crashes and is counted. An Actor already counted as -// crashed is released again without being counted twice. +// anything still in flight crashes and is counted. An Actor being deleted stays +// DELETING so it can never be resumed. An Actor already counted as crashed is +// released again without being counted twice. func TestDeleteWorkerWorkflow_ReleasedActorStateTransitions(t *testing.T) { tests := []struct { name string @@ -161,7 +163,7 @@ func TestDeleteWorkerWorkflow_ReleasedActorStateTransitions(t *testing.T) { {name: "resuming becomes crashed", start: ateapipb.ActorState_ACTOR_STATE_RESUMING, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantOp: ateattr.OperationResume, wantMetric: true}, {name: "suspending becomes crashed", start: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantOp: ateattr.OperationSuspend, wantMetric: true}, {name: "reverting becomes crashed", start: ateapipb.ActorState_ACTOR_STATE_REVERTING, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantOp: ateattr.OperationRevert, wantMetric: true}, - {name: "deleting becomes crashed", start: ateapipb.ActorState_ACTOR_STATE_DELETING, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantOp: ateattr.OperationDelete, wantMetric: true}, + {name: "deleting stays deleting", start: ateapipb.ActorState_ACTOR_STATE_DELETING, wantState: ateapipb.ActorState_ACTOR_STATE_DELETING, wantMetric: false}, {name: "suspended stays suspended", start: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, wantState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, wantMetric: false}, {name: "crashed is not counted twice", start: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantMetric: false}, } @@ -317,8 +319,22 @@ func TestDeleteWorkerWorkflow_FailedReleaseKeepsWorker(t *testing.T) { t.Errorf("DeleteWorker() code = %v (err %v), want %v", got, err, tc.wantCode) } - if _, err := persistence.GetWorker(ctx, apiWorkerName); err != nil { - t.Errorf("worker gone after a failed release: %v", err) + worker, err := persistence.GetWorker(ctx, apiWorkerName) + if err != nil { + t.Fatalf("worker gone after a failed release: %v", err) + } + if worker.GetMetadata().GetDeleteTime() == nil { + t.Errorf("a failed delete left delete_time unset") + } + if _, err := wf.DeleteWorker(ctx, apiWorkerName, store.DeletePreconditions{}); err == nil { + t.Fatal("retried DeleteWorker() = nil error, want the release failure reported") + } + retried, err := persistence.GetWorker(ctx, apiWorkerName) + if err != nil { + t.Fatalf("worker gone after a retried failed release: %v", err) + } + if got, want := retried.GetMetadata().GetVersion(), worker.GetMetadata().GetVersion(); got != want { + t.Errorf("a retried delete moved the worker version to %d, want %d", got, want) } got, err := persistence.GetActor(ctx, apiActorRef) if err != nil { @@ -362,7 +378,7 @@ func TestDeleteWorkerWorkflow_AbsentReportsNotFoundThroughStepWrap(t *testing.T) if got := apierror.Code(err); got != codes.NotFound { t.Fatalf("DeleteWorker() code = %v (err %v), want %v", got, err, codes.NotFound) } - if want := "step LoadWorkerForDelete"; !strings.Contains(err.Error(), want) { + if want := "step MarkWorkerForDeletion"; !strings.Contains(err.Error(), want) { t.Errorf("DeleteWorker() error = %q, want it to name the step it failed at (%q)", err, want) } } @@ -377,3 +393,30 @@ type failingUpdateActorStore struct { func (f failingUpdateActorStore) UpdateActor(context.Context, resources.ActorRef, store.Precondition, func(*ateapipb.Actor) error) (*ateapipb.Actor, error) { return nil, f.err } + +func TestDeleteWorkerWorkflow_DrainedWorkerIsStillAdmitted(t *testing.T) { + ctx := context.Background() + wf, persistence := newWorkerDeleteWorkflow(t) + worker := seedAPIWorker(t, ctx, persistence, validWorker(apiWorkerName)) + drained, err := persistence.UpdateWorker(ctx, apiWorkerName, store.PreconditionFrom(worker), func(toUpdate *ateapipb.Worker) error { + if toUpdate.Status == nil { + toUpdate.Status = &ateapipb.WorkerStatus{} + } + toUpdate.Status.State = ateapipb.WorkerState_WORKER_STATE_DRAINING + return nil + }) + if err != nil { + t.Fatalf("UpdateWorker failed: %v", err) + } + + deleted, err := wf.DeleteWorker(ctx, apiWorkerName, store.DeletePreconditions{}) + if err != nil { + t.Fatalf("DeleteWorker() failed: %v", err) + } + if deleted.GetMetadata().GetDeleteTime() == nil { + t.Fatal("an operator-drained worker was removed without being marked for deletion") + } + if got, want := deleted.GetMetadata().GetVersion(), drained.GetMetadata().GetVersion()+1; got != want { + t.Errorf("version = %d, want %d: admission is one write", got, want) + } +} diff --git a/cmd/ateapi/internal/defaults/defaults.go b/cmd/ateapi/internal/defaults/defaults.go index 21ef87abad..59b09648b6 100644 --- a/cmd/ateapi/internal/defaults/defaults.go +++ b/cmd/ateapi/internal/defaults/defaults.go @@ -56,6 +56,7 @@ func applyActorTemplateDefaults(t *ateapipb.ActorTemplate) { t.SnapshotConfig = &ateapipb.SnapshotConfig{} } applySnapshotConfigDefaults(t.SnapshotConfig) + applyEgressRuleDefaults(t.GetGoldenEgressPolicy().GetRules()) for _, c := range t.Containers { applyContainerDefaults(c) } @@ -98,14 +99,15 @@ func applyActorDefaults(*ateapipb.Actor) {} func applyAtespaceDefaults(*ateapipb.Atespace) {} -// applyEgressPolicyDefaults fills the port of an http or https rule that +func applyEgressPolicyDefaults(p *ateapipb.EgressPolicy) { + applyEgressRuleDefaults(p.GetRules()) +} + +// applyEgressRuleDefaults fills the port of an http or https rule that // names none: the protocol's well-known port. A tls_passthrough rule has no // default; validation requires its ports. -func applyEgressPolicyDefaults(p *ateapipb.EgressPolicy) { - if p == nil { - return - } - for _, r := range p.Rules { +func applyEgressRuleDefaults(rules []*ateapipb.EgressRule) { + for _, r := range rules { switch { case r.GetHttp() != nil && r.Http.Ports == nil: r.Http.Ports = &ateapipb.Ports{Numbers: []int32{80}} diff --git a/cmd/ateapi/internal/defaults/defaults_test.go b/cmd/ateapi/internal/defaults/defaults_test.go index 84f281e6b3..c798c7c35e 100644 --- a/cmd/ateapi/internal/defaults/defaults_test.go +++ b/cmd/ateapi/internal/defaults/defaults_test.go @@ -137,6 +137,20 @@ func TestApply(t *testing.T) { {TlsPassthrough: &ateapipb.TlsPassthroughRule{Hostnames: []string{"d.example.com"}}}, {}, }}, + }, { + name: "template defaults egress ports", + in: &ateapipb.ActorTemplate{GoldenEgressPolicy: &ateapipb.EgressPolicyTemplate{Rules: []*ateapipb.EgressRule{ + {Http: &ateapipb.HttpRule{Hostnames: []string{"api.example.com"}}}, + {Https: &ateapipb.HttpsRule{Hostnames: []string{"api.example.com"}}}, + {Https: &ateapipb.HttpsRule{Hostnames: []string{"custom.example.com"}, Ports: &ateapipb.Ports{Numbers: []int32{8443}}}}, + {TlsPassthrough: &ateapipb.TlsPassthroughRule{Hostnames: []string{"tcp.example.com"}}}, + }}}, + want: &ateapipb.ActorTemplate{SnapshotConfig: defaultSnapshotConfig(), GoldenEgressPolicy: &ateapipb.EgressPolicyTemplate{Rules: []*ateapipb.EgressRule{ + {Http: &ateapipb.HttpRule{Hostnames: []string{"api.example.com"}, Ports: &ateapipb.Ports{Numbers: []int32{80}}}}, + {Https: &ateapipb.HttpsRule{Hostnames: []string{"api.example.com"}, Ports: &ateapipb.Ports{Numbers: []int32{443}}}}, + {Https: &ateapipb.HttpsRule{Hostnames: []string{"custom.example.com"}, Ports: &ateapipb.Ports{Numbers: []int32{8443}}}}, + {TlsPassthrough: &ateapipb.TlsPassthroughRule{Hostnames: []string{"tcp.example.com"}}}, + }}}, }, { name: "tag has no defaults", in: &ateapipb.Tag{Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED}, diff --git a/cmd/ateapi/internal/scheduling/scheduling.go b/cmd/ateapi/internal/scheduling/scheduling.go index f7f4f91dd3..6cbfdccbc2 100644 --- a/cmd/ateapi/internal/scheduling/scheduling.go +++ b/cmd/ateapi/internal/scheduling/scheduling.go @@ -213,10 +213,15 @@ func utilization(capacity, used *ateapipb.WorkerResources) float64 { } func (s *scheduler) Applies(worker *ateapipb.Worker, constraints Constraints) bool { - if worker.GetSandboxClass() == "" { + // The class is the one the worker registered with WorkerService. Until + // that report lands the Worker has no class, and nothing is placed on it + // (see ServiceImpl.CreateWorker): the assignment records the runtime, + // and atelet requires it on Run and Restore. + workerClass := hardware.SandboxClass(worker.GetStatus()) + if workerClass == "" { return false } - if constraints.SandboxClass != "" && worker.GetSandboxClass() != constraints.SandboxClass { + if constraints.SandboxClass != "" && workerClass != constraints.SandboxClass { return false } @@ -224,14 +229,6 @@ func (s *scheduler) Applies(worker *ateapipb.Worker, constraints Constraints) bo return false } - // The assignment records this runtime and atelet requires it on - // Run/Restore, so a worker that has not registered one cannot host - // anything yet (see ServiceImpl.CreateWorker: "Nothing is placed on it - // in the meantime"). - if worker.GetStatus().GetDefaultRuntime() == nil { - return false - } - if len(constraints.RequiredNodes) > 0 && !slices.Contains(constraints.RequiredNodes, worker.GetNodeName()) { return false } diff --git a/cmd/ateapi/internal/scheduling/scheduling_test.go b/cmd/ateapi/internal/scheduling/scheduling_test.go index 36aa42547a..e04084253c 100644 --- a/cmd/ateapi/internal/scheduling/scheduling_test.go +++ b/cmd/ateapi/internal/scheduling/scheduling_test.go @@ -466,7 +466,7 @@ func TestApplies(t *testing.T) { want: false, }, { - name: "skips worker with no default runtime", + name: "skips worker with no sandbox classes", worker: worker("w", "gvisor", "node-a", nil, withRuntimes(nil)), constraints: Constraints{}, want: false, @@ -507,34 +507,31 @@ type fleet []*ateapipb.Worker func (f fleet) Workers() ([]*ateapipb.Worker, error) { return f, nil } func worker(pod, class, node string, lbls map[string]string, opts ...func(*ateapipb.Worker)) *ateapipb.Worker { - var defRuntime *ateapipb.SandboxRuntime - if class != "" { - defRuntime = sandboxRuntime(class, "amd64") - } w := &ateapipb.Worker{ - WorkerPod: pod, - SandboxClass: class, - NodeName: node, - Labels: lbls, + WorkerPod: pod, + NodeName: node, + Labels: lbls, // A stored Worker always carries a ceiling; CreateWorker reifies one. Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - Capacity: &ateapipb.WorkerResources{Actors: 1}, - DefaultRuntime: defRuntime, + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + Capacity: &ateapipb.WorkerResources{Actors: 1}, }, } + if class != "" { + withRuntimes(sandboxRuntime(class, "amd64"))(w) + } for _, opt := range opts { opt(w) } return w } -func sandboxRuntime(class, arch string, bundleHash ...string) *ateapipb.SandboxRuntime { +func sandboxRuntime(class, arch string, bundleHash ...string) *ateapipb.ActorSandboxRuntime { attrs := []*ateapipb.AttributeEntry{{Key: "architecture", Value: arch}} if len(bundleHash) > 0 && bundleHash[0] != "" { attrs = append(attrs, &ateapipb.AttributeEntry{Key: "sandbox_bundle_hash", Value: bundleHash[0]}) } - return &ateapipb.SandboxRuntime{ + return &ateapipb.ActorSandboxRuntime{ SandboxClass: class, Version: &ateapipb.VersionedSandboxCompat{ SchemaVersion: "v1", @@ -543,10 +540,24 @@ func sandboxRuntime(class, arch string, bundleHash ...string) *ateapipb.SandboxR } } -func withRuntimes(def *ateapipb.SandboxRuntime, restorable ...*ateapipb.SandboxRuntime) func(*ateapipb.Worker) { +// sandboxClass is the WorkerSandboxClass entry of def's class, with def as +// its default runtime and restorable as the others. +func sandboxClass(def *ateapipb.ActorSandboxRuntime, restorable ...*ateapipb.ActorSandboxRuntime) *ateapipb.WorkerSandboxClass { + c := &ateapipb.WorkerSandboxClass{SandboxClass: def.GetSandboxClass(), DefaultRuntime: &ateapipb.WorkerSandboxRuntime{Version: def.GetVersion()}} + for _, r := range restorable { + c.RestorableRuntimes = append(c.RestorableRuntimes, &ateapipb.WorkerSandboxRuntime{Version: r.GetVersion()}) + } + return c +} + +// withRuntimes gives the worker one sandbox class, def's. A nil def leaves +// the worker as before it registered: no class at all. +func withRuntimes(def *ateapipb.ActorSandboxRuntime, restorable ...*ateapipb.ActorSandboxRuntime) func(*ateapipb.Worker) { return func(w *ateapipb.Worker) { - w.Status.DefaultRuntime = def - w.Status.RestorableRuntimes = restorable + w.Status.SandboxClasses = nil + if def != nil { + w.Status.SandboxClasses = []*ateapipb.WorkerSandboxClass{sandboxClass(def, restorable...)} + } } } diff --git a/cmd/ateapi/internal/store/atepg/access_policy.go b/cmd/ateapi/internal/store/atepg/access_policy.go index 93759d49dd..94b63783d4 100644 --- a/cmd/ateapi/internal/store/atepg/access_policy.go +++ b/cmd/ateapi/internal/store/atepg/access_policy.go @@ -23,6 +23,7 @@ import ( "go.ate.dev/substrate/cmd/ateapi/internal/store" "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" ) func (p *Persistence) CreateGlobalAccessPolicy(ctx context.Context, policy *ateapipb.AccessPolicy) (*ateapipb.AccessPolicy, error) { @@ -227,6 +228,7 @@ func (p *Persistence) DeleteAtespaceAccessPolicy(ctx context.Context, name strin if err := tx.Commit(ctx); err != nil { return nil, fmt.Errorf("committing access policy delete for %s: %w", name, err) } + deleted.Metadata.DeleteTime = timestamppb.Now() return deleted, nil } diff --git a/cmd/ateapi/internal/store/atepg/access_policy_test.go b/cmd/ateapi/internal/store/atepg/access_policy_test.go index e2e89775b6..3a118ae358 100644 --- a/cmd/ateapi/internal/store/atepg/access_policy_test.go +++ b/cmd/ateapi/internal/store/atepg/access_policy_test.go @@ -133,9 +133,12 @@ func TestAtespaceAccessPolicy_LifecycleAndCascade(t *testing.T) { UID: updated.GetMetadata().GetUid(), Version: updated.GetMetadata().GetVersion(), }) - if err != nil || !cmp.Equal(deleted, updated, protocmp.Transform()) { + if err != nil || !cmp.Equal(deleted, updated, protocmp.Transform(), protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "delete_time")) { t.Fatalf("DeleteAtespaceAccessPolicy = %v, %v; want %v", deleted, err, updated) } + if deleted.GetMetadata().GetDeleteTime() == nil { + t.Error("the deleted policy carries no delete_time") + } if _, err := p.GetAtespaceAccessPolicy(ctx, "team-a"); !errors.Is(err, store.ErrNotFound) { t.Fatalf("GetAtespaceAccessPolicy after delete = %v, want ErrNotFound", err) } diff --git a/cmd/ateapi/internal/store/atepg/actor.go b/cmd/ateapi/internal/store/atepg/actor.go index f370f911a2..e702bcfb5f 100644 --- a/cmd/ateapi/internal/store/atepg/actor.go +++ b/cmd/ateapi/internal/store/atepg/actor.go @@ -80,6 +80,14 @@ func (p *Persistence) UpdateActor(ctx context.Context, actorRef resources.ActorR if err := precondition.Validate(); err != nil { return nil, err } + return p.updateActor(ctx, actorRef, store.DeletePreconditions(precondition), mutate, false) +} + +func (p *Persistence) MarkActorForDeletion(ctx context.Context, actorRef resources.ActorRef, precondition store.DeletePreconditions, mutate func(*ateapipb.Actor) error) (*ateapipb.Actor, error) { + return p.updateActor(ctx, actorRef, precondition, mutate, true) +} + +func (p *Persistence) updateActor(ctx context.Context, actorRef resources.ActorRef, precondition store.DeletePreconditions, mutate func(*ateapipb.Actor) error, markForDeletion bool) (*ateapipb.Actor, error) { atespace, name := actorRef.Atespace, actorRef.Name var currentUID string var currentVersion int64 @@ -104,12 +112,17 @@ func (p *Persistence) UpdateActor(ctx context.Context, actorRef resources.ActorR return nil, err } oldMeta := proto.CloneOf(dbActor.Metadata) - if err := mutate(dbActor); err != nil { - return nil, err + if mutate != nil { + if err := mutate(dbActor); err != nil { + return nil, err + } } // Stored metadata is authoritative; discard any metadata edits made by the // closure and derive the next revision from the state this attempt read. setUpdateMetadata(dbActor.Metadata, oldMeta) + if markForDeletion && dbActor.Metadata.DeleteTime == nil { + dbActor.Metadata.DeleteTime = proto.CloneOf(dbActor.Metadata.UpdateTime) + } updatedBytes, err := proto.Marshal(dbActor) if err != nil { diff --git a/cmd/ateapi/internal/store/atepg/actor_template.go b/cmd/ateapi/internal/store/atepg/actor_template.go index d49f6edee0..6a4b95faea 100644 --- a/cmd/ateapi/internal/store/atepg/actor_template.go +++ b/cmd/ateapi/internal/store/atepg/actor_template.go @@ -83,6 +83,14 @@ func (p *Persistence) UpdateActorTemplate(ctx context.Context, templateRef resou if err := precondition.Validate(); err != nil { return nil, err } + return p.updateActorTemplate(ctx, templateRef, store.DeletePreconditions(precondition), mutate, false) +} + +func (p *Persistence) MarkActorTemplateForDeletion(ctx context.Context, templateRef resources.ActorTemplateRef, precondition store.DeletePreconditions, mutate func(*ateapipb.ActorTemplate) error) (*ateapipb.ActorTemplate, error) { + return p.updateActorTemplate(ctx, templateRef, precondition, mutate, true) +} + +func (p *Persistence) updateActorTemplate(ctx context.Context, templateRef resources.ActorTemplateRef, precondition store.DeletePreconditions, mutate func(*ateapipb.ActorTemplate) error, markForDeletion bool) (*ateapipb.ActorTemplate, error) { var currentUID string var currentVersion int64 var currentBytes []byte @@ -106,8 +114,10 @@ func (p *Persistence) UpdateActorTemplate(ctx context.Context, templateRef resou return nil, err } templateBeforeMutation := proto.Clone(dbTemplate).(*ateapipb.ActorTemplate) - if err := mutate(dbTemplate); err != nil { - return nil, err + if mutate != nil { + if err := mutate(dbTemplate); err != nil { + return nil, err + } } if err := validateUpdateActorTemplateMutation(templateBeforeMutation, dbTemplate); err != nil { return nil, err @@ -116,6 +126,9 @@ func (p *Persistence) UpdateActorTemplate(ctx context.Context, templateRef resou dbTemplate.Metadata = &ateapipb.ResourceMetadata{} } setUpdateMetadata(dbTemplate.Metadata, templateBeforeMutation.GetMetadata()) + if markForDeletion && dbTemplate.Metadata.DeleteTime == nil { + dbTemplate.Metadata.DeleteTime = proto.CloneOf(dbTemplate.Metadata.UpdateTime) + } updatedBytes, err := proto.Marshal(dbTemplate) if err != nil { return nil, fmt.Errorf("marshaling actor template: %w", err) diff --git a/cmd/ateapi/internal/store/atepg/atepg.go b/cmd/ateapi/internal/store/atepg/atepg.go index 34ada16a68..73bde914ce 100644 --- a/cmd/ateapi/internal/store/atepg/atepg.go +++ b/cmd/ateapi/internal/store/atepg/atepg.go @@ -409,6 +409,7 @@ func setCreateMetadata(metadata *ateapipb.ResourceMetadata) { metadata.Version = 1 metadata.CreateTime = timestamppb.Now() metadata.UpdateTime = metadata.CreateTime + metadata.DeleteTime = nil } // validateProtoMetadataMatchesColumns verifies that the metadata in the database @@ -428,6 +429,7 @@ func setUpdateMetadata(newMeta, oldMeta *ateapipb.ResourceMetadata) { newMeta.Version = oldMeta.Version + 1 newMeta.CreateTime = oldMeta.CreateTime newMeta.UpdateTime = timestamppb.Now() + newMeta.DeleteTime = oldMeta.DeleteTime } func mapDeleteError(err error, uid string, version int64, precondition store.DeletePreconditions) error { diff --git a/cmd/ateapi/internal/store/atepg/atespace.go b/cmd/ateapi/internal/store/atepg/atespace.go index c948b74850..12822bb1b5 100644 --- a/cmd/ateapi/internal/store/atepg/atespace.go +++ b/cmd/ateapi/internal/store/atepg/atespace.go @@ -23,6 +23,7 @@ import ( "go.ate.dev/substrate/cmd/ateapi/internal/store" "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" ) func (p *Persistence) CreateAtespace(ctx context.Context, atespace *ateapipb.Atespace) (*ateapipb.Atespace, error) { @@ -153,5 +154,6 @@ func (p *Persistence) DeleteAtespace(ctx context.Context, name string, precondit if err := unmarshalStored(protoBytes, out); err != nil { return nil, fmt.Errorf("unmarshaling deleted atespace: %w", err) } + out.Metadata.DeleteTime = timestamppb.Now() return out, nil } diff --git a/cmd/ateapi/internal/store/atepg/egress_policy.go b/cmd/ateapi/internal/store/atepg/egress_policy.go index 763c8ff37e..d9ed4ee795 100644 --- a/cmd/ateapi/internal/store/atepg/egress_policy.go +++ b/cmd/ateapi/internal/store/atepg/egress_policy.go @@ -24,6 +24,7 @@ import ( "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" ) func (p *Persistence) CreateEgressPolicy(ctx context.Context, actorRef resources.ActorRef, policy *ateapipb.EgressPolicy) (*ateapipb.EgressPolicy, error) { @@ -112,7 +113,12 @@ func (p *Persistence) DeleteEgressPolicy(ctx context.Context, actorRef resources if err != nil { return nil, fmt.Errorf("deleting egress policy for %s: %w", actorRef, err) } - return unmarshalEgressPolicy(uid, version, protoBytes) + deleted, err := unmarshalEgressPolicy(uid, version, protoBytes) + if err != nil { + return nil, err + } + deleted.Metadata.DeleteTime = timestamppb.Now() + return deleted, nil } func getEgressPolicyRow(ctx context.Context, q querier, query string, args ...any) (*ateapipb.EgressPolicy, error) { diff --git a/cmd/ateapi/internal/store/atepg/tag.go b/cmd/ateapi/internal/store/atepg/tag.go index 794c9fada7..a90190ed21 100644 --- a/cmd/ateapi/internal/store/atepg/tag.go +++ b/cmd/ateapi/internal/store/atepg/tag.go @@ -221,6 +221,14 @@ func (p *Persistence) UpdateTag(ctx context.Context, tagRef resources.TagRef, pr if err := precondition.Validate(); err != nil { return nil, err } + return p.updateTag(ctx, tagRef, store.DeletePreconditions(precondition), mutate, false) +} + +func (p *Persistence) MarkTagForDeletion(ctx context.Context, tagRef resources.TagRef, precondition store.DeletePreconditions, mutate func(*ateapipb.Tag) error) (*ateapipb.Tag, error) { + return p.updateTag(ctx, tagRef, precondition, mutate, true) +} + +func (p *Persistence) updateTag(ctx context.Context, tagRef resources.TagRef, precondition store.DeletePreconditions, mutate func(*ateapipb.Tag) error, markForDeletion bool) (*ateapipb.Tag, error) { atespace, name := tagRef.Atespace, tagRef.Name var currentUID string var currentVersion int64 @@ -246,8 +254,10 @@ func (p *Persistence) UpdateTag(ctx context.Context, tagRef resources.TagRef, pr } tagBeforeMutation := proto.Clone(dbTag).(*ateapipb.Tag) oldMeta := proto.CloneOf(dbTag.Metadata) - if err := mutate(dbTag); err != nil { - return nil, err + if mutate != nil { + if err := mutate(dbTag); err != nil { + return nil, err + } } // TODO: this should be done through DV and removed from here if err := validateUpdateTagMutation(tagBeforeMutation, dbTag); err != nil { @@ -256,6 +266,9 @@ func (p *Persistence) UpdateTag(ctx context.Context, tagRef resources.TagRef, pr // Stored metadata is authoritative; discard any metadata edits made by the // closure and derive the next revision from the state this attempt read. setUpdateMetadata(dbTag.Metadata, oldMeta) + if markForDeletion && dbTag.Metadata.DeleteTime == nil { + dbTag.Metadata.DeleteTime = proto.CloneOf(dbTag.Metadata.UpdateTime) + } updatedBytes, err := proto.Marshal(dbTag) if err != nil { diff --git a/cmd/ateapi/internal/store/atepg/worker.go b/cmd/ateapi/internal/store/atepg/worker.go index 4e86bfed49..03e4027d58 100644 --- a/cmd/ateapi/internal/store/atepg/worker.go +++ b/cmd/ateapi/internal/store/atepg/worker.go @@ -103,6 +103,14 @@ func (p *Persistence) UpdateWorker(ctx context.Context, name string, preconditio if err := precondition.Validate(); err != nil { return nil, err } + return p.updateWorker(ctx, name, store.DeletePreconditions(precondition), mutate, false) +} + +func (p *Persistence) MarkWorkerForDeletion(ctx context.Context, name string, precondition store.DeletePreconditions, mutate func(*ateapipb.Worker) error) (*ateapipb.Worker, error) { + return p.updateWorker(ctx, name, precondition, mutate, true) +} + +func (p *Persistence) updateWorker(ctx context.Context, name string, precondition store.DeletePreconditions, mutate func(*ateapipb.Worker) error, markForDeletion bool) (*ateapipb.Worker, error) { return p.writeAndAppendEvent(ctx, store.WorkerEventUpdated, func(ctx context.Context, tx pgx.Tx) (*ateapipb.Worker, error) { dbWorker, err := getWorkerRowForUpdate(ctx, tx, name) if err != nil { @@ -116,12 +124,17 @@ func (p *Persistence) UpdateWorker(ctx context.Context, name string, preconditio // mutate is free to edit anything it is given; immutable fields are // the service layer's to enforce, via declarative validation. oldMeta := proto.CloneOf(dbWorker.GetMetadata()) - if err := mutate(dbWorker); err != nil { - return nil, err + if mutate != nil { + if err := mutate(dbWorker); err != nil { + return nil, err + } } // Stored metadata is authoritative; discard any metadata edits made by // the closure and derive the next revision from the row we locked. setUpdateMetadata(dbWorker.Metadata, oldMeta) + if markForDeletion && dbWorker.Metadata.DeleteTime == nil { + dbWorker.Metadata.DeleteTime = proto.CloneOf(dbWorker.Metadata.UpdateTime) + } protoBytes, err := proto.Marshal(dbWorker) if err != nil { diff --git a/cmd/ateapi/internal/store/store.go b/cmd/ateapi/internal/store/store.go index 254bf64f44..318cc60107 100644 --- a/cmd/ateapi/internal/store/store.go +++ b/cmd/ateapi/internal/store/store.go @@ -100,6 +100,13 @@ type Interface interface { // validation before the write. UpdateActor(ctx context.Context, actorRef resources.ActorRef, precondition Precondition, mutate func(toUpdate *ateapipb.Actor) error) (*ateapipb.Actor, error) + // MarkActorForDeletion initiates the deletion of an actor. It sets + // metadata.delete_time if unset and applies mutate, if non-nil, so callers can + // update the actor's status. Returns ErrNotFound if missing, + // ErrUIDConflict/ErrVersionConflict if precondition fails or a concurrent + // write wins, or mutate's error. + MarkActorForDeletion(ctx context.Context, actorRef resources.ActorRef, precondition DeletePreconditions, mutate func(toUpdate *ateapipb.Actor) error) (*ateapipb.Actor, error) + // Removes an actor and returns the deleted resource. Returns ErrNotFound if // missing, or ErrUIDConflict/ErrVersionConflict if precondition does not // describe the actor the caller observed. @@ -154,6 +161,13 @@ type Interface interface { // status.snapshot is immutable once set UpdateTag(ctx context.Context, tagRef resources.TagRef, precondition Precondition, mutate func(toUpdate *ateapipb.Tag) error) (*ateapipb.Tag, error) + // MarkTagForDeletion initiates the deletion of a tag. It sets + // metadata.delete_time if unset and applies mutate, if non-nil, so callers can + // update the tag's status. Returns ErrNotFound if missing, + // ErrUIDConflict/ErrVersionConflict if precondition fails or a concurrent + // write wins, or mutate's error. + MarkTagForDeletion(ctx context.Context, tagRef resources.TagRef, precondition DeletePreconditions, mutate func(toUpdate *ateapipb.Tag) error) (*ateapipb.Tag, error) + // Deletes and returns a tag. Returns ErrNotFound if missing, or // ErrUIDConflict/ErrVersionConflict if precondition does not describe the // tag the caller observed. @@ -217,6 +231,13 @@ type Interface interface { // version guards are required. UpdateActorTemplate(ctx context.Context, templateRef resources.ActorTemplateRef, precondition Precondition, mutate func(dbTemplate *ateapipb.ActorTemplate) error) (*ateapipb.ActorTemplate, error) + // MarkActorTemplateForDeletion initiates the deletion of an actor + // template. It sets metadata.delete_time if unset and applies mutate, if + // non-nil, so callers can update the template's status. Returns ErrNotFound + // if missing, ErrUIDConflict/ErrVersionConflict if precondition fails or a + // concurrent write wins, or mutate's error. + MarkActorTemplateForDeletion(ctx context.Context, templateRef resources.ActorTemplateRef, precondition DeletePreconditions, mutate func(toUpdate *ateapipb.ActorTemplate) error) (*ateapipb.ActorTemplate, error) + // Removes an ActorTemplate and returns the deleted resource. Returns // ErrNotFound if missing, or ErrUIDConflict/ErrVersionConflict if // precondition does not describe the template the caller observed. @@ -246,6 +267,13 @@ type Interface interface { // exhausted, or the mutate's error verbatim otherwise. UpdateWorker(ctx context.Context, name string, precondition Precondition, mutate func(toUpdate *ateapipb.Worker) error) (*ateapipb.Worker, error) + // MarkWorkerForDeletion initiates the deletion of a worker. It sets + // metadata.delete_time if unset and applies mutate, if non-nil, so callers can + // update the worker's status. Returns ErrNotFound if missing, + // ErrUIDConflict/ErrVersionConflict if precondition fails or a concurrent + // write wins, or mutate's error. + MarkWorkerForDeletion(ctx context.Context, name string, precondition DeletePreconditions, mutate func(toUpdate *ateapipb.Worker) error) (*ateapipb.Worker, error) + // Removes a worker by name, along with every assignment it holds, and // returns the deleted resource. Returns ErrNotFound if missing, or // ErrUIDConflict/ErrVersionConflict if precondition does not describe the diff --git a/cmd/ateapi/internal/store/storecontract/contract.go b/cmd/ateapi/internal/store/storecontract/contract.go index 8cd37f1d8b..77cf92e241 100644 --- a/cmd/ateapi/internal/store/storecontract/contract.go +++ b/cmd/ateapi/internal/store/storecontract/contract.go @@ -77,7 +77,7 @@ const ( var ( ignoreUID = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "uid") ignoreVersion = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "version") - ignoreTimestamps = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "create_time", "update_time") + ignoreTimestamps = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "create_time", "update_time", "delete_time") ) func newTestAtespace(name string) *ateapipb.Atespace { @@ -201,6 +201,129 @@ func assertPruned(t *testing.T, op string, want, got proto.Message) { } } +// resourceMessage is a stored resource, which carries ResourceMetadata. +type resourceMessage interface { + proto.Message + GetMetadata() *ateapipb.ResourceMetadata +} + +// markForDeletionCase binds one resource's store methods for +// testMarkForDeletion. R is the key the methods take. +type markForDeletionCase[R any, T resourceMessage] struct { + ref R + input T + create func(context.Context, T) (T, error) + get func(context.Context, R) (T, error) + update func(context.Context, R, store.Precondition, func(T) error) (T, error) + mark func(context.Context, R, store.DeletePreconditions, func(T) error) (T, error) + // mutate, if set, is passed to the first mark, and mutated reports whether + // it landed in the same write. + mutate func(T) error + mutated func(T) bool +} + +// testMarkForDeletion checks the MarkForDeletion contract. Only a +// mark sets delete_time, and no update clears it. A mark checks its guards, +// writes nothing when mutate refuses, and keeps the first stamp when repeated. +func testMarkForDeletion[R any, T resourceMessage](t *testing.T, c markForDeletionCase[R, T]) { + t.Helper() + ctx := context.Background() + c.input.GetMetadata().DeleteTime = timestamppb.Now() + created, err := c.create(ctx, c.input) + if err != nil { + t.Fatalf("create failed: %v", err) + } + if created.GetMetadata().GetDeleteTime() != nil { + t.Fatalf("a new resource carries delete_time %v", created.GetMetadata().GetDeleteTime()) + } + + // A plain update cannot set it. + smuggled, err := c.update(ctx, c.ref, store.PreconditionFrom(created), func(toUpdate T) error { + toUpdate.GetMetadata().DeleteTime = timestamppb.Now() + return nil + }) + if err != nil { + t.Fatalf("update failed: %v", err) + } + if smuggled.GetMetadata().GetDeleteTime() != nil { + t.Errorf("a plain update set delete_time to %v, want it ignored", smuggled.GetMetadata().GetDeleteTime()) + } + + if _, err := c.mark(ctx, c.ref, store.DeletePreconditions{UID: "other-uid"}, nil); !errors.Is(err, store.ErrUIDConflict) { + t.Errorf("mark with a foreign uid = %v, want ErrUIDConflict", err) + } + if _, err := c.mark(ctx, c.ref, store.DeletePreconditions{Version: created.GetMetadata().GetVersion()}, nil); !errors.Is(err, store.ErrVersionConflict) { + t.Errorf("mark with a stale version = %v, want ErrVersionConflict", err) + } + + // mutate's refusal writes nothing, so nothing is stamped. + refused := errors.New("refused") + if _, err := c.mark(ctx, c.ref, store.DeletePreconditions{}, func(T) error { return refused }); !errors.Is(err, refused) { + t.Errorf("mark with a refusing mutate = %v, want it verbatim", err) + } + unmarked, err := c.get(ctx, c.ref) + if err != nil { + t.Fatalf("get failed: %v", err) + } + if unmarked.GetMetadata().GetDeleteTime() != nil || unmarked.GetMetadata().GetVersion() != smuggled.GetMetadata().GetVersion() { + t.Errorf("a refused mark wrote: delete_time %v, version %d, want unset and %d", unmarked.GetMetadata().GetDeleteTime(), unmarked.GetMetadata().GetVersion(), smuggled.GetMetadata().GetVersion()) + } + + // Unguarded, so the version having moved since created was read is fine. + marked, err := c.mark(ctx, c.ref, store.DeletePreconditions{}, c.mutate) + if err != nil { + t.Fatalf("mark failed: %v", err) + } + if c.mutated != nil && !c.mutated(marked) { + t.Errorf("mark returned %v, want mutate's change in the same write", marked) + } + if marked.GetMetadata().GetDeleteTime() == nil { + t.Fatal("mark left delete_time unset") + } + if !marked.GetMetadata().GetDeleteTime().AsTime().Equal(marked.GetMetadata().GetUpdateTime().AsTime()) { + t.Errorf("delete_time = %v, want the write's update_time %v", marked.GetMetadata().GetDeleteTime().AsTime(), marked.GetMetadata().GetUpdateTime().AsTime()) + } + if got, want := marked.GetMetadata().GetVersion(), smuggled.GetMetadata().GetVersion()+1; got != want { + t.Errorf("version = %d, want %d", got, want) + } + stored, err := c.get(ctx, c.ref) + if err != nil { + t.Fatalf("get failed: %v", err) + } + if diff := cmp.Diff(marked, stored, protocmp.Transform()); diff != "" { + t.Errorf("stored resource mismatch (-marked +stored):\n%s", diff) + } + + // A plain update cannot clear it. + cleared, err := c.update(ctx, c.ref, store.PreconditionFrom(marked), func(toUpdate T) error { + toUpdate.GetMetadata().DeleteTime = nil + return nil + }) + if err != nil { + t.Fatalf("update failed: %v", err) + } + if !proto.Equal(cleared.GetMetadata().GetDeleteTime(), marked.GetMetadata().GetDeleteTime()) { + t.Errorf("a plain update changed delete_time to %v, want %v kept", cleared.GetMetadata().GetDeleteTime(), marked.GetMetadata().GetDeleteTime()) + } + + // The guards still bite after the mark: a version from before it is stale. + if _, err := c.mark(ctx, c.ref, store.DeletePreconditions{Version: smuggled.GetMetadata().GetVersion()}, nil); !errors.Is(err, store.ErrVersionConflict) { + t.Errorf("mark after the first with a stale version = %v, want ErrVersionConflict", err) + } + + // A second mark writes but keeps the first stamp. + again, err := c.mark(ctx, c.ref, store.DeletePreconditions{UID: cleared.GetMetadata().GetUid()}, nil) + if err != nil { + t.Fatalf("second mark failed: %v", err) + } + if !proto.Equal(again.GetMetadata().GetDeleteTime(), marked.GetMetadata().GetDeleteTime()) { + t.Errorf("a second mark changed delete_time to %v, want %v kept", again.GetMetadata().GetDeleteTime(), marked.GetMetadata().GetDeleteTime()) + } + if got, want := again.GetMetadata().GetVersion(), cleared.GetMetadata().GetVersion()+1; got != want { + t.Errorf("version after the second mark = %d, want %d", got, want) + } +} + // RunContractTests runs the backend-neutral store.Interface assertions // against a fresh store.Interface built by setup for each subtest. setup is // responsible for its own cleanup (e.g. via t.Cleanup). @@ -275,8 +398,14 @@ func runEgressPolicyContractTests(t *testing.T, setup func(t *testing.T) store.I t.Fatalf("UpdateEgressPolicy = %v, %v; want version 2", updated, err) } deleted, err := s.DeleteEgressPolicy(ctx, actorRef, store.DeletePreconditions{}) - if err != nil || !proto.Equal(deleted, updated) { - t.Fatalf("DeleteEgressPolicy = %v, %v; want %v", deleted, err, updated) + if err != nil { + t.Fatalf("DeleteEgressPolicy failed: %v", err) + } + if diff := cmp.Diff(updated, deleted, protocmp.Transform(), protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "delete_time")); diff != "" { + t.Fatalf("deleted policy mismatch (-updated +deleted):\n%s", diff) + } + if deleted.GetMetadata().GetDeleteTime() == nil { + t.Error("the deleted policy carries no delete_time") } if _, err := s.GetEgressPolicy(ctx, actorRef); !errors.Is(err, store.ErrNotFound) { t.Fatalf("GetEgressPolicy after delete error = %v, want ErrNotFound", err) @@ -747,8 +876,28 @@ func runActorContractTests(t *testing.T, setup func(t *testing.T) store.Interfac } }) + t.Run("MarkActorForDeletion", func(t *testing.T) { + s := setup(t) + mustCreateAtespace(t, s, testAtespace) + testMarkForDeletion(t, markForDeletionCase[resources.ActorRef, *ateapipb.Actor]{ + ref: resources.ActorRef{Atespace: testAtespace, Name: "session-1"}, + input: newTestSuspendedActor(testAtespace, "session-1"), + create: s.CreateActor, + get: s.GetActor, + update: s.UpdateActor, + mark: s.MarkActorForDeletion, + mutate: func(toUpdate *ateapipb.Actor) error { + toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_DELETING + return nil + }, + mutated: func(marked *ateapipb.Actor) bool { + return marked.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_DELETING + }, + }) + }) + // The store does not gate the delete on state; the workflow does, and pins - // the row delete to the uid and version it last saw. + // the row delete to the uid it admitted. t.Run("DeleteActor", func(t *testing.T) { for _, state := range []ateapipb.ActorState{ ateapipb.ActorState_ACTOR_STATE_SUSPENDED, @@ -1147,6 +1296,19 @@ func runActorTemplateContractTests(t *testing.T, setup func(t *testing.T) store. } }) + t.Run("MarkActorTemplateForDeletion", func(t *testing.T) { + s := setup(t) + mustCreateAtespace(t, s, testAtespace) + testMarkForDeletion(t, markForDeletionCase[resources.ActorTemplateRef, *ateapipb.ActorTemplate]{ + ref: resources.ActorTemplateRef{Atespace: testAtespace, Name: "tmpl-a"}, + input: newTestActorTemplate(testAtespace, "tmpl-a"), + create: s.CreateActorTemplate, + get: s.GetActorTemplate, + update: s.UpdateActorTemplate, + mark: s.MarkActorTemplateForDeletion, + }) + }) + t.Run("ActorTemplateResources_BlockAtespaceDeletion", func(t *testing.T) { s := setup(t) ctx := context.Background() @@ -1514,6 +1676,23 @@ func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) } }) + t.Run("MarkTagForDeletion", func(t *testing.T) { + s := setup(t) + mustCreateAtespace(t, s, testAtespace) + actor, err := s.CreateActor(context.Background(), newTestSuspendedActor(testAtespace, "actor-1")) + if err != nil { + t.Fatalf("CreateActor failed: %v", err) + } + testMarkForDeletion(t, markForDeletionCase[resources.TagRef, *ateapipb.Tag]{ + ref: resources.TagRef{Atespace: testAtespace, Name: "production"}, + input: newTestInProgressTag("production", actor), + create: s.CreateTag, + get: s.GetTag, + update: s.UpdateTag, + mark: s.MarkTagForDeletion, + }) + }) + t.Run("ListTags_PaginationAndScope", func(t *testing.T) { s := setup(t) ctx := context.Background() @@ -1754,6 +1933,52 @@ func runWorkerContractTests(t *testing.T, setup func(t *testing.T) store.Interfa } }) + t.Run("MarkWorkerForDeletion", func(t *testing.T) { + s := setup(t) + testMarkForDeletion(t, markForDeletionCase[string, *ateapipb.Worker]{ + ref: testWorkerName, + input: newTestWorker(testWorkerName, "pod-1"), + create: s.CreateWorker, + get: s.GetWorker, + update: s.UpdateWorker, + mark: s.MarkWorkerForDeletion, + mutate: func(toUpdate *ateapipb.Worker) error { + toUpdate.Status = &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_DRAINING} + return nil + }, + mutated: func(marked *ateapipb.Worker) bool { + return marked.GetStatus().GetState() == ateapipb.WorkerState_WORKER_STATE_DRAINING + }, + }) + }) + + // A mark publishes the marked worker like any other update. + t.Run("MarkWorkerForDeletion_Event", func(t *testing.T) { + s := setup(t) + ctx := context.Background() + if _, err := s.CreateWorker(ctx, newTestWorker(testWorkerName, "pod-1")); err != nil { + t.Fatalf("CreateWorker failed: %v", err) + } + // Subscribed here so only the mark's event is on the channel. + watch, err := s.WatchWorkers(ctx) + if err != nil { + t.Fatalf("WatchWorkers failed: %v", err) + } + defer watch.Close() + + marked, err := s.MarkWorkerForDeletion(ctx, testWorkerName, store.DeletePreconditions{}, nil) + if err != nil { + t.Fatalf("mark failed: %v", err) + } + event := receiveEvent(t, watch.Events) + if event.Type != store.WorkerEventUpdated { + t.Errorf("expected WorkerEventUpdated, got %v", event.Type) + } + if diff := cmp.Diff(marked, event.Worker, protocmp.Transform()); diff != "" { + t.Errorf("mark event worker mismatch (-marked +event):\n%s", diff) + } + }) + // A worker name is a pod UID, so a name is only reused when the same pod is // re-registered. The store still hands out a fresh uid, and a guard naming // the old one must not reach the new incarnation. @@ -2934,6 +3159,9 @@ func runAtespaceContractTests(t *testing.T, setup func(t *testing.T) store.Inter if got := deleted.GetMetadata().GetName(); got != "team-a" { t.Errorf("deleted atespace name = %q, want team-a", got) } + if deleted.GetMetadata().GetDeleteTime() == nil { + t.Error("the deleted atespace carries no delete_time") + } if _, err := s.GetAtespace(ctx, "team-a"); !errors.Is(err, store.ErrNotFound) { t.Errorf("after delete, GetAtespace = %v, want ErrNotFound", err) } diff --git a/cmd/ateapi/internal/workerservice/certificate.go b/cmd/ateapi/internal/workerservice/certificate.go index 7d75515811..e0433346aa 100644 --- a/cmd/ateapi/internal/workerservice/certificate.go +++ b/cmd/ateapi/internal/workerservice/certificate.go @@ -56,16 +56,17 @@ func (s *Server) MintWorkerActorCertificate(ctx context.Context, req *ateapipb.M return nil, fmt.Errorf("while retrieving actor: %w", err) } // Only the assigned worker may mint, via its node's atelet, checked before the - // UID so other callers get NotFound. Unlike MintWorkerActorJwt, a paused actor - // gets NotFound too: it has no worker to bind the certificate to. + // UID so other callers get NotFound. Unlike MintWorkerActorJwt, a SUSPENDED + // actor gets NotFound too: it has no worker to bind the certificate to. if err := s.checkActorOnNode(ctx, dbActor, caller.NodeName, "certificate"); err != nil { return nil, err } // The status must name the requesting worker. When it names one, // checkActorOnNode has confirmed in the store's worker assignments that - // this worker holds the actor. When it names none (a paused actor, or a - // resume that has bound a worker but not yet recorded it), checkActorOnNode - // read assigned_node alone, and no worker runs the actor yet. + // this worker holds the actor. When it names none (a SUSPENDED actor on + // assigned_node, or a resume that has bound a worker but not yet recorded + // it), checkActorOnNode read assigned_node alone, and no worker runs the + // actor yet. if assigned := dbActor.GetStatus().GetWorkerAssignment().GetWorker().GetName(); assigned == "" || assigned != req.GetWorker().GetName() { slog.WarnContext(ctx, "Refusing a worker-for-actor certificate for an actor not assigned to the calling worker", slog.String("actor", resources.ActorRefFromActor(dbActor).String()), diff --git a/cmd/ateapi/internal/workerservice/certificate_test.go b/cmd/ateapi/internal/workerservice/certificate_test.go index 76f98a871e..a112c964ae 100644 --- a/cmd/ateapi/internal/workerservice/certificate_test.go +++ b/cmd/ateapi/internal/workerservice/certificate_test.go @@ -202,8 +202,8 @@ func TestMintWorkerActorCertificate_Errors(t *testing.T) { onWorker := func(state ateapipb.ActorState) *ateapipb.ActorStatus { return &ateapipb.ActorStatus{State: state, WorkerAssignment: &ateapipb.WorkerAssignment{Worker: &ateapipb.ObjectRef{Name: "worker-1"}}} } - // pausingOnWorker is a pause in progress: SUSPENDING on worker-1 with a - // latest snapshot that is node-local and not yet captured. + // pausingOnWorker is an async suspend in progress: SUSPENDING on worker-1 + // with a latest snapshot that is not yet captured. pausingOnWorker := func() *ateapipb.ActorStatus { status := onWorker(ateapipb.ActorState_ACTOR_STATE_SUSPENDING) status.AssignedNode = "node-1" @@ -254,7 +254,7 @@ func TestMintWorkerActorCertificate_Errors(t *testing.T) { want: codes.OK, }, { - // Pause, suspend and crash release the worker before they clear the + // Suspend and crash release the worker before they clear the // actor's status, which keeps naming the worker until they do. name: "running actor whose worker was released", ctx: authed, @@ -364,8 +364,8 @@ func TestMintWorkerActorCertificate_Errors(t *testing.T) { want: codes.NotFound, }, { - // MintWorkerActorJwt serves a paused actor's checkpoint node. No - // worker runs a paused actor, so no worker gets its certificate. + // MintWorkerActorJwt serves a node holding an actor's local snapshot. No + // worker runs a SUSPENDED actor, so no worker gets its certificate. name: "paused actor on this node", ctx: authed, status: pausedStatus(ateapipb.ActorState_ACTOR_STATE_SUSPENDED, "node-1"), @@ -399,7 +399,7 @@ func TestMintWorkerActorCertificate_Errors(t *testing.T) { want: codes.NotFound, }, { - // An empty worker name must not match a paused actor's empty + // An empty worker name must not match a SUSPENDED actor's empty // assignment. name: "paused actor, empty worker name", ctx: authed, diff --git a/cmd/ateapi/internal/workerservice/jwt.go b/cmd/ateapi/internal/workerservice/jwt.go index 977fbbcb7c..ca678eddeb 100644 --- a/cmd/ateapi/internal/workerservice/jwt.go +++ b/cmd/ateapi/internal/workerservice/jwt.go @@ -49,7 +49,7 @@ func (s *Server) MintWorkerActorJwt(ctx context.Context, req *ateapipb.MintWorke } // The actor doesn't need to be running: tokens are needed to fetch a - // snapshot before resume and to upload a checkpoint after pause. + // snapshot before resume and to upload a checkpoint during or after suspend. dbActor, err := s.store.GetActor(ctx, resources.ActorRefFromObjectRef(req.GetActor())) if errors.Is(err, store.ErrNotFound) { return nil, apierror.NotFound("actor not found") diff --git a/cmd/ateapi/internal/workerservice/jwt_test.go b/cmd/ateapi/internal/workerservice/jwt_test.go index 66f2779bd1..40ea33db01 100644 --- a/cmd/ateapi/internal/workerservice/jwt_test.go +++ b/cmd/ateapi/internal/workerservice/jwt_test.go @@ -113,9 +113,10 @@ func TestMintWorkerActorJwt(t *testing.T) { } } -// pausedStatus is how a paused actor looks: SUSPENDED, no worker, assigned_node -// on the node holding its local snapshot, and a latest generation that is -// local-only. The latest generation is the one last_assigned_generation names. +// pausedStatus is how an asynchronously suspended actor looks: SUSPENDED, no +// worker, assigned_node on the node holding its local snapshot, and a latest +// generation that is local-only. The latest generation is the one +// last_assigned_generation names. func pausedStatus(state ateapipb.ActorState, node string) *ateapipb.ActorStatus { return &ateapipb.ActorStatus{ State: state, @@ -125,9 +126,9 @@ func pausedStatus(state ateapipb.ActorState, node string) *ateapipb.ActorStatus } } -// uploadPendingStatus is how a paused actor looks until atelet's background -// upload of its local snapshot is recorded: SUSPENDED, no worker, and the -// latest generation's durable copy PENDING. +// uploadPendingStatus is how an asynchronously suspended actor looks until +// atelet's background upload of its local snapshot is recorded: SUSPENDED, no +// worker, and the latest generation's durable copy PENDING. func uploadPendingStatus(node string) *ateapipb.ActorStatus { status := pausedStatus(ateapipb.ActorState_ACTOR_STATE_SUSPENDED, node) status.Snapshots[0].DurableSnapshot = &ateapipb.SnapshotStorage{ @@ -154,8 +155,8 @@ func TestMintWorkerActorJwt_Errors(t *testing.T) { onWorker := func(state ateapipb.ActorState) *ateapipb.ActorStatus { return &ateapipb.ActorStatus{State: state, WorkerAssignment: &ateapipb.WorkerAssignment{Worker: &ateapipb.ObjectRef{Name: "worker-1"}}} } - // pausingOnWorker is a pause in progress: SUSPENDING on worker-1 with a - // latest snapshot that is node-local and not yet captured. + // pausingOnWorker is an async suspend in progress: SUSPENDING on worker-1 + // with a latest snapshot that is not yet captured. pausingOnWorker := func() *ateapipb.ActorStatus { status := onWorker(ateapipb.ActorState_ACTOR_STATE_SUSPENDING) status.AssignedNode = "node-1" @@ -206,7 +207,7 @@ func TestMintWorkerActorJwt_Errors(t *testing.T) { want: codes.OK, }, { - // Pause, suspend and crash release the worker before they clear the + // Suspend and crash release the worker before they clear the // actor's status, which keeps naming the worker until they do. name: "running actor whose worker was released", ctx: authed, @@ -340,8 +341,8 @@ func TestMintWorkerActorJwt_Errors(t *testing.T) { want: codes.NotFound, }, { - // atelet uploads the paused actor's checkpoint in the background - // from the node that holds it, with no worker assigned. + // atelet uploads the asynchronously suspended actor's checkpoint in + // the background from the node that holds it, with no worker assigned. name: "paused actor with its upload pending on this node", ctx: authed, status: uploadPendingStatus("node-1"), diff --git a/cmd/ateapi/internal/workerservice/register.go b/cmd/ateapi/internal/workerservice/register.go index 625e8c7e6b..a5318bb5fc 100644 --- a/cmd/ateapi/internal/workerservice/register.go +++ b/cmd/ateapi/internal/workerservice/register.go @@ -26,12 +26,13 @@ import ( "go.ate.dev/substrate/cmd/ateapi/internal/ateletauth" "go.ate.dev/substrate/cmd/ateapi/internal/store" "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/hardware" "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" ) -// RegisterWorker records a Worker's reported capacity and sandbox runtimes in +// RegisterWorker records a Worker's reported capacity and sandbox classes in // one write. As with MintCert, the caller must be an atelet running on the // Worker's node. func (s *Server) RegisterWorker(ctx context.Context, req *ateapipb.RegisterWorkerRequest) (*ateapipb.RegisterWorkerResponse, error) { @@ -44,12 +45,13 @@ func (s *Server) RegisterWorker(ctx context.Context, req *ateapipb.RegisterWorke return nil, resources.ToAPIError(errs) } reported := req.GetCapacity() - defaultRuntime := req.GetDefaultRuntime() - restorable := req.GetRestorableRuntimes() + classes := req.GetSandboxClasses() name := req.GetWorker().GetName() - sortAttributes(defaultRuntime) - for _, rt := range restorable { - sortAttributes(rt) + for _, c := range classes { + sortAttributes(c.GetDefaultRuntime().GetVersion()) + for _, rt := range c.GetRestorableRuntimes() { + sortAttributes(rt.GetVersion()) + } } // Use authoritative state to authorize the write. @@ -69,14 +71,12 @@ func (s *Server) RegisterWorker(ctx context.Context, req *ateapipb.RegisterWorke slog.String("caller_pod", caller.PodName)) return nil, apierror.NotFound("Worker %s not found", name) } - if err := validateRuntimeClasses(worker.GetSandboxClass(), req); err != nil { + status := worker.GetStatus() + if err := validateSandboxClasses(hardware.SandboxClass(status), classes); err != nil { return nil, err } - status := worker.GetStatus() - if proto.Equal(status.GetCapacity(), reported) && - proto.Equal(status.GetDefaultRuntime(), defaultRuntime) && - sameRuntimes(status.GetRestorableRuntimes(), restorable) { + if proto.Equal(status.GetCapacity(), reported) && sameClasses(status.GetSandboxClasses(), classes) { return &ateapipb.RegisterWorkerResponse{Worker: worker}, nil } @@ -85,8 +85,7 @@ func (s *Server) RegisterWorker(ctx context.Context, req *ateapipb.RegisterWorke // dimension or runtime this report leaves out is one it no longer // supplies. toUpdate.Status.Capacity = reported - toUpdate.Status.DefaultRuntime = defaultRuntime - toUpdate.Status.RestorableRuntimes = restorable + toUpdate.Status.SandboxClasses = classes return nil }) switch { @@ -98,48 +97,44 @@ func (s *Server) RegisterWorker(ctx context.Context, req *ateapipb.RegisterWorke default: return nil, fmt.Errorf("while recording capacity for worker %s: %w", name, err) } - slog.InfoContext(ctx, "Worker registered its capacity and sandbox runtimes", + slog.InfoContext(ctx, "Worker registered its capacity and sandbox classes", slog.String("worker", name), slog.String("was", worker.GetStatus().GetCapacity().String()), slog.String("now", updated.GetStatus().GetCapacity().String()), - slog.String("default_runtime", updated.GetStatus().GetDefaultRuntime().String()), - slog.Int("restorable_runtimes", len(updated.GetStatus().GetRestorableRuntimes()))) + slog.String("sandbox_classes", fmt.Sprint(updated.GetStatus().GetSandboxClasses()))) return &ateapipb.RegisterWorkerResponse{Worker: updated}, nil } -// validateRuntimeClasses rejects a runtime of a class other than the Worker's -// own. The class is fixed when the pool creates the Worker and names the -// sandbox its worker runs, so a runtime of another class contradicts the -// recorded Worker state. A Worker recorded without a class has nothing for a -// report to contradict. -func validateRuntimeClasses(class string, req *ateapipb.RegisterWorkerRequest) error { - if class == "" { +// validateSandboxClasses rejects an entry of a class other than the one +// already recorded on the Worker. A worker pod runs one sandbox class for its +// lifetime, fixed by its first registration, so a later report of another +// class contradicts the recorded state. A Worker that has not registered yet +// has nothing for a report to contradict. +func validateSandboxClasses(recorded string, classes []*ateapipb.WorkerSandboxClass) error { + if recorded == "" { return nil } - if got := req.GetDefaultRuntime().GetSandboxClass(); got != class { - return apierror.FailedPrecondition("default_runtime.sandbox_class %q does not match Worker's sandbox_class %q", got, class) - } - for i, rt := range req.GetRestorableRuntimes() { - if got := rt.GetSandboxClass(); got != class { - return apierror.FailedPrecondition("restorable_runtimes[%d].sandbox_class %q does not match Worker's sandbox_class %q", i, got, class) + for i, c := range classes { + if got := c.GetSandboxClass(); got != recorded { + return apierror.FailedPrecondition("sandbox_classes[%d].sandbox_class %q does not match Worker's recorded sandbox_class %q", i, got, recorded) } } return nil } -// sameRuntimes reports whether two runtime lists are identical entry for -// entry, in order. A report is compared as sent, so a Worker that reorders its -// runtimes between reports is recorded again. -func sameRuntimes(a, b []*ateapipb.SandboxRuntime) bool { - return slices.EqualFunc(a, b, func(x, y *ateapipb.SandboxRuntime) bool { return proto.Equal(x, y) }) +// sameClasses reports whether two class lists are identical entry for entry, +// in order, restorable runtimes included. A report is compared as sent, so a +// Worker that reorders its runtimes between reports is recorded again. +func sameClasses(a, b []*ateapipb.WorkerSandboxClass) bool { + return slices.EqualFunc(a, b, func(x, y *ateapipb.WorkerSandboxClass) bool { return proto.Equal(x, y) }) } // sortAttributes orders a runtime's compatibility attributes by key, in place. // Attribute order carries no meaning, but proto equality is order-sensitive, so // a report is canonicalized before it is compared with what is stored: the same // attributes in another order are the same runtime and write nothing. -func sortAttributes(rt *ateapipb.SandboxRuntime) { - slices.SortFunc(rt.GetVersion().GetAttributes(), func(a, b *ateapipb.AttributeEntry) int { +func sortAttributes(compat *ateapipb.VersionedSandboxCompat) { + slices.SortFunc(compat.GetAttributes(), func(a, b *ateapipb.AttributeEntry) int { return cmp.Compare(a.GetKey(), b.GetKey()) }) } diff --git a/cmd/ateapi/internal/workerservice/register_test.go b/cmd/ateapi/internal/workerservice/register_test.go index c1cdaf9cc4..153cda1e7d 100644 --- a/cmd/ateapi/internal/workerservice/register_test.go +++ b/cmd/ateapi/internal/workerservice/register_test.go @@ -20,7 +20,6 @@ import ( "testing" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "google.golang.org/protobuf/testing/protocmp" "go.ate.dev/substrate/cmd/ateapi/internal/ateletauth/ateletauthtest" @@ -30,28 +29,34 @@ import ( "google.golang.org/grpc/codes" ) -// testRuntime is a SandboxRuntime of the given class whose compatibility -// identity is a single architecture attribute, which is enough to tell two -// runtimes apart. -func testRuntime(class, arch string) *ateapipb.SandboxRuntime { - return &ateapipb.SandboxRuntime{ - SandboxClass: class, - Version: &ateapipb.VersionedSandboxCompat{ - SchemaVersion: "v1", - Attributes: []*ateapipb.AttributeEntry{{Key: "architecture", Value: arch}}, - }, +// testCompat is a compatibility version whose identity is a single +// architecture attribute, which is enough to tell two runtimes apart. +func testCompat(arch string) *ateapipb.VersionedSandboxCompat { + return &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{{Key: "architecture", Value: arch}}, } } -// testDefaultRuntime is what seedReportedWorker records as already reported, -// of the class it gives the Worker. -var testDefaultRuntime = testRuntime("gvisor", "amd64") +// testClass is one reported sandbox class: def as its default runtime and +// restorable as the others. +func testClass(name string, def *ateapipb.VersionedSandboxCompat, restorable ...*ateapipb.VersionedSandboxCompat) *ateapipb.WorkerSandboxClass { + c := &ateapipb.WorkerSandboxClass{SandboxClass: name, DefaultRuntime: &ateapipb.WorkerSandboxRuntime{Version: def}} + for _, r := range restorable { + c.RestorableRuntimes = append(c.RestorableRuntimes, &ateapipb.WorkerSandboxRuntime{Version: r}) + } + return c +} + +// testSandboxClasses is what seedReportedWorker records as already reported: +// the class it gives the Worker. +var testSandboxClasses = []*ateapipb.WorkerSandboxClass{testClass("gvisor", testCompat("amd64"))} func setRequest(actors int32) *ateapipb.RegisterWorkerRequest { return &ateapipb.RegisterWorkerRequest{ Worker: &ateapipb.ObjectRef{Name: testWorkerName}, Capacity: &ateapipb.WorkerResources{Actors: actors}, - DefaultRuntime: testDefaultRuntime, + SandboxClasses: testSandboxClasses, } } @@ -78,111 +83,98 @@ func TestRegisterWorker(t *testing.T) { } } -// A report's runtimes are checked against the Worker's class and recorded only -// when they differ from what is stored. Capacity is held at what was seeded, -// so a write here is one the runtimes alone caused. -func TestRegisterWorker_Runtimes(t *testing.T) { +// A report's classes are checked against the class the Worker recorded at its +// first registration and recorded only when they differ from what is stored. +// Capacity is held at what was seeded, so a write here is one the classes +// alone caused. +func TestRegisterWorker_SandboxClasses(t *testing.T) { st, cleanup := storetest.SetupTestStore(t) defer cleanup() s := New(st, &fakeControl{}, testAteletSPIFFEID, nil, "", nil) authed := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)) - gvisorAMD64, gvisorARM64 := testRuntime("gvisor", "amd64"), testRuntime("gvisor", "arm64") - gvisorV8, gvisorV9 := testRuntime("gvisor", "arm64-v8"), testRuntime("gvisor", "arm64-v9") - microvm := testRuntime("microvm", "amd64") - // Every Worker here has already reported gvisorAMD64 with gvisorV8 and - // gvisorV9 restorable, in that order, at a capacity of one actor. + amd64, arm64 := testCompat("amd64"), testCompat("arm64") + v8, v9 := testCompat("arm64-v8"), testCompat("arm64-v9") + // Every Worker here has already reported gvisor with amd64 as its default + // and v8 and v9 restorable, in that order, at a capacity of one actor. seeded := func() *ateapipb.WorkerStatus { return &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - Capacity: &ateapipb.WorkerResources{Actors: 1}, - DefaultRuntime: gvisorAMD64, - RestorableRuntimes: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + SandboxClasses: []*ateapipb.WorkerSandboxClass{testClass("gvisor", amd64, v8, v9)}, } } tests := []struct { name string - // workerClass is the class the pool created the Worker with; "" is - // a Worker recorded without one. - workerClass string - def *ateapipb.SandboxRuntime - restorable []*ateapipb.SandboxRuntime - wantCode codes.Code + // unreported seeds a Worker that has not registered yet, with no + // sandbox classes, instead of seeded(). + unreported bool + classes []*ateapipb.WorkerSandboxClass + wantCode codes.Code // wantWrite is whether the report changes the record. wantWrite bool }{{ - name: "identical report writes nothing", - workerClass: "gvisor", - def: gvisorAMD64, - restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, + name: "identical report writes nothing", + classes: []*ateapipb.WorkerSandboxClass{testClass("gvisor", amd64, v8, v9)}, }, { - name: "changed default runtime writes", - workerClass: "gvisor", - def: gvisorARM64, - restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, - wantWrite: true, + name: "changed default runtime writes", + classes: []*ateapipb.WorkerSandboxClass{testClass("gvisor", arm64, v8, v9)}, + wantWrite: true, }, { - name: "changed restorable runtime writes", - workerClass: "gvisor", - def: gvisorAMD64, - restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorARM64}, - wantWrite: true, + name: "changed restorable runtime writes", + classes: []*ateapipb.WorkerSandboxClass{testClass("gvisor", amd64, v8, arm64)}, + wantWrite: true, }, { // A report is compared as sent: the list is atomic, so a new order // is a new report. - name: "reordered restorable runtimes write", - workerClass: "gvisor", - def: gvisorAMD64, - restorable: []*ateapipb.SandboxRuntime{gvisorV9, gvisorV8}, - wantWrite: true, + name: "reordered restorable runtimes write", + classes: []*ateapipb.WorkerSandboxClass{testClass("gvisor", amd64, v9, v8)}, + wantWrite: true, }, { // Runtimes are replaced like capacity: a report that lists no // restorable runtimes is a Worker that can restore from none but its // default, as when a version is disabled, and the record must not // keep advertising them. - name: "dropped restorable runtimes write", - workerClass: "gvisor", - def: gvisorAMD64, - wantWrite: true, + name: "dropped restorable runtimes write", + classes: []*ateapipb.WorkerSandboxClass{testClass("gvisor", amd64)}, + wantWrite: true, }, { - // The class is fixed by the pool that created the Worker. A runtime - // of another class is one it cannot run, so the report is refused - // rather than recorded for the scheduler to act on. - name: "default runtime of another class is refused", - workerClass: "gvisor", - def: microvm, - restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, - wantCode: codes.FailedPrecondition, + // Once a Worker has registered a sandbox class, an entry of another + // class contradicts its recorded state and is refused. + name: "another class is refused", + classes: []*ateapipb.WorkerSandboxClass{testClass("microvm", amd64, v8, v9)}, + wantCode: codes.FailedPrecondition, }, { - name: "restorable runtime of another class is refused", - workerClass: "gvisor", - def: gvisorAMD64, - restorable: []*ateapipb.SandboxRuntime{gvisorV8, microvm}, - wantCode: codes.FailedPrecondition, + // A pool declares one class, so does a Worker: the list's size is an + // API rule, checked before the recorded class is. + name: "a second class is refused", + classes: []*ateapipb.WorkerSandboxClass{testClass("gvisor", amd64, v8, v9), testClass("microvm", amd64)}, + wantCode: codes.InvalidArgument, }, { - // A Worker recorded without a class has nothing for a report to - // contradict. - name: "unclassed Worker takes a default runtime of any class", - def: microvm, - restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, + // A Worker that has not registered yet accepts any class on its first + // report. + name: "unreported Worker takes a class of any name", + unreported: true, + classes: []*ateapipb.WorkerSandboxClass{testClass("microvm", amd64)}, wantWrite: true, }, { - name: "unclassed Worker takes restorable runtimes of mixed classes", - def: gvisorAMD64, - restorable: []*ateapipb.SandboxRuntime{gvisorV8, microvm}, - wantWrite: true, + name: "a report of no classes is refused", + wantCode: codes.InvalidArgument, }} for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { name := fmt.Sprintf("1a7e4c83-6d20-4f95-b3c8-%012d", i) - before := seedWorker(t, st, name, testNode, tt.workerClass, seeded()) + initialStatus := seeded() + if tt.unreported { + initialStatus = &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE} + } + before := seedWorker(t, st, name, testNode, initialStatus) got, err := s.RegisterWorker(authed, &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: name}, - Capacity: &ateapipb.WorkerResources{Actors: 1}, - DefaultRuntime: tt.def, - RestorableRuntimes: tt.restorable, + Worker: &ateapipb.ObjectRef{Name: name}, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + SandboxClasses: tt.classes, }) if code := apierror.Code(err); code != tt.wantCode { t.Fatalf("code = %v (err %v), want %v", code, err, tt.wantCode) @@ -200,11 +192,8 @@ func TestRegisterWorker_Runtimes(t *testing.T) { } return } - if diff := cmp.Diff(tt.def, after.GetStatus().GetDefaultRuntime(), protocmp.Transform()); diff != "" { - t.Errorf("default_runtime mismatch (-want +got):\n%s", diff) - } - if diff := cmp.Diff(tt.restorable, after.GetStatus().GetRestorableRuntimes(), protocmp.Transform(), cmpopts.EquateEmpty()); diff != "" { - t.Errorf("restorable_runtimes mismatch (-want +got):\n%s", diff) + if diff := cmp.Diff(tt.classes, after.GetStatus().GetSandboxClasses(), protocmp.Transform()); diff != "" { + t.Errorf("sandbox_classes mismatch (-want +got):\n%s", diff) } if diff := cmp.Diff(after, got.GetWorker(), protocmp.Transform()); diff != "" { t.Errorf("response is not the stored Worker (-want +got):\n%s", diff) @@ -270,21 +259,19 @@ func TestRegisterWorker_SortsAttributes(t *testing.T) { seedReportedWorker(t, st, testNode, &ateapipb.WorkerResources{Actors: 1}) authed := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)) - runtimeWith := func(keys ...string) *ateapipb.SandboxRuntime { - rt := testRuntime("gvisor", "amd64") - rt.Version.Attributes = nil + compatWith := func(keys ...string) *ateapipb.VersionedSandboxCompat { + c := &ateapipb.VersionedSandboxCompat{SchemaVersion: "v1"} for _, k := range keys { - rt.Version.Attributes = append(rt.Version.Attributes, &ateapipb.AttributeEntry{Key: k, Value: "v"}) + c.Attributes = append(c.Attributes, &ateapipb.AttributeEntry{Key: k, Value: "v"}) } - return rt + return c } - register := func(def, restorable *ateapipb.SandboxRuntime) *ateapipb.Worker { + register := func(def, restorable *ateapipb.VersionedSandboxCompat) *ateapipb.Worker { t.Helper() got, err := s.RegisterWorker(authed, &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - Capacity: &ateapipb.WorkerResources{Actors: 1}, - DefaultRuntime: def, - RestorableRuntimes: []*ateapipb.SandboxRuntime{restorable}, + Worker: &ateapipb.ObjectRef{Name: testWorkerName}, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + SandboxClasses: []*ateapipb.WorkerSandboxClass{testClass("gvisor", def, restorable)}, }) if err != nil { t.Fatalf("RegisterWorker() failed: %v", err) @@ -292,15 +279,13 @@ func TestRegisterWorker_SortsAttributes(t *testing.T) { return got.GetWorker() } - first := register(runtimeWith("cpu_features", "architecture"), runtimeWith("sandbox_bundle_hash", "architecture")) - if diff := cmp.Diff(runtimeWith("architecture", "cpu_features"), first.GetStatus().GetDefaultRuntime(), protocmp.Transform()); diff != "" { - t.Errorf("default_runtime not stored sorted by key (-want +got):\n%s", diff) - } - if diff := cmp.Diff([]*ateapipb.SandboxRuntime{runtimeWith("architecture", "sandbox_bundle_hash")}, first.GetStatus().GetRestorableRuntimes(), protocmp.Transform()); diff != "" { - t.Errorf("restorable_runtimes not stored sorted by key (-want +got):\n%s", diff) + first := register(compatWith("cpu_features", "architecture"), compatWith("sandbox_bundle_hash", "architecture")) + want := []*ateapipb.WorkerSandboxClass{testClass("gvisor", compatWith("architecture", "cpu_features"), compatWith("architecture", "sandbox_bundle_hash"))} + if diff := cmp.Diff(want, first.GetStatus().GetSandboxClasses(), protocmp.Transform()); diff != "" { + t.Errorf("sandbox_classes not stored sorted by key (-want +got):\n%s", diff) } - again := register(runtimeWith("architecture", "cpu_features"), runtimeWith("architecture", "sandbox_bundle_hash")) + again := register(compatWith("architecture", "cpu_features"), compatWith("architecture", "sandbox_bundle_hash")) if got, want := again.GetMetadata().GetVersion(), first.GetMetadata().GetVersion(); got != want { t.Errorf("version = %d after a reordered repeat, want %d unchanged", got, want) } @@ -322,20 +307,30 @@ func TestRegisterWorker_Errors(t *testing.T) { {"unauthenticated", ateletauthtest.ContextWith(nil), setRequest(2), codes.Unauthenticated}, {"no worker ref", authed, &ateapipb.RegisterWorkerRequest{ Capacity: &ateapipb.WorkerResources{Actors: 2}, - DefaultRuntime: testDefaultRuntime, + SandboxClasses: testSandboxClasses, }, codes.InvalidArgument}, {"no capacity", authed, &ateapipb.RegisterWorkerRequest{ Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - DefaultRuntime: testDefaultRuntime, + SandboxClasses: testSandboxClasses, }, codes.InvalidArgument}, - {"no default runtime", authed, &ateapipb.RegisterWorkerRequest{ + {"no sandbox classes", authed, &ateapipb.RegisterWorkerRequest{ Worker: &ateapipb.ObjectRef{Name: testWorkerName}, Capacity: &ateapipb.WorkerResources{Actors: 2}, }, codes.InvalidArgument}, + {"class without a default runtime", authed, &ateapipb.RegisterWorkerRequest{ + Worker: &ateapipb.ObjectRef{Name: testWorkerName}, + Capacity: &ateapipb.WorkerResources{Actors: 2}, + SandboxClasses: []*ateapipb.WorkerSandboxClass{{SandboxClass: "gvisor"}}, + }, codes.InvalidArgument}, + {"class that is not a sandbox class", authed, &ateapipb.RegisterWorkerRequest{ + Worker: &ateapipb.ObjectRef{Name: testWorkerName}, + Capacity: &ateapipb.WorkerResources{Actors: 2}, + SandboxClasses: []*ateapipb.WorkerSandboxClass{testClass("plain", testCompat("amd64"))}, + }, codes.InvalidArgument}, {"absent worker", authed, &ateapipb.RegisterWorkerRequest{ Worker: &ateapipb.ObjectRef{Name: "3b9f1e77-2c4d-4a80-91be-6d5c8f0a7e21"}, Capacity: &ateapipb.WorkerResources{Actors: 2}, - DefaultRuntime: testDefaultRuntime, + SandboxClasses: testSandboxClasses, }, codes.NotFound}, } for _, tc := range tests { @@ -370,7 +365,7 @@ func TestRegisterWorker_RejectsNonsense(t *testing.T) { _, err := s.RegisterWorker(authed, &ateapipb.RegisterWorkerRequest{ Worker: &ateapipb.ObjectRef{Name: testWorkerName}, Capacity: tc.capacity, - DefaultRuntime: testDefaultRuntime, + SandboxClasses: testSandboxClasses, }) if got := apierror.Code(err); got != codes.InvalidArgument { t.Fatalf("code = %v (err %v), want %v", got, err, codes.InvalidArgument) diff --git a/cmd/ateapi/internal/workerservice/service_test.go b/cmd/ateapi/internal/workerservice/service_test.go index 56d72a0a3c..5ad949601d 100644 --- a/cmd/ateapi/internal/workerservice/service_test.go +++ b/cmd/ateapi/internal/workerservice/service_test.go @@ -85,15 +85,14 @@ func (f *fakeControl) CrashActorReportedByWorker(_ context.Context, actorRef res // this seeds. func seedReportedWorker(t *testing.T, st store.Interface, nodeName string, capacity *ateapipb.WorkerResources) *ateapipb.Worker { t.Helper() - return seedWorker(t, st, testWorkerName, nodeName, testDefaultRuntime.GetSandboxClass(), &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: capacity, DefaultRuntime: testDefaultRuntime, + return seedWorker(t, st, testWorkerName, nodeName, &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: capacity, SandboxClasses: testSandboxClasses, }) } -// seedWorker registers a Worker named name, of sandbox class class, on -// nodeName, with status already recorded. name doubles as the pod UID, so it -// must be UUID-shaped. class "" is a Worker recorded without one. -func seedWorker(t *testing.T, st store.Interface, name, nodeName, class string, status *ateapipb.WorkerStatus) *ateapipb.Worker { +// seedWorker registers a Worker named name on nodeName, with status already +// recorded. name doubles as the pod UID, so it must be UUID-shaped. +func seedWorker(t *testing.T, st store.Interface, name, nodeName string, status *ateapipb.WorkerStatus) *ateapipb.Worker { t.Helper() created, err := st.CreateWorker(context.Background(), &ateapipb.Worker{ Metadata: &ateapipb.ResourceMetadata{Name: name}, @@ -103,7 +102,6 @@ func seedWorker(t *testing.T, st store.Interface, name, nodeName, class string, WorkerPodUid: name, NodeName: nodeName, Ips: []string{"10.1.2.3"}, - SandboxClass: class, Status: status, }) if err != nil { diff --git a/cmd/ateapi/internal/workerservice/suspend.go b/cmd/ateapi/internal/workerservice/suspend.go index 6035fa29e2..4c58c44319 100644 --- a/cmd/ateapi/internal/workerservice/suspend.go +++ b/cmd/ateapi/internal/workerservice/suspend.go @@ -47,7 +47,7 @@ func (s *Server) RequestActorSuspend(ctx context.Context, req *ateapipb.RequestA // From here it is an ordinary suspend. Running the same workflow Control // runs is the point: its preconditions already refuse an Actor that is not - // suspendable, and a proposal that races a resume, pause, or delete loses + // suspendable, and a proposal that races a resume, suspend, or delete loses // to whichever holds the Actor's lease. // // TODO: the checks above run outside the Actor's lease, so there is a small @@ -119,7 +119,8 @@ func checkActorHostedBy(ctx context.Context, actor *ateapipb.Actor, workerName, return apierror.NotFound("Actor %s not found", actorRef) } // An Actor with no assignment has no worker to speak for it: it is - // SUSPENDED (paused or not) or CRASHED, and nothing hosts it. + // SUSPENDED (with or without a local snapshot) or CRASHED, and nothing + // hosts it. if assigned := actor.GetStatus().GetWorkerAssignment().GetWorker().GetName(); assigned != workerName { slog.WarnContext(ctx, "Refusing a request for an actor hosted elsewhere", slog.String("worker", workerName), diff --git a/cmd/ateapi/internal/workerservice/suspend_test.go b/cmd/ateapi/internal/workerservice/suspend_test.go index 1599627c55..1a7425a642 100644 --- a/cmd/ateapi/internal/workerservice/suspend_test.go +++ b/cmd/ateapi/internal/workerservice/suspend_test.go @@ -209,7 +209,7 @@ func TestRequestActorSuspend_Errors(t *testing.T) { } // The suspend is a proposal, and the control plane's refusal is the answer: a -// request that loses a race to a resume, pause, or delete comes back as what +// request that loses a race to a resume, suspend, or delete comes back as what // the workflow said, not as something the worker has to interpret. func TestRequestActorSuspend_PassesThroughTheWorkflowsRefusal(t *testing.T) { st, cleanup := storetest.SetupTestStore(t) diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 4638889a18..5658a10ab1 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -21,9 +21,11 @@ import ( "fmt" "log/slog" "net" + "net/url" "os" "os/signal" "strconv" + "strings" "syscall" "time" @@ -36,7 +38,6 @@ import ( "go.ate.dev/substrate/cmd/ateapi/internal/store/atepg" "go.ate.dev/substrate/cmd/ateapi/internal/workercache" "go.ate.dev/substrate/cmd/ateapi/internal/workerservice" - "go.ate.dev/substrate/internal/actoridjwt" "go.ate.dev/substrate/internal/ateinterceptors" "go.ate.dev/substrate/internal/credbundle" "go.ate.dev/substrate/internal/installdefaults" @@ -59,6 +60,7 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/reflection" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" @@ -85,11 +87,10 @@ var ( postgresOwnerRole = pflag.String("postgres-owner-role", "", "Required PostgreSQL role assumed by owner connections.") postgresSchema = pflag.String("postgres-schema", "substrate", "PostgreSQL schema for Substrate tables. This overrides a search_path connection parameter.") postgresPoolMaxConns = pflag.Int32("postgres-pool-max-conns", 0, "Maximum connections in the shared Substrate and OpenFGA read/write PostgreSQL pool. Does not affect the owner, watch, or readiness probe pools. The DSN or pgx default is used when unset.") - experimentalEnableAuthz = pflag.Bool("experimental-enable-authz", false, "Enforce OpenFGA authorization checks on all registered RPCs (experimental). AccessPolicy RPCs are always checked.") // TODO: Move the authz settings into the hot-reloadable config proto // (agent-substrate/substrate#2021) once it lands, so bootstrap owner // changes take effect without a restart. - authzBootstrapOwners = pflag.StringSlice("authz-bootstrap-owners", nil, "Principal IDs that are always global owners while listed, independent of the stored global AccessPolicy. Removing an ID revokes its access on restart. At least one is required when --experimental-enable-authz is set.") + authzBootstrapOwners = pflag.StringSlice("authz-bootstrap-owners", nil, "Principal IDs that are always global owners while listed, independent of the stored global AccessPolicy. Removing an ID revokes its access on restart. Required: at least one.") // These SPIFFE IDs hold the matching system component role when they call // over mTLS. controllerSPIFFEID = pflag.String("controller-spiffe-id", installdefaults.ControllerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of ate-controller. Holds the controller authz role, which manages workers.") @@ -98,7 +99,7 @@ var ( apiServerSidecarSPIFFEID = pflag.String("api-server-sidecar-spiffe-id", installdefaults.APIServerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of the ate-api-server pod, which its sidecars share. Holds the api_server_sidecar role, which reads actors so the objectstore control plugin can scope their storage credentials.") actorIDJWTPoolFile = pflag.String("actor-id-jwt-pool", "", "The file that contains the serialized JWT authority pool for signing actor JWTs") - actorJWTIssuer = pflag.String("actor-jwt-issuer", "", "Issuer URL placed in the iss claim of actor JWTs. Relying parties fetch /.well-known/openid-configuration to verify them. Must be https://, with no path. Required.") + actorJWTIssuer = pflag.String("actor-jwt-issuer", "", "Issuer URL placed in the iss claim of actor JWTs. Relying parties fetch /.well-known/openid-configuration to verify them. Must be https with no query or fragment. Required.") defaultEgressGatewayAddress = pflag.String("default-egress-gateway-address", "", "Default address (host:port) of the egress PEP that each actor's atunnel dials. Sent on every atelet Run and Restore, so it takes effect at the actor's next activation. Empty leaves actors with no TCP egress.") actorIDCAPoolFile = pflag.String("actor-id-ca-pool", "", "The file that contains the CA pool for signing actor JWTs") @@ -234,13 +235,10 @@ func main() { // (atepg's outbox maintenance loop); stop it on shutdown before closing pool. defer persistence.Close() - // The authz stack is always wired so policy tuples stay in sync with the - // store; --experimental-enable-authz only controls enforcement in the - // interceptor. - if *experimentalEnableAuthz && len(*authzBootstrapOwners) == 0 { + if len(*authzBootstrapOwners) == 0 { // Without a bootstrap owner, nobody could create the global - // AccessPolicy, so the enforced API would be unusable. - serverboot.Fatal(ctx, "Invalid flags", fmt.Errorf("--authz-bootstrap-owners must list at least one principal when --experimental-enable-authz is set")) + // AccessPolicy, so the API would be unusable. + serverboot.Fatal(ctx, "Invalid flags", fmt.Errorf("--authz-bootstrap-owners must list at least one principal")) } // atelet shares ateapi's namespace in every supported deployment topology, // so we read it from Kubernetes' downward API rather than expose a flag. @@ -331,7 +329,12 @@ func main() { } volPlugins := make(map[string]volume.VolumePluginControlPlane) - var dialerOpts []controlapi.DialerOption + dialerOpts := []controlapi.DialerOption{ + controlapi.WithNodeGetter(func(ctx context.Context, nodeName string) error { + _, err := clientset.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + return err + }), + } if *ateletInsecure { dialerOpts = append(dialerOpts, controlapi.WithInsecureCredentials()) } @@ -386,7 +389,7 @@ func main() { apiauthn.UnaryServerInterceptor(authCfg), ateinterceptors.MaxDeadlineUnaryInterceptor(maxRPCDeadline), ateinterceptors.ServerUnaryInterceptor, - authz.UnaryServerInterceptor(authorizer, *experimentalEnableAuthz), + authz.UnaryServerInterceptor(authorizer), ateinterceptors.RejectUnknownFieldsUnaryInterceptor, preview.UnaryServerInterceptor, } @@ -511,7 +514,6 @@ func logFlagValues(ctx context.Context) { slog.String("postgres-owner-role", *postgresOwnerRole), slog.String("postgres-schema", *postgresSchema), slog.Int("postgres-pool-max-conns", int(*postgresPoolMaxConns)), - slog.Bool("experimental-enable-authz", *experimentalEnableAuthz), slog.Any("authz-bootstrap-owners", *authzBootstrapOwners), slog.String("actor-id-jwt-pool", *actorIDJWTPoolFile), slog.String("actor-jwt-issuer", *actorJWTIssuer), @@ -696,9 +698,28 @@ func buildJWTProviders(ctx context.Context, cfg *apiauthn.AuthenticationConfig) return serverCfg, nil } +// validateActorJWTIssuer checks that issuer is an OpenID issuer identifier: an +// https URL with a host and no user info, query, or fragment. A path is +// allowed. func validateActorJWTIssuer(issuer string) error { if issuer == "" { return errors.New("required") } - return actoridjwt.ValidateIssuer(issuer) + u, err := url.Parse(issuer) + if err != nil { + return fmt.Errorf("invalid issuer: %w", err) + } + switch { + case u.Scheme != "https": + return fmt.Errorf("issuer %q must use the https scheme", issuer) + case u.Hostname() == "": + return fmt.Errorf("issuer %q has no host", issuer) + case u.User != nil: + return fmt.Errorf("issuer %q must not contain user info", issuer) + case strings.Contains(issuer, "?"): + return fmt.Errorf("issuer %q must not contain a query", issuer) + case strings.Contains(issuer, "#"): + return fmt.Errorf("issuer %q must not contain a fragment", issuer) + } + return nil } diff --git a/cmd/ateapi/main_test.go b/cmd/ateapi/main_test.go index 5a94f9e225..36fad8d0a3 100644 --- a/cmd/ateapi/main_test.go +++ b/cmd/ateapi/main_test.go @@ -105,7 +105,11 @@ func TestValidateActorJWTIssuer(t *testing.T) { {issuer: "", wantErr: true}, {issuer: "http://idp.example.com", wantErr: true}, {issuer: "https://idp.ate-system.svc"}, - {issuer: "https://idp.example.com/prod", wantErr: true}, + {issuer: "https://idp.example.com/prod"}, + {issuer: "https:///prod", wantErr: true}, + {issuer: "https://user@idp.example.com", wantErr: true}, + {issuer: "https://idp.example.com?env=prod", wantErr: true}, + {issuer: "https://idp.example.com#prod", wantErr: true}, } { err := validateActorJWTIssuer(tt.issuer) if gotErr := err != nil; gotErr != tt.wantErr { diff --git a/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig_test.go b/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig_test.go index f01ba39756..e5b1e7dbfc 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig_test.go @@ -48,8 +48,6 @@ import ( // testGvisorDefault is the gvisor class default TestMain creates. const testGvisorDefault = "gvisor-default-test" -const testPauseImage = "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" - // testHoldFinalizer keeps a SandboxConfig on the API server after it is // deleted, so a test can observe how a config that is being deleted is treated. const testHoldFinalizer = "test.ate.dev/hold" @@ -2397,9 +2395,6 @@ func TestAddProtectionFinalizer(t *testing.T) { func makeSandboxConfig(name string, class atev1alpha1.SandboxClass, isDefault bool) *atev1alpha1.SandboxConfig { v := atev1alpha1.SandboxVersionConfig{Name: "v1"} - if class == atev1alpha1.SandboxClassGvisor { - v.PauseImage = testPauseImage - } sc := &atev1alpha1.SandboxConfig{ ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: atev1alpha1.SandboxConfigSpec{ diff --git a/cmd/atecontroller/internal/workersync/syncer.go b/cmd/atecontroller/internal/workersync/syncer.go index 7f485bc26b..d6ecb1661e 100644 --- a/cmd/atecontroller/internal/workersync/syncer.go +++ b/cmd/atecontroller/internal/workersync/syncer.go @@ -346,13 +346,11 @@ func (s *WorkerPoolSyncer) createOrUpdateWorker(ctx context.Context, key workerK WorkerPodUid: string(pod.UID), NodeName: pod.Spec.NodeName, WorkerSpiffeId: workerSPIFFEID(pod), - SandboxClass: string(pool.Spec.DefaultSandboxClass()), Labels: pool.GetLabels(), Epoch: podEpoch(pod), - // Capacity is the Worker's to report, not the syncer's to infer - // from the pod: it is what the worker can actually supply. Until - // that report lands, CreateWorker's reified ceiling holds the - // Worker to a single Actor. + // Capacity and runtimes are the Worker's to report, not the + // syncer's to infer from the pod: they are what the worker can + // actually supply. } // status is output-only: CreateWorker registers the Worker UNAVAILABLE. // @@ -397,16 +395,6 @@ func (s *WorkerPoolSyncer) createOrUpdateWorker(ctx context.Context, key workerK w.Labels = pool.GetLabels() changed = true } - if w.GetSandboxClass() != string(pool.Spec.DefaultSandboxClass()) { - // Expected mid-rollout: sandboxClass drives the worker pod's shape, so - // editing it on the pool replaces every pod rather than reclassifying - // any. This pod predates that edit and is on its way out; its successor - // registers under a new key with the new class. Writing the pool's value - // back would be rejected, and would misreport this pod's shape to the - // scheduler if it were not. - slog.DebugContext(ctx, "Syncer: registered worker sandbox class predates its pool", - append(key.logAttrs(), slog.String("registered", w.GetSandboxClass()), slog.String("pool", string(pool.Spec.DefaultSandboxClass())))...) - } if ips := podIPs(pod); len(ips) > 0 && !slices.Equal(w.GetIps(), ips) { // TODO: I don't think this is possible, but handling this case so we can // log it just in case we can reproduce it. It is logged rather than diff --git a/cmd/atecontroller/internal/workersync/syncer_restart_test.go b/cmd/atecontroller/internal/workersync/syncer_restart_test.go index ec540dd996..6a788cc33d 100644 --- a/cmd/atecontroller/internal/workersync/syncer_restart_test.go +++ b/cmd/atecontroller/internal/workersync/syncer_restart_test.go @@ -50,7 +50,7 @@ func TestSyncer_RegistersEpoch(t *testing.T) { ns, podName, poolName := "ns-syncer-epoch", "worker-epoch-1", "pool1" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) key := seedPod(t, pods, withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 2)) mustReconcile(t, ctx, s, key) @@ -73,7 +73,7 @@ func TestSyncer_RaisesEpochOnWorkerRestart(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 0) key := seedPod(t, pods, pod) mustReconcile(t, ctx, s, key) @@ -173,7 +173,7 @@ func TestSyncer_NoPlacementUnderAnEarlierEpoch(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) key := seedPod(t, pods, withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 0)) mustReconcile(t, ctx, s, key) if state := api.get(testPodUID).GetStatus().GetState(); state != ateapipb.WorkerState_WORKER_STATE_ACTIVE { @@ -209,7 +209,7 @@ func TestSyncer_NotReadyPodRegistersAtItsEpoch(t *testing.T) { ns, poolName := "ns-syncer-notready-restart", "pool1" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := withWorkerRestarts(workerPod(ns, "worker-1", poolName, testPodUID, "10.0.0.1"), 3) pod.Status.Conditions = nil key := seedPod(t, pods, pod) @@ -229,9 +229,8 @@ func TestSyncer_EpochNeverLowered(t *testing.T) { ns, podName, poolName := "ns-syncer-lower", "worker-lower-1", "pool1" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) registered := registeredWorker(ns, poolName, podName, testPodUID, "10.0.0.1") - registered.SandboxClass = "gvisor" registered.Epoch = 5 api.put(registered) key := seedPod(t, pods, withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 0)) diff --git a/cmd/atecontroller/internal/workersync/syncer_test.go b/cmd/atecontroller/internal/workersync/syncer_test.go index f331bfecba..437e37ae69 100644 --- a/cmd/atecontroller/internal/workersync/syncer_test.go +++ b/cmd/atecontroller/internal/workersync/syncer_test.go @@ -83,10 +83,10 @@ func workerPod(ns, name, poolName, uid, ip string) *corev1.Pod { return pod } -func workerPool(ns, name, sandboxClass string, labels map[string]string) *atev1alpha1.WorkerPool { +func workerPool(ns, name string, labels map[string]string) *atev1alpha1.WorkerPool { return &atev1alpha1.WorkerPool{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}, - Spec: atev1alpha1.WorkerPoolSpec{SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClass(sandboxClass)}}}, + Spec: atev1alpha1.WorkerPoolSpec{SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClassGvisor}}}, } } @@ -194,6 +194,13 @@ func waitForWorker(t *testing.T, ctx context.Context, api *fakeControl, name str return got } +// workerActive reports whether w is registered and ACTIVE. The syncer registers +// a Worker UNAVAILABLE and marks it available in a later call, so a Worker that +// exists is not yet one that takes Actors. +func workerActive(w *ateapipb.Worker) bool { + return w.GetStatus().GetState() == ateapipb.WorkerState_WORKER_STATE_ACTIVE +} + func TestSyncer_Lifecycle(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -201,7 +208,7 @@ func TestSyncer_Lifecycle(t *testing.T) { ns, podName, poolName := "ns-syncer-lifecycle", "worker-unit-1", "pool1" api := newFakeControl() - fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, "gvisor", map[string]string{"foo": "bar"})) + fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, map[string]string{"foo": "bar"})) // A pod with no IP is not registered: there is nothing to route to yet. pod := workerPod(ns, podName, poolName, testPodUID, "") @@ -226,19 +233,13 @@ func TestSyncer_Lifecycle(t *testing.T) { t.Fatalf("update pod: %v", err) } - got := waitForWorker(t, ctx, api, testPodUID, func(w *ateapipb.Worker) bool { return w != nil }) + got := waitForWorker(t, ctx, api, testPodUID, workerActive) if want := []string{"127.0.0.1"}; !slices.Equal(got.GetIps(), want) { t.Errorf("worker ips = %q, want %q", got.GetIps(), want) } - if got.GetSandboxClass() != "gvisor" { - t.Errorf("worker sandbox class = %q, want gvisor", got.GetSandboxClass()) - } if !maps.Equal(got.GetLabels(), map[string]string{"foo": "bar"}) { t.Errorf("worker labels = %v, want map[foo:bar]", got.GetLabels()) } - if got.GetStatus().GetState() != ateapipb.WorkerState_WORKER_STATE_ACTIVE { - t.Errorf("worker state = %v, want ACTIVE", got.GetStatus().GetState()) - } if err := fakeK8s.CoreV1().Pods(ns).Delete(ctx, podName, metav1.DeleteOptions{}); err != nil { t.Fatalf("delete pod: %v", err) @@ -246,8 +247,8 @@ func TestSyncer_Lifecycle(t *testing.T) { waitForWorker(t, ctx, api, testPodUID, func(w *ateapipb.Worker) bool { return w == nil }) } -// A pool that sets neither a sandbox class nor labels registers a worker -// carrying neither, rather than one carrying some default. +// A pool that sets no labels registers a worker carrying none, rather than one +// carrying some default. func TestSyncer_OmittedFields(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -255,7 +256,7 @@ func TestSyncer_OmittedFields(t *testing.T) { ns, podName, poolName := "ns-syncer-omitted", "worker-unit-1", "pool1" api := newFakeControl() - fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, "", nil)) + fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "127.0.0.1") if _, err := fakeK8s.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil { @@ -263,9 +264,6 @@ func TestSyncer_OmittedFields(t *testing.T) { } got := waitForWorker(t, ctx, api, testPodUID, func(w *ateapipb.Worker) bool { return w != nil }) - if got.GetSandboxClass() != "" { - t.Errorf("worker sandbox class = %q, want it empty", got.GetSandboxClass()) - } if len(got.GetLabels()) != 0 { t.Errorf("worker labels = %v, want none", got.GetLabels()) } @@ -282,7 +280,7 @@ func TestSyncer_DoesNotInferCapacityFromThePod(t *testing.T) { ns, podName, poolName := "ns-syncer-capacity", "worker-unit-1", "pool1" api := newFakeControl() - fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, "gvisor", nil)) + fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "127.0.0.1") pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{ @@ -438,7 +436,7 @@ func TestSyncer_ReconcileOrphanedWorkers(t *testing.T) { api.put(registeredWorker(ns, poolName, "worker-live", testPodUID, "10.0.0.9")) api.put(registeredWorker(ns, poolName, "worker-orphan", otherPodUID, "10.0.0.10")) - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) // The cache holds only the live pod, so it is an authoritative snapshot. seedPod(t, pods, workerPod(ns, "worker-live", poolName, testPodUID, "10.0.0.9")) @@ -554,7 +552,7 @@ func TestSyncer_CreateWorker_DualStack(t *testing.T) { ns, podName, poolName := "ns-syncer-dual-stack", "worker-unit-dual-stack", "pool-dual-stack" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "fd00::1") pod.Status.PodIPs = append(pod.Status.PodIPs, corev1.PodIP{IP: "10.0.0.1"}) key := seedPod(t, pods, pod) @@ -571,7 +569,7 @@ func TestSyncer_CreateWorker_SPIFFEID(t *testing.T) { ctx := context.Background() ns, podName, poolName := "ns-syncer-sa", "worker-unit-sa", "pool-sa" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "10.0.0.1") pod.Spec.ServiceAccountName = "substrate-worker" key := seedPod(t, pods, pod) @@ -593,7 +591,7 @@ func TestSyncer_UpdateWorker_RetryOnVersionConflict(t *testing.T) { ns, podName, poolName := "ns-syncer-conflict", "worker-unit-conflict", "pool-conflict" api := newFakeControl() - s, pods, poolIndexer := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", map[string]string{"foo": "bar"})) + s, pods, poolIndexer := setupReconcileTest(t, api, workerPool(ns, poolName, map[string]string{"foo": "bar"})) key := seedPod(t, pods, workerPod(ns, podName, poolName, testPodUID, "10.0.0.1")) mustReconcile(t, ctx, s, key) @@ -603,7 +601,7 @@ func TestSyncer_UpdateWorker_RetryOnVersionConflict(t *testing.T) { // Change a mutable worker field on the pool, so the next reconcile has an // update to make. - if err := poolIndexer.Update(workerPool(ns, poolName, "gvisor", map[string]string{"foo": "baz"})); err != nil { + if err := poolIndexer.Update(workerPool(ns, poolName, map[string]string{"foo": "baz"})); err != nil { t.Fatalf("updating pool: %v", err) } @@ -641,44 +639,13 @@ func TestSyncer_UpdateWorker_RetryOnVersionConflict(t *testing.T) { } } -// Editing a pool's sandboxClass rolls its pods rather than reclassifying them, -// so a pod that outlives the edit keeps the class it was built with. Writing the -// pool's new value back would be rejected as an immutable-field change, and -// would misreport this pod's shape to the scheduler if it were not. -func TestSyncer_SandboxClassDriftLeavesWorkerAlone(t *testing.T) { - ctx := context.Background() - - ns, podName, poolName := "ns-syncer-class", "worker-unit-class", "pool-class" - - api := newFakeControl() - s, pods, poolIndexer := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) - key := seedPod(t, pods, workerPod(ns, podName, poolName, testPodUID, "10.0.0.1")) - - mustReconcile(t, ctx, s, key) - registered := api.get(testPodUID) - - if err := poolIndexer.Update(workerPool(ns, poolName, "microvm", nil)); err != nil { - t.Fatalf("updating pool: %v", err) - } - mustReconcile(t, ctx, s, key) - - got := api.get(testPodUID) - if got.GetSandboxClass() != "gvisor" { - t.Errorf("worker sandbox class = %q, want it left at gvisor", got.GetSandboxClass()) - } - if got.GetMetadata().GetVersion() != registered.GetMetadata().GetVersion() { - t.Errorf("worker version = %d, want %d: the drift must not provoke a write", - got.GetMetadata().GetVersion(), registered.GetMetadata().GetVersion()) - } -} - // TestSyncer_MissingWorkerPool verifies that a pod whose WorkerPool is not in // the lister gets no Worker, and that the Worker comes once the pool does. func TestSyncer_MissingWorkerPool(t *testing.T) { ctx := context.Background() ns, podName, poolName := "ns-syncer-missingpool", "worker-orphan-1", "pool-orphan" - pool := workerPool(ns, poolName, "gvisor", nil) + pool := workerPool(ns, poolName, map[string]string{"foo": "bar"}) api := newFakeControl() // The pod is there; its pool is not yet. @@ -699,8 +666,8 @@ func TestSyncer_MissingWorkerPool(t *testing.T) { t.Fatalf("queued workers = %d after the pool appeared, want 1", got) } mustReconcile(t, ctx, s, key) - if got := api.get(testPodUID).GetSandboxClass(); got != "gvisor" { - t.Errorf("worker sandbox class = %q, want gvisor", got) + if got := api.get(testPodUID).GetLabels(); !maps.Equal(got, map[string]string{"foo": "bar"}) { + t.Errorf("worker labels = %v, want map[foo:bar]", got) } } @@ -926,7 +893,7 @@ func TestSyncer_OrphanedPodsOfDeletedPool_ViaInformer(t *testing.T) { ns, podName, poolName := "ns-syncer-orphaned", "worker-orphaned-1", "pool1" api := newFakeControl() - fakeK8s, poolClient := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, "gvisor", nil)) + fakeK8s, poolClient := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "10.0.0.7") if _, err := fakeK8s.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil { @@ -962,7 +929,7 @@ func TestSyncer_StartWaitsForWorkerPoolCache(t *testing.T) { //nolint:staticcheck // NewSimpleClientset is what the informer machinery takes. fakeK8s := fake.NewSimpleClientset(workerPod(ns, podName, poolName, testPodUID, "10.0.0.8")) workerFactory, workerInformer := WorkerPodInformer(fakeK8s) - _, poolFactory, workerPoolInformer := newWorkerPoolInformer(workerPool(ns, poolName, "", poolLabels)) + _, poolFactory, workerPoolInformer := newWorkerPoolInformer(workerPool(ns, poolName, poolLabels)) NewWorkerPoolSyncer(api, fakeK8s.CoreV1(), fakeK8s.AppsV1(), workerInformer, workerPoolInformer, 2).Start(ctx) workerFactory.Start(ctx.Done()) @@ -996,7 +963,7 @@ func TestSyncer_StartWaitsForWorkerPoolCache(t *testing.T) { func TestEnqueueWorkerPool(t *testing.T) { api := newFakeControl() s, pods, _ := setupReconcileTest(t, api) - pool := workerPool("ns-pool-update", "pool-a", "gvisor", nil) + pool := workerPool("ns-pool-update", "pool-a", nil) first := seedPod(t, pods, workerPod(pool.Namespace, "worker-1", pool.Name, testPodUID, "10.0.0.1")) second := seedPod(t, pods, workerPod(pool.Namespace, "worker-2", pool.Name, otherPodUID, "10.0.0.2")) @@ -1026,7 +993,7 @@ func TestEnqueueWorkerPool(t *testing.T) { func TestEnqueueWorkerPool_DeletedFinalStateUnknown(t *testing.T) { api := newFakeControl() s, pods, _ := setupReconcileTest(t, api) - pool := workerPool("ns-pool-tombstone", "pool-a", "gvisor", nil) + pool := workerPool("ns-pool-tombstone", "pool-a", nil) seedPod(t, pods, workerPod(pool.Namespace, "worker-1", pool.Name, testPodUID, "10.0.0.1")) s.enqueueWorkerPool(cache.DeletedFinalStateUnknown{Key: pool.Namespace + "/" + pool.Name, Obj: pool}) @@ -1051,15 +1018,13 @@ func TestSyncer_SoftDelete_ViaInformer(t *testing.T) { ns, podName, poolName := "ns-syncer-softdelete", "worker-soft-1", "pool1" api := newFakeControl() - fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, "gvisor", nil)) + fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "10.0.0.6") if _, err := fakeK8s.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil { t.Fatalf("create pod: %v", err) } - waitForWorker(t, ctx, api, testPodUID, func(w *ateapipb.Worker) bool { - return w.GetStatus().GetState() == ateapipb.WorkerState_WORKER_STATE_ACTIVE - }) + waitForWorker(t, ctx, api, testPodUID, workerActive) // The fake clientset stores updates verbatim, so setting DeletionTimestamp // simulates a pod in graceful termination that is still in the cache. @@ -1086,7 +1051,7 @@ func TestSyncer_TerminalPod_DeletesPod(t *testing.T) { ctx := context.Background() api := newFakeControl() api.put(registeredWorker(ns, poolName, podName, testPodUID, ip)) - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, ip) pod.Status.Phase = phase @@ -1158,15 +1123,13 @@ func TestSyncer_TerminalPod_ViaInformer(t *testing.T) { ns, podName, poolName := "ns-syncer-terminal", "worker-terminal-1", "pool1" api := newFakeControl() - fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, "gvisor", nil)) + fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "10.0.0.21") if _, err := fakeK8s.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil { t.Fatalf("create pod: %v", err) } - waitForWorker(t, ctx, api, testPodUID, func(w *ateapipb.Worker) bool { - return w.GetStatus().GetState() == ateapipb.WorkerState_WORKER_STATE_ACTIVE - }) + waitForWorker(t, ctx, api, testPodUID, workerActive) failed := pod.DeepCopy() failed.Status.Phase = corev1.PodFailed @@ -1191,7 +1154,7 @@ func TestSyncer_PodRecreatedWithNewUID(t *testing.T) { ns, podName, poolName := "ns-syncer-recreate", "worker-recreate-1", "pool1" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) oldKey := seedPod(t, pods, workerPod(ns, podName, poolName, testPodUID, "10.0.0.7")) mustReconcile(t, ctx, s, oldKey) @@ -1235,7 +1198,7 @@ func TestSyncer_DeletePodWithoutIP(t *testing.T) { ns, podName, poolName := "ns-syncer-neverip", "worker-noip-1", "pool1" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "") key := seedPod(t, pods, pod) @@ -1274,7 +1237,7 @@ func TestSyncer_PodWithIPButNotReadyIsUnavailable(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, "worker-notready-1", poolName, testPodUID, "10.0.0.9") tc.mutate(pod) key := seedPod(t, pods, pod) @@ -1298,7 +1261,7 @@ func TestSyncer_DeleteNeverReadyPod(t *testing.T) { ns, podName, poolName := "ns-syncer-neverready", "worker-neverready-1", "pool1" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "10.0.0.9") pod.Status.Conditions = nil key := seedPod(t, pods, pod) @@ -1323,7 +1286,7 @@ func TestSyncer_RegisteredPodWithoutIPIsUnavailable(t *testing.T) { ns, podName, poolName := "ns-syncer-lostip", "worker-lostip-1", "pool1" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) key := seedPod(t, pods, withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.9"), 0)) mustReconcile(t, ctx, s, key) @@ -1352,7 +1315,7 @@ func TestSyncer_ReadinessFlapDoesNotDeregister(t *testing.T) { ns, podName, poolName := "ns-syncer-flap", "worker-flap-1", "pool1" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "10.0.0.9") key := seedPod(t, pods, pod) @@ -1408,7 +1371,7 @@ func TestSyncer_ReconcileErrorsAreClassified(t *testing.T) { api.setCreateHook(func(*ateapipb.Worker) error { return status.Error(tc.code, "rejected") }) - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) key := seedPod(t, pods, workerPod(ns, podName, poolName, testPodUID, "10.0.0.9")) s.queue.Add(key) diff --git a/cmd/atelet/imagegc.go b/cmd/atelet/imagegc.go index 0325c9b74a..6a08e2cf1b 100644 --- a/cmd/atelet/imagegc.go +++ b/cmd/atelet/imagegc.go @@ -29,10 +29,7 @@ import ( "errors" "fmt" "log/slog" - "os" - "path/filepath" "runtime/debug" - "strings" "time" "github.com/spf13/pflag" @@ -76,7 +73,10 @@ func validateImageCacheGCFlags() error { // future), making just-pulled layers evictable. return fmt.Errorf("--image-cache-min-age %v must be >= 0", *imageCacheMinAge) } - if imageCacheDirOutsideBasePath(*imageCacheDir) { + // Warn-worthy, not an error: a separate cache volume is legitimate + // (recommended for IOPS), but the watermarks then measure a different + // volume than actor state. + if !nodepath.UnderBasePath(*imageCacheDir) { slog.Warn("Image cache dir is outside the worker base path; its volume watermarks are measured separately from actor state", slog.String("image_cache_dir", *imageCacheDir), slog.String("actors_dir", nodepath.ActorsDir)) @@ -84,18 +84,6 @@ func validateImageCacheGCFlags() error { return nil } -// imageCacheDirOutsideBasePath reports whether the cache dir is outside -// the worker base path — the watermarks then measure a different volume -// than actor state. Warn-worthy, not an error: a separate cache volume is -// legitimate (recommended for IOPS). -func imageCacheDirOutsideBasePath(dir string) bool { - abs, err := filepath.Abs(dir) - if err != nil { - abs = filepath.Clean(dir) - } - return !strings.HasPrefix(abs, nodepath.BasePath+string(os.PathSeparator)) -} - // imageCacheGCTarget computes the bytes a pass should free: the larger // of the watermark shortfall (kubelet's formula — usage at highPct frees // down to lowPct) and the pool's overage past maxBytes, but never more diff --git a/cmd/atelet/imagegc_test.go b/cmd/atelet/imagegc_test.go index 5555768af1..a23f114a64 100644 --- a/cmd/atelet/imagegc_test.go +++ b/cmd/atelet/imagegc_test.go @@ -26,7 +26,6 @@ import ( "time" "go.ate.dev/substrate/internal/imagecache" - "go.ate.dev/substrate/internal/nodepath" ) func TestImageCacheGCTarget(t *testing.T) { @@ -154,30 +153,6 @@ func TestValidateImageCacheGCFlags(t *testing.T) { } } -func TestImageCacheDirOutsideBasePath(t *testing.T) { - cases := []struct { - name string - dir string - want bool - }{ - {"inside", filepath.Join(nodepath.BasePath, "image-cache"), false}, - {"inside with doubled separator", nodepath.BasePath + "//image-cache", false}, - {"inside via dot-dot", nodepath.BasePath + "/x/../image-cache", false}, - {"base path itself is not inside", nodepath.BasePath, true}, - {"sibling with the base path as name prefix", nodepath.BasePath + "-other/image-cache", true}, - {"outside", "/var/lib/elsewhere/image-cache", true}, - {"dot-dot escaping the base path", nodepath.BasePath + "/../elsewhere/image-cache", true}, - {"relative resolves against the cwd, not the base path", "image-cache", true}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if got := imageCacheDirOutsideBasePath(tc.dir); got != tc.want { - t.Errorf("imageCacheDirOutsideBasePath(%q) = %v, want %v", tc.dir, got, tc.want) - } - }) - } -} - type fakeGCStore struct { size int64 sizeErr error diff --git a/cmd/atelet/internal/apivalidation/validate.go b/cmd/atelet/internal/apivalidation/validate.go index 0f07acd06d..19ec1e6877 100644 --- a/cmd/atelet/internal/apivalidation/validate.go +++ b/cmd/atelet/internal/apivalidation/validate.go @@ -83,7 +83,11 @@ func ValidateCustom_EgressGateway_Address(_ context.Context, _ operation.Operati return resources.ValidateHostPort(fldPath, *value) } -func ValidateCustom_SandboxRuntime_SandboxClass(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { +func ValidateCustom_ActorSandboxRuntime_SandboxClass(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { + return resources.ValidateSandboxClass(fldPath, *value) +} + +func ValidateCustom_WorkerSandboxClass_SandboxClass(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { return resources.ValidateSandboxClass(fldPath, *value) } diff --git a/cmd/atelet/internal/apivalidation/validation_test.go b/cmd/atelet/internal/apivalidation/validation_test.go index fff8d0b2b6..6276cf600f 100644 --- a/cmd/atelet/internal/apivalidation/validation_test.go +++ b/cmd/atelet/internal/apivalidation/validation_test.go @@ -35,26 +35,29 @@ func assertValidateErr(t *testing.T, got field.ErrorList, want field.ErrorList) field.ErrorMatcher{}.ByType().ByField().ByOrigin().Test(t, want, got) } -// runtimeWithAttributes returns a valid gVisor runtime carrying n attributes -// with distinct keys. -func runtimeWithAttributes(n int) *ateletpb.SandboxRuntime { +// compatWithAttributes returns a valid v1 compatibility identity carrying n +// attributes with distinct keys. +func compatWithAttributes(n int) *ateletpb.VersionedSandboxCompat { attrs := make([]*ateletpb.AttributeEntry, n) for i := range attrs { attrs[i] = &ateletpb.AttributeEntry{Key: fmt.Sprintf("k%d", i), Value: "v"} } - return &ateletpb.SandboxRuntime{ - SandboxClass: "gvisor", - Version: &ateletpb.VersionedSandboxCompat{SchemaVersion: "v1", Attributes: attrs}, - } + return &ateletpb.VersionedSandboxCompat{SchemaVersion: "v1", Attributes: attrs} +} + +// runtimeWithAttributes returns a valid gVisor runtime carrying n attributes +// with distinct keys. +func runtimeWithAttributes(n int) *ateletpb.ActorSandboxRuntime { + return &ateletpb.ActorSandboxRuntime{SandboxClass: "gvisor", Version: compatWithAttributes(n)} } -// sandboxRuntimes returns n runtimes built by mk. -func sandboxRuntimes(n int, mk func() *ateletpb.SandboxRuntime) []*ateletpb.SandboxRuntime { - rts := make([]*ateletpb.SandboxRuntime, n) - for i := range rts { - rts[i] = mk() +// sandboxCompats returns n compatibility identities built by mk. +func sandboxCompats(n int, mk func() *ateletpb.VersionedSandboxCompat) []*ateletpb.VersionedSandboxCompat { + compats := make([]*ateletpb.VersionedSandboxCompat, n) + for i := range compats { + compats[i] = mk() } - return rts + return compats } func TestValidateRequestActorSuspendRequest(t *testing.T) { @@ -477,7 +480,7 @@ func TestValidateCheckpointRequest(t *testing.T) { }, { name: "spec errors surface at the spec's path", obj: valid(func(r *ateletpb.CheckpointRequest) { - r.Spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "_pause", Image: testDigestImage}}} + r.Spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "_sandbox", Image: testDigestImage}}} }), want: field.ErrorList{field.Invalid(field.NewPath("spec", "containers").Index(0).Child("name"), nil, "").WithOrigin("format=k8s-short-name")}, }, { @@ -652,7 +655,7 @@ func TestValidateRestoreRequest(t *testing.T) { }, { name: "spec errors surface at the spec's path", obj: valid(func(r *ateletpb.RestoreRequest) { - r.Spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "_pause", Image: testDigestImage}}} + r.Spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "_sandbox", Image: testDigestImage}}} }), want: field.ErrorList{field.Invalid(field.NewPath("spec", "containers").Index(0).Child("name"), nil, "").WithOrigin("format=k8s-short-name")}, }, { @@ -939,7 +942,7 @@ func TestValidateTerminateRequest(t *testing.T) { }, { name: "spec errors surface at the spec's path", obj: valid(func(r *ateletpb.TerminateRequest) { - r.Spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "_pause", Image: testDigestImage}}} + r.Spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "_sandbox", Image: testDigestImage}}} }), want: field.ErrorList{field.Invalid(field.NewPath("spec", "containers").Index(0).Child("name"), nil, "").WithOrigin("format=k8s-short-name")}, }, { @@ -1012,26 +1015,58 @@ func TestValidateDetachActorRequest(t *testing.T) { } func TestValidateRegisterWorkerRequest(t *testing.T) { - testRuntime := func() *ateletpb.SandboxRuntime { - return &ateletpb.SandboxRuntime{ - SandboxClass: "gvisor", - Version: &ateletpb.VersionedSandboxCompat{ - SchemaVersion: "v1", - Attributes: []*ateletpb.AttributeEntry{ - {Key: "architecture", Value: "amd64"}, - }, + testCompat := func() *ateletpb.VersionedSandboxCompat { + return &ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, }, } } - withCapacity := func(c *ateletpb.WorkerResources) *ateletpb.RegisterWorkerRequest { + // testRuntime wraps compat as one of the worker's runtimes. + testRuntime := func(compat *ateletpb.VersionedSandboxCompat) *ateletpb.WorkerSandboxRuntime { + return &ateletpb.WorkerSandboxRuntime{Version: compat} + } + testClass := func() *ateletpb.WorkerSandboxClass { + return &ateletpb.WorkerSandboxClass{SandboxClass: "gvisor", DefaultRuntime: testRuntime(testCompat())} + } + // withClass returns a request reporting c as its only sandbox class. + withClass := func(c *ateletpb.WorkerSandboxClass) *ateletpb.RegisterWorkerRequest { return &ateletpb.RegisterWorkerRequest{ - Capacity: c, - DefaultRuntime: testRuntime(), + Capacity: &ateletpb.WorkerResources{}, + SandboxClasses: []*ateletpb.WorkerSandboxClass{c}, + } + } + // withDefaultRuntime returns a request whose gVisor class cold-boots + // under compat. + withDefaultRuntime := func(compat *ateletpb.VersionedSandboxCompat) *ateletpb.RegisterWorkerRequest { + return withClass(&ateletpb.WorkerSandboxClass{SandboxClass: "gvisor", DefaultRuntime: testRuntime(compat)}) + } + // withRestorable returns a request whose gVisor class can also restore + // compats. + withRestorable := func(compats ...*ateletpb.VersionedSandboxCompat) *ateletpb.RegisterWorkerRequest { + c := testClass() + for _, compat := range compats { + var rt *ateletpb.WorkerSandboxRuntime + if compat != nil { + rt = testRuntime(compat) + } + c.RestorableRuntimes = append(c.RestorableRuntimes, rt) + } + return withClass(c) + } + withCapacity := func(capacity *ateletpb.WorkerResources) *ateletpb.RegisterWorkerRequest { + return &ateletpb.RegisterWorkerRequest{ + Capacity: capacity, + SandboxClasses: []*ateletpb.WorkerSandboxClass{testClass()}, } } capacityPath := field.NewPath("capacity") - defaultRuntimePath := field.NewPath("default_runtime") - restorablePath := field.NewPath("restorable_runtimes") + classesPath := field.NewPath("sandbox_classes") + classPath := classesPath.Index(0) + defaultRuntimePath := classPath.Child("default_runtime") + defaultVersionPath := defaultRuntimePath.Child("version") + restorablePath := classPath.Child("restorable_runtimes") tests := []struct { name string @@ -1039,228 +1074,146 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { want field.ErrorList }{{ name: "valid", - obj: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{}, DefaultRuntime: testRuntime()}, + obj: withClass(testClass()), }, { name: "valid with restorable runtimes", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: testRuntime(), - RestorableRuntimes: []*ateletpb.SandboxRuntime{testRuntime()}, - }, + obj: withRestorable(testCompat()), }, { name: "missing capacity", - obj: &ateletpb.RegisterWorkerRequest{DefaultRuntime: testRuntime()}, + obj: withCapacity(nil), want: field.ErrorList{field.Required(field.NewPath("capacity"), "")}, }, { - name: "missing default_runtime", + name: "missing sandbox_classes", obj: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{}}, - want: field.ErrorList{field.Required(defaultRuntimePath, "")}, + want: field.ErrorList{field.Required(classesPath, "")}, }, { - name: "missing default_runtime.sandbox_class", + name: "empty sandbox_classes", obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: &ateletpb.SandboxRuntime{ - Version: testRuntime().Version, - }, + Capacity: &ateletpb.WorkerResources{}, + SandboxClasses: []*ateletpb.WorkerSandboxClass{}, }, - want: field.ErrorList{field.Required(defaultRuntimePath.Child("sandbox_class"), "")}, + want: field.ErrorList{field.Required(classesPath, "")}, }, { - name: "default_runtime.sandbox_class too long", + name: "too many sandbox_classes", obj: &ateletpb.RegisterWorkerRequest{ Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: &ateletpb.SandboxRuntime{ - SandboxClass: strings.Repeat("c", 64), - Version: testRuntime().Version, + SandboxClasses: []*ateletpb.WorkerSandboxClass{ + testClass(), + {SandboxClass: "microvm", DefaultRuntime: testRuntime(testCompat())}, }, }, + want: field.ErrorList{field.TooMany(classesPath, 2, 1).WithOrigin("maxItems")}, + }, { + name: "missing sandbox_classes.sandbox_class", + obj: withClass(&ateletpb.WorkerSandboxClass{DefaultRuntime: testRuntime(testCompat())}), + want: field.ErrorList{field.Required(classPath.Child("sandbox_class"), "")}, + }, { + name: "sandbox_classes.sandbox_class too long", + obj: withClass(&ateletpb.WorkerSandboxClass{SandboxClass: strings.Repeat("c", 64), DefaultRuntime: testRuntime(testCompat())}), want: field.ErrorList{ - field.TooLong(defaultRuntimePath.Child("sandbox_class"), "", 63).WithOrigin("maxLength"), - field.NotSupported[string](defaultRuntimePath.Child("sandbox_class"), nil, nil), + field.TooLong(classPath.Child("sandbox_class"), "", 63).WithOrigin("maxLength"), + field.NotSupported[string](classPath.Child("sandbox_class"), nil, nil), }, }, { - name: "unsupported default_runtime.sandbox_class", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: &ateletpb.SandboxRuntime{ - SandboxClass: "kata", - Version: testRuntime().Version, - }, - }, - want: field.ErrorList{field.NotSupported[string](defaultRuntimePath.Child("sandbox_class"), nil, nil)}, + name: "unsupported sandbox_classes.sandbox_class", + obj: withClass(&ateletpb.WorkerSandboxClass{SandboxClass: "kata", DefaultRuntime: testRuntime(testCompat())}), + want: field.ErrorList{field.NotSupported[string](classPath.Child("sandbox_class"), nil, nil)}, + }, { + name: "missing sandbox_classes.default_runtime", + obj: withClass(&ateletpb.WorkerSandboxClass{SandboxClass: "gvisor"}), + want: field.ErrorList{field.Required(defaultRuntimePath, "")}, }, { name: "missing default_runtime.version", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: &ateletpb.SandboxRuntime{SandboxClass: "gvisor"}, - }, - want: field.ErrorList{field.Required(defaultRuntimePath.Child("version"), "")}, + obj: withClass(&ateletpb.WorkerSandboxClass{SandboxClass: "gvisor", DefaultRuntime: &ateletpb.WorkerSandboxRuntime{}}), + want: field.ErrorList{field.Required(defaultVersionPath, "")}, }, { - name: "missing default_runtime.version.schema_version", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: &ateletpb.SandboxRuntime{ - SandboxClass: "gvisor", - Version: &ateletpb.VersionedSandboxCompat{ - Attributes: []*ateletpb.AttributeEntry{{Key: "architecture", Value: "amd64"}}, - }, - }, - }, - want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "schema_version"), "")}, + name: "missing default_runtime.schema_version", + obj: withDefaultRuntime(&ateletpb.VersionedSandboxCompat{ + Attributes: []*ateletpb.AttributeEntry{{Key: "architecture", Value: "amd64"}}, + }), + want: field.ErrorList{field.Required(defaultVersionPath.Child("schema_version"), "")}, }, { - name: "default_runtime.version.schema_version too long", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: &ateletpb.SandboxRuntime{ - SandboxClass: "gvisor", - Version: &ateletpb.VersionedSandboxCompat{ - SchemaVersion: strings.Repeat("v", 65), - Attributes: []*ateletpb.AttributeEntry{{Key: "architecture", Value: "amd64"}}, - }, - }, - }, - want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("version", "schema_version"), "", 64).WithOrigin("maxLength")}, + name: "default_runtime.schema_version too long", + obj: withDefaultRuntime(&ateletpb.VersionedSandboxCompat{ + SchemaVersion: strings.Repeat("v", 65), + Attributes: []*ateletpb.AttributeEntry{{Key: "architecture", Value: "amd64"}}, + }), + want: field.ErrorList{field.TooLong(defaultVersionPath.Child("schema_version"), "", 64).WithOrigin("maxLength")}, }, { - name: "missing default_runtime.version.attributes", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: &ateletpb.SandboxRuntime{ - SandboxClass: "gvisor", - Version: &ateletpb.VersionedSandboxCompat{SchemaVersion: "v1"}, - }, - }, - want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "attributes"), "")}, + name: "missing default_runtime.attributes", + obj: withDefaultRuntime(&ateletpb.VersionedSandboxCompat{SchemaVersion: "v1"}), + want: field.ErrorList{field.Required(defaultVersionPath.Child("attributes"), "")}, }, { name: "duplicate attribute key", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: &ateletpb.SandboxRuntime{ - SandboxClass: "gvisor", - Version: &ateletpb.VersionedSandboxCompat{ - SchemaVersion: "v1", - Attributes: []*ateletpb.AttributeEntry{ - {Key: "architecture", Value: "amd64"}, - {Key: "architecture", Value: "arm64"}, - }, - }, + obj: withDefaultRuntime(&ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "architecture", Value: "arm64"}, }, - }, - want: field.ErrorList{field.Duplicate(defaultRuntimePath.Child("version", "attributes").Index(1), nil)}, + }), + want: field.ErrorList{field.Duplicate(defaultVersionPath.Child("attributes").Index(1), nil)}, }, { name: "missing attribute key", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: &ateletpb.SandboxRuntime{ - SandboxClass: "gvisor", - Version: &ateletpb.VersionedSandboxCompat{ - SchemaVersion: "v1", - Attributes: []*ateletpb.AttributeEntry{{Value: "v"}}, - }, - }, - }, - want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "attributes").Index(0).Child("key"), "")}, + obj: withDefaultRuntime(&ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{{Value: "v"}}, + }), + want: field.ErrorList{field.Required(defaultVersionPath.Child("attributes").Index(0).Child("key"), "")}, }, { name: "attribute key too long", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: &ateletpb.SandboxRuntime{ - SandboxClass: "gvisor", - Version: &ateletpb.VersionedSandboxCompat{ - SchemaVersion: "v1", - Attributes: []*ateletpb.AttributeEntry{{Key: strings.Repeat("k", 129), Value: "v"}}, - }, - }, - }, - want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("version", "attributes").Index(0).Child("key"), "", 128).WithOrigin("maxLength")}, + obj: withDefaultRuntime(&ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{{Key: strings.Repeat("k", 129), Value: "v"}}, + }), + want: field.ErrorList{field.TooLong(defaultVersionPath.Child("attributes").Index(0).Child("key"), "", 128).WithOrigin("maxLength")}, }, { name: "missing attribute value", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: &ateletpb.SandboxRuntime{ - SandboxClass: "gvisor", - Version: &ateletpb.VersionedSandboxCompat{ - SchemaVersion: "v1", - Attributes: []*ateletpb.AttributeEntry{{Key: "k"}}, - }, - }, - }, - want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "attributes").Index(0).Child("value"), "")}, + obj: withDefaultRuntime(&ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{{Key: "k"}}, + }), + want: field.ErrorList{field.Required(defaultVersionPath.Child("attributes").Index(0).Child("value"), "")}, }, { name: "attribute value too long", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: &ateletpb.SandboxRuntime{ - SandboxClass: "gvisor", - Version: &ateletpb.VersionedSandboxCompat{ - SchemaVersion: "v1", - Attributes: []*ateletpb.AttributeEntry{{Key: "k", Value: strings.Repeat("v", 257)}}, - }, - }, - }, - want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("version", "attributes").Index(0).Child("value"), "", 256).WithOrigin("maxLength")}, + obj: withDefaultRuntime(&ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{{Key: "k", Value: strings.Repeat("v", 257)}}, + }), + want: field.ErrorList{field.TooLong(defaultVersionPath.Child("attributes").Index(0).Child("value"), "", 256).WithOrigin("maxLength")}, }, { name: "attributes at the bound", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: runtimeWithAttributes(32), - }, + obj: withDefaultRuntime(compatWithAttributes(32)), }, { name: "too many attributes", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: runtimeWithAttributes(33), - }, - want: field.ErrorList{field.TooMany(defaultRuntimePath.Child("version", "attributes"), 33, 32).WithOrigin("maxItems")}, + obj: withDefaultRuntime(compatWithAttributes(33)), + want: field.ErrorList{field.TooMany(defaultVersionPath.Child("attributes"), 33, 32).WithOrigin("maxItems")}, }, { name: "nil restorable_runtimes entry", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: testRuntime(), - RestorableRuntimes: []*ateletpb.SandboxRuntime{nil}, - }, + obj: withRestorable(nil), want: field.ErrorList{field.Required(restorablePath.Index(0), "")}, }, { name: "invalid restorable_runtimes entry", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: testRuntime(), - RestorableRuntimes: []*ateletpb.SandboxRuntime{{SandboxClass: "gvisor"}}, - }, - want: field.ErrorList{field.Required(restorablePath.Index(0).Child("version"), "")}, - }, { - name: "unsupported restorable_runtimes sandbox_class", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: testRuntime(), - RestorableRuntimes: []*ateletpb.SandboxRuntime{{ - SandboxClass: "GVISOR", - Version: testRuntime().Version, - }}, - }, - want: field.ErrorList{field.NotSupported[string](restorablePath.Index(0).Child("sandbox_class"), nil, nil)}, + obj: withRestorable(&ateletpb.VersionedSandboxCompat{ + Attributes: []*ateletpb.AttributeEntry{{Key: "architecture", Value: "amd64"}}, + }), + want: field.ErrorList{field.Required(restorablePath.Index(0).Child("version", "schema_version"), "")}, }, { name: "restorable_runtimes at the bound", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: testRuntime(), - RestorableRuntimes: sandboxRuntimes(32, testRuntime), - }, + obj: withRestorable(sandboxCompats(32, testCompat)...), }, { name: "too many restorable_runtimes", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, - DefaultRuntime: testRuntime(), - RestorableRuntimes: sandboxRuntimes(33, testRuntime), - }, + obj: withRestorable(sandboxCompats(33, testCompat)...), want: field.ErrorList{field.TooMany(restorablePath, 33, 32).WithOrigin("maxItems")}, }, { name: "full capacity", obj: &ateletpb.RegisterWorkerRequest{ Capacity: &ateletpb.WorkerResources{Actors: 4, CpuMillicores: 4000, MemoryBytes: 8 << 30}, - DefaultRuntime: testRuntime(), + SandboxClasses: []*ateletpb.WorkerSandboxClass{testClass()}, }, }, { name: "negative actors", - obj: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: -1}, DefaultRuntime: testRuntime()}, + obj: withCapacity(&ateletpb.WorkerResources{Actors: -1}), want: field.ErrorList{field.Invalid(field.NewPath("capacity", "actors"), nil, "").WithOrigin("minimum")}, }, { name: "negative cpu_millicores", diff --git a/cmd/atelet/internal/apivalidation/zz_generated.validation.go b/cmd/atelet/internal/apivalidation/zz_generated.validation.go index d26f538dd1..62e02b5a9a 100644 --- a/cmd/atelet/internal/apivalidation/zz_generated.validation.go +++ b/cmd/atelet/internal/apivalidation/zz_generated.validation.go @@ -171,6 +171,82 @@ func Validate_ActorMetadataItem( return errs } +// Validate_ActorSandboxRuntime validates an instance of ActorSandboxRuntime according +// to declarative validation rules in the API schema. +func Validate_ActorSandboxRuntime( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateletpb.ActorSandboxRuntime) (errs field.ErrorList) { + + { // field ateletpb.ActorSandboxRuntime.SandboxClass + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // custom validation + if e := ValidateCustom_ActorSandboxRuntime_SandboxClass(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 63); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.ActorSandboxRuntime) *string { + return &oldObj.SandboxClass + }) + errs = append(errs, fn(fldPath.Child("sandbox_class"), &obj.SandboxClass, oldVal, oldObj != nil)...) + } + + { // field ateletpb.ActorSandboxRuntime.Version + fn := func( + fldPath *field.Path, + obj, oldObj *ateletpb.VersionedSandboxCompat, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_VersionedSandboxCompat(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.ActorSandboxRuntime) *ateletpb.VersionedSandboxCompat { + return oldObj.Version + }) + errs = append(errs, fn(fldPath.Child("version"), obj.Version, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_AttributeEntry validates an instance of AttributeEntry according // to declarative validation rules in the API schema. func Validate_AttributeEntry( @@ -1711,6 +1787,37 @@ func Validate_ObjectStorage( errs = append(errs, fn(fldPath.Child("snapshot_uri"), &obj.SnapshotUri, oldVal, oldObj != nil)...) } + { // field ateletpb.ObjectStorage.Sharing + fn := func( + fldPath *field.Path, + obj, oldObj *ateletpb.SnapshotSharing, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 2); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.ObjectStorage) *ateletpb.SnapshotSharing { + return &oldObj.Sharing + }) + errs = append(errs, fn(fldPath.Child("sharing"), &obj.Sharing, oldVal, oldObj != nil)...) + } + return errs } @@ -1949,10 +2056,10 @@ func Validate_RegisterWorkerRequest( errs = append(errs, fn(fldPath.Child("capacity"), obj.Capacity, oldVal, oldObj != nil)...) } - { // field ateletpb.RegisterWorkerRequest.DefaultRuntime + { // field ateletpb.RegisterWorkerRequest.SandboxClasses fn := func( fldPath *field.Path, - obj, oldObj *ateletpb.SandboxRuntime, + obj, oldObj []*ateletpb.WorkerSandboxClass, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -1962,62 +2069,45 @@ func Validate_RegisterWorkerRequest( } // call field-attached validations earlyReturn := false - if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.PtrSliceNoNils[ateletpb.WorkerSandboxClass](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } - if earlyReturn { - return // do not proceed - } - // call the type's validation function - errs = append(errs, Validate_SandboxRuntime(ctx, op, fldPath, obj, oldObj)...) - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateletpb.RegisterWorkerRequest) *ateletpb.SandboxRuntime { - return oldObj.DefaultRuntime - }) - errs = append(errs, fn(fldPath.Child("default_runtime"), obj.DefaultRuntime, oldVal, oldObj != nil)...) - } - - { // field ateletpb.RegisterWorkerRequest.RestorableRuntimes - fn := func( - fldPath *field.Path, - obj, oldObj []*ateletpb.SandboxRuntime, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.PtrSliceNoNils[ateletpb.SandboxRuntime](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 1).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } - if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + if e := validate.RequiredSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } - if e := validate.OptionalSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } if earlyReturn { return // do not proceed } + if e := validate.MinItems(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + // lists with map semantics require unique keys + if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, + func(a *ateletpb.WorkerSandboxClass, b *ateletpb.WorkerSandboxClass) bool { + return a.SandboxClass == b.SandboxClass + }); len(e) != 0 { + errs = append(errs, e...) + } // iterate the list and call the type's validation function - if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, nil, nil, Validate_SandboxRuntime); len(e) != 0 { + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, + func(a *ateletpb.WorkerSandboxClass, b *ateletpb.WorkerSandboxClass) bool { + return a.SandboxClass == b.SandboxClass + }, ateDeepEqual, Validate_WorkerSandboxClass); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, - func(oldObj *ateletpb.RegisterWorkerRequest) []*ateletpb.SandboxRuntime { - return oldObj.RestorableRuntimes + func(oldObj *ateletpb.RegisterWorkerRequest) []*ateletpb.WorkerSandboxClass { + return oldObj.SandboxClasses }) - errs = append(errs, fn(fldPath.Child("restorable_runtimes"), obj.RestorableRuntimes, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("sandbox_classes"), obj.SandboxClasses, oldVal, oldObj != nil)...) } return errs @@ -2699,7 +2789,7 @@ func Validate_RestoreRequest( { // field ateletpb.RestoreRequest.SandboxRuntime fn := func( fldPath *field.Path, - obj, oldObj *ateletpb.SandboxRuntime, + obj, oldObj *ateletpb.ActorSandboxRuntime, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -2717,11 +2807,11 @@ func Validate_RestoreRequest( return // do not proceed } // call the type's validation function - errs = append(errs, Validate_SandboxRuntime(ctx, op, fldPath, obj, oldObj)...) + errs = append(errs, Validate_ActorSandboxRuntime(ctx, op, fldPath, obj, oldObj)...) return } oldVal := safe.Field(oldObj, - func(oldObj *ateletpb.RestoreRequest) *ateletpb.SandboxRuntime { + func(oldObj *ateletpb.RestoreRequest) *ateletpb.ActorSandboxRuntime { return oldObj.SandboxRuntime }) errs = append(errs, fn(fldPath.Child("sandbox_runtime"), obj.SandboxRuntime, oldVal, oldObj != nil)...) @@ -3020,7 +3110,7 @@ func Validate_RunRequest( { // field ateletpb.RunRequest.SandboxRuntime fn := func( fldPath *field.Path, - obj, oldObj *ateletpb.SandboxRuntime, + obj, oldObj *ateletpb.ActorSandboxRuntime, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -3038,11 +3128,11 @@ func Validate_RunRequest( return // do not proceed } // call the type's validation function - errs = append(errs, Validate_SandboxRuntime(ctx, op, fldPath, obj, oldObj)...) + errs = append(errs, Validate_ActorSandboxRuntime(ctx, op, fldPath, obj, oldObj)...) return } oldVal := safe.Field(oldObj, - func(oldObj *ateletpb.RunRequest) *ateletpb.SandboxRuntime { + func(oldObj *ateletpb.RunRequest) *ateletpb.ActorSandboxRuntime { return oldObj.SandboxRuntime }) errs = append(errs, fn(fldPath.Child("sandbox_runtime"), obj.SandboxRuntime, oldVal, oldObj != nil)...) @@ -3051,82 +3141,6 @@ func Validate_RunRequest( return errs } -// Validate_SandboxRuntime validates an instance of SandboxRuntime according -// to declarative validation rules in the API schema. -func Validate_SandboxRuntime( - ctx context.Context, op operation.Operation, fldPath *field.Path, - obj, oldObj *ateletpb.SandboxRuntime) (errs field.ErrorList) { - - { // field ateletpb.SandboxRuntime.SandboxClass - fn := func( - fldPath *field.Path, - obj, oldObj *string, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - // custom validation - if e := ValidateCustom_SandboxRuntime_SandboxClass(ctx, op, fldPath, obj, oldObj); len(e) != 0 { - errs = append(errs, e...) - } - if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 63); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateletpb.SandboxRuntime) *string { - return &oldObj.SandboxClass - }) - errs = append(errs, fn(fldPath.Child("sandbox_class"), &obj.SandboxClass, oldVal, oldObj != nil)...) - } - - { // field ateletpb.SandboxRuntime.Version - fn := func( - fldPath *field.Path, - obj, oldObj *ateletpb.VersionedSandboxCompat, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - // call the type's validation function - errs = append(errs, Validate_VersionedSandboxCompat(ctx, op, fldPath, obj, oldObj)...) - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateletpb.SandboxRuntime) *ateletpb.VersionedSandboxCompat { - return oldObj.Version - }) - errs = append(errs, fn(fldPath.Child("version"), obj.Version, oldVal, oldObj != nil)...) - } - - return errs -} - // Validate_SecurityContext validates an instance of SecurityContext according // to declarative validation rules in the API schema. func Validate_SecurityContext( @@ -4351,6 +4365,162 @@ func Validate_WorkerResources( return errs } +// Validate_WorkerSandboxClass validates an instance of WorkerSandboxClass according +// to declarative validation rules in the API schema. +func Validate_WorkerSandboxClass( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateletpb.WorkerSandboxClass) (errs field.ErrorList) { + + { // field ateletpb.WorkerSandboxClass.SandboxClass + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // custom validation + if e := ValidateCustom_WorkerSandboxClass_SandboxClass(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 63); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.WorkerSandboxClass) *string { + return &oldObj.SandboxClass + }) + errs = append(errs, fn(fldPath.Child("sandbox_class"), &obj.SandboxClass, oldVal, oldObj != nil)...) + } + + { // field ateletpb.WorkerSandboxClass.DefaultRuntime + fn := func( + fldPath *field.Path, + obj, oldObj *ateletpb.WorkerSandboxRuntime, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_WorkerSandboxRuntime(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.WorkerSandboxClass) *ateletpb.WorkerSandboxRuntime { + return oldObj.DefaultRuntime + }) + errs = append(errs, fn(fldPath.Child("default_runtime"), obj.DefaultRuntime, oldVal, oldObj != nil)...) + } + + { // field ateletpb.WorkerSandboxClass.RestorableRuntimes + fn := func( + fldPath *field.Path, + obj, oldObj []*ateletpb.WorkerSandboxRuntime, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateletpb.WorkerSandboxRuntime](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, nil, nil, Validate_WorkerSandboxRuntime); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.WorkerSandboxClass) []*ateletpb.WorkerSandboxRuntime { + return oldObj.RestorableRuntimes + }) + errs = append(errs, fn(fldPath.Child("restorable_runtimes"), obj.RestorableRuntimes, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_WorkerSandboxRuntime validates an instance of WorkerSandboxRuntime according +// to declarative validation rules in the API schema. +func Validate_WorkerSandboxRuntime( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateletpb.WorkerSandboxRuntime) (errs field.ErrorList) { + + { // field ateletpb.WorkerSandboxRuntime.Version + fn := func( + fldPath *field.Path, + obj, oldObj *ateletpb.VersionedSandboxCompat, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_VersionedSandboxCompat(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.WorkerSandboxRuntime) *ateletpb.VersionedSandboxCompat { + return oldObj.Version + }) + errs = append(errs, fn(fldPath.Child("version"), obj.Version, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_WorkloadSpec validates an instance of WorkloadSpec according // to declarative validation rules in the API schema. func Validate_WorkloadSpec( diff --git a/cmd/atelet/internal/ateletpath/ateletpath.go b/cmd/atelet/internal/ateletpath/ateletpath.go index dc0dbec4b3..55d1b4450f 100644 --- a/cmd/atelet/internal/ateletpath/ateletpath.go +++ b/cmd/atelet/internal/ateletpath/ateletpath.go @@ -55,6 +55,15 @@ var ( // runtime registered before a restart resolves the moment atelet serves // again. SandboxRuntimeIndexFile = filepath.Join(nodepath.BasePath, "sandbox-runtime-index.json") + + // SnapshotCacheDir is the node-local cache of shared snapshot files, the + // snapshots many actors are created from (see + // cmd/atelet/internal/filecache). It lives under BasePath because cache + // hits are served as hard links into the per-actor restore dirs under + // SnapshotsDir, and link(2) requires both ends on one mounted filesystem, + // the same mount, not merely the same disk. With SnapshotsDir on its own + // mount, hits are served as copies instead. + SnapshotCacheDir = filepath.Join(nodepath.BasePath, "snapshot-cache") ) func ActorPath(actorUID string) string { @@ -64,6 +73,16 @@ func ActorPath(actorUID string) string { ) } +// ActorSnapshotPath is the actor's snapshot staging directory, which may be on +// a different filesystem from ActorPath. Checkpoints rename files between its +// subdirectories, so they must stay on one filesystem. +func ActorSnapshotPath(actorUID string) string { + return filepath.Join( + nodepath.SnapshotsDir, + actorUID, + ) +} + // ActorSandboxAssetsFile is the per-actor file where atelet records the sandbox // binaries (class + content-addressed asset set, for this node's architecture) // the actor is currently running. It is written at Run/Restore and read at @@ -123,19 +142,19 @@ func OCIBundlePath(actorUID, containerName string) string { func CheckpointStateDir(actorUID string) string { return filepath.Join( - ActorPath(actorUID), + ActorSnapshotPath(actorUID), "checkpoint-state", ) } func LocalCheckpointsDir(actorUID string) string { return filepath.Join( - ActorPath(actorUID), + ActorSnapshotPath(actorUID), "local-checkpoint", ) } -// LocalSnapshotDir is the directory holding one named local (pause) snapshot +// LocalSnapshotDir is the directory holding one named node-local snapshot // of an actor: the checkpoint files plus their manifest. func LocalSnapshotDir(actorUID, snapshotName string) string { return filepath.Join(LocalCheckpointsDir(actorUID), snapshotName) @@ -202,7 +221,7 @@ func SystemInfoVolumeRoot(actorUID, volumeName string) string { // yet been loaded. func RestoreStateDir(actorUID string) string { return filepath.Join( - ActorPath(actorUID), + ActorSnapshotPath(actorUID), "restore-state", ) } diff --git a/cmd/atelet/internal/ateletpath/ateletpath_test.go b/cmd/atelet/internal/ateletpath/ateletpath_test.go index 5968e3f2af..a095aafba9 100644 --- a/cmd/atelet/internal/ateletpath/ateletpath_test.go +++ b/cmd/atelet/internal/ateletpath/ateletpath_test.go @@ -26,24 +26,41 @@ func TestActorDirs(t *testing.T) { if actorDirs.GetRootDir() != ActorPath(actorUID) { t.Fatalf("root_dir = %q, want %q", actorDirs.GetRootDir(), ActorPath(actorUID)) } - under := map[string]string{ + snapshotRoot := ActorSnapshotPath(actorUID) + // Disjoint trees: neither root may sit inside (or equal) the other. + for _, p := range [][2]string{{snapshotRoot, actorDirs.GetRootDir()}, {actorDirs.GetRootDir(), snapshotRoot}} { + if rel, err := filepath.Rel(p[1], p[0]); err == nil && !strings.HasPrefix(rel, "..") { + t.Fatalf("%q is inside %q; ActorSnapshotPath and ActorPath must be disjoint", p[0], p[1]) + } + } + + underActorRoot := map[string]string{ "oci_bundle_dir": actorDirs.GetOciBundleDir(), - "checkpoint_dir": actorDirs.GetCheckpointDir(), - "restore_dir": actorDirs.GetRestoreDir(), "durable_dir_volume_mounts_dir": actorDirs.GetDurableDirVolumeMountsDir(), "system_info_volume_roots_dir": actorDirs.GetSystemInfoVolumeRootsDir(), "external_volume_mounts_dir": actorDirs.GetExternalVolumeMountsDir(), "sandbox_assets_dir": actorDirs.GetSandboxAssetsDir(), } + underSnapshotRoot := map[string]string{ + "checkpoint_dir": actorDirs.GetCheckpointDir(), + "restore_dir": actorDirs.GetRestoreDir(), + "local_checkpoints_dir": LocalCheckpointsDir(actorUID), + "local_snapshot_dir": LocalSnapshotDir(actorUID, "snap-1"), + } + seen := map[string]string{} - for field, dir := range under { - rel, err := filepath.Rel(actorDirs.GetRootDir(), dir) - if err != nil || rel == "." || strings.HasPrefix(rel, "..") { - t.Errorf("%s = %q is not a proper subdirectory of root_dir %q", field, dir, actorDirs.GetRootDir()) - } - if other, dup := seen[dir]; dup { - t.Errorf("%s and %s share %q", field, other, dir) + checkSubdirs := func(parentName, parent string, dirs map[string]string) { + for field, dir := range dirs { + rel, err := filepath.Rel(parent, dir) + if err != nil || rel == "." || strings.HasPrefix(rel, "..") { + t.Errorf("%s = %q is not a proper subdirectory of %s %q", field, dir, parentName, parent) + } + if other, dup := seen[dir]; dup { + t.Errorf("%s and %s share %q", field, other, dir) + } + seen[dir] = field } - seen[dir] = field } + checkSubdirs("root_dir", actorDirs.GetRootDir(), underActorRoot) + checkSubdirs("snapshot_root", snapshotRoot, underSnapshotRoot) } diff --git a/cmd/atelet/internal/snapshottracker/actor.go b/cmd/atelet/internal/snapshottracker/actor.go index b72a8af303..9365f3ee74 100644 --- a/cmd/atelet/internal/snapshottracker/actor.go +++ b/cmd/atelet/internal/snapshottracker/actor.go @@ -140,10 +140,10 @@ type actorSnapshotsRecord struct { // checkpoint has superseded it. // // Measuring from UndurableSince rather than LatestLocal.CommitTime - // ensures an actor that pauses and resumes repeatedly still uploads - // within uploadDelay instead of postponing durability indefinitely. An - // upload changes UndurableSince only once it succeeds, and a failed or - // interrupted one leaves it as it was. + // ensures an actor that suspends asynchronously and resumes repeatedly + // still uploads within uploadDelay instead of postponing durability + // indefinitely. An upload changes UndurableSince only once it succeeds, and + // a failed or interrupted one leaves it as it was. UndurableSince time.Time `json:"undurableSince,omitzero"` // DetachAt is when atelet next checks whether to detach the actor. Zero // means no detach is scheduled (see detachScheduled). @@ -161,9 +161,9 @@ type actorSnapshotsRecord struct { // // It is not updated, but the actor is queued again, when a worker pulls // the actor out of the workqueue and: - // - A Run, Restore, Checkpoint, or PromoteSnapshot of the actor is in - // progress (BeginStart, BeginCheckpoint, or BeginPromote). Its end - // queues the actor again. + // - A Run, Restore, Checkpoint, PromoteSnapshot, or Terminate of the + // actor is in progress (BeginStart, BeginCheckpoint, BeginPromote, or + // BeginTerminate). Its end queues the actor again. // - Another operation on the actor, such as a Detach call, is in // progress. Its end queues the actor again. // - The detach is not due yet. The worker queues the actor again for @@ -172,25 +172,39 @@ type actorSnapshotsRecord struct { // atelet never detaches a snapshot that no detach can make durable (see // detachTarget). DetachAt time.Time `json:"detachAt,omitzero"` + // Started is set when a Run or Restore of the actor begins on this node + // (markStarted), and cleared when a Checkpoint commits a snapshot + // (CommitLocal or CommitDurable). While it is unset, the actor's + // directories and volumes are still that Checkpoint's to clean up. + Started bool `json:"started,omitempty"` // ReportDueSince is when the latest background change atelet owes a // report of (a new LatestDurable, or local snapshots cleared by a // detach) finished. Zero once no report is due: the control plane - // acknowledged or refused it, a commit or Detach call returned the - // state, or atelet gave up after reportQueue.giveUpAfter. + // acknowledged or refused it, a commit returned the state, or atelet gave + // up after reportQueue.giveUpAfter. ReportDueSince time.Time `json:"reportDueSince,omitzero"` } // holdKind is the kind of RPC in progress on an actor, or holdNone. Callers -// serialize lifecycle RPCs per actor, so at most one holds an actor at once. +// serialize lifecycle RPCs per actor, so at most one RPC holds an actor at +// once; a retried immediate Checkpoint promotes inside its checkpoint hold. type holdKind int const ( holdNone holdKind = iota holdStart // Run or Restore - holdCheckpoint // Checkpoint - holdPromote // PromoteSnapshot + holdCheckpoint // Checkpoint under UPLOAD_MODE_DEFERRED + holdDurable // Checkpoint under UPLOAD_MODE_IMMEDIATE, or PromoteSnapshot + holdTerminate // Terminate ) +// blocksUploads reports whether no background upload of the actor starts while +// an RPC of kind k holds it: one that makes a snapshot durable, which would +// only cancel the upload, or Terminate, which forgets the actor. +func (k holdKind) blocksUploads() bool { + return k == holdDurable || k == holdTerminate +} + // actorSnapshots is what atelet tracks of one actor's snapshots on this node. // Its fields are guarded by the actor's lock in Tracker.locks. type actorSnapshots struct { @@ -204,10 +218,10 @@ type actorSnapshots struct { // A background worker or an RPC call such as Detach runs it; an actor has // at most one. The entry stays in memory while op is set. op *actorOp - // hold records the Run, Restore, Checkpoint, or PromoteSnapshot of the - // actor in progress on this node. While set, atelet does not detach the - // actor or report its state (nor start a background upload during a - // promote), and the entry stays in memory. + // hold records the Run, Restore, Checkpoint, PromoteSnapshot, or Terminate + // of the actor in progress on this node. While set, atelet does not detach + // the actor or report its state (nor start a background upload when + // hold.blocksUploads), and the entry stays in memory. hold holdKind // reportDeferred is set when a report of the actor's state is held back // while hold is set, so holdLocked queues it once the RPC ends. @@ -224,6 +238,25 @@ type actorSnapshots struct { unsaved bool } +// checkpointCommitted reports whether a Checkpoint of snap has committed it +// already: LatestLocal or LatestDurable is snap. It fails with +// FailedPrecondition when one of them has snap's UUID but a different +// generation, fidelity, or URI: the control plane named one snapshot two +// ways, so refuse rather than guess which is right. +func (a *actorSnapshots) checkpointCommitted(snap *ateletpb.Snapshot) (bool, error) { + for _, r := range []*SnapshotRef{a.LatestLocal, a.LatestDurable} { + if r == nil || r.uuid() != snap.GetSnapshotUuid() { + continue + } + if r.generation() != snap.GetGeneration() || r.Snapshot.GetFidelity() != snap.GetFidelity() || r.uri() != snap.GetObject().GetSnapshotUri() { + return false, apierror.FailedPrecondition("snapshot %s is committed as generation %d with fidelity %s at %q, but requested as generation %d with fidelity %s at %q", + r.uuid(), r.generation(), r.Snapshot.GetFidelity(), r.uri(), snap.GetGeneration(), snap.GetFidelity(), snap.GetObject().GetSnapshotUri()) + } + return true, nil + } + return false, nil +} + // localNeeded reports whether CommitLocal should record snap as LatestLocal: // LatestLocal does not already hold snap or a newer snapshot, and no other // committed snapshot has a higher generation or the same generation under @@ -379,6 +412,24 @@ func (a *actorSnapshots) runningUpload() *SnapshotRef { return &a.op.target } +// supersededUpload returns the upload in progress when a snapshot newer than +// the one it uploads is durable, or nil. +func (a *actorSnapshots) supersededUpload() *actorOp { + up := a.runningUpload() + if up == nil || a.LatestDurable == nil || up.generation() >= a.LatestDurable.generation() { + return nil + } + return a.op +} + +// localInUse reports whether a Run or Restore of the actor in progress may use +// a superseded local snapshot. A Restore may read any local snapshot in place; +// a Checkpoint or PromoteSnapshot only reads LatestLocal, which is not +// superseded until the new snapshot is committed, and a Terminate reads none. +func (a *actorSnapshots) localInUse() bool { + return a.hold == holdStart +} + // runningDetach returns the snapshot an opDetach goroutine is currently // detaching, or nil. func (a *actorSnapshots) runningDetach() *SnapshotRef { @@ -473,13 +524,15 @@ func (a *actorSnapshots) clearLatestLocal() { a.DetachAt = time.Time{} } -// markStarted unschedules the actor's detach as a Run or Restore of the actor -// begins on this node: atelet does not detach the actor again until its next -// checkpoint schedules a detach. It reports whether a detach was scheduled. +// markStarted records that a Run or Restore of the actor begins on this node: +// it sets Started and unschedules the actor's detach, so atelet does not +// detach the actor again until its next checkpoint schedules a detach. It +// reports whether either changed. func (a *actorSnapshots) markStarted() bool { - if a.DetachAt.IsZero() { + if a.Started && a.DetachAt.IsZero() { return false } + a.Started = true a.DetachAt = time.Time{} return true } @@ -487,8 +540,8 @@ func (a *actorSnapshots) markStarted() bool { // detachScheduled reports whether a detach of the actor is scheduled: atelet // holds a local snapshot of the actor, and no Run or Restore of the actor has // begun since that snapshot was committed. A scheduled detach runs once it is -// due (see detachDue) and no Run, Restore, Checkpoint, or PromoteSnapshot of -// the actor is in progress. +// due (see detachDue) and no Run, Restore, Checkpoint, PromoteSnapshot, or +// Terminate of the actor is in progress. func (a *actorSnapshots) detachScheduled() bool { return a.LatestLocal != nil && !a.DetachAt.IsZero() } @@ -497,7 +550,7 @@ func (a *actorSnapshots) detachScheduled() bool { // whether it is durable already. It fails with FailedPrecondition when no // detach can make it durable until a commit replaces it: LatestDurable names // the same snapshot differently, or LatestLocal is not durable and names no -// object storage location to upload it to, as after a pause today. +// object storage location to upload it to. func (a *actorSnapshots) detachTarget() (target SnapshotRef, alreadyDurable bool, err error) { target = *a.LatestLocal alreadyDurable, err = a.isLatestDurable(target) diff --git a/cmd/atelet/internal/snapshottracker/background.go b/cmd/atelet/internal/snapshottracker/background.go index eef10454c6..d836af2f07 100644 --- a/cmd/atelet/internal/snapshottracker/background.go +++ b/cmd/atelet/internal/snapshottracker/background.go @@ -19,7 +19,6 @@ import ( "log/slog" "time" - "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/proto/ateletpb" ) @@ -36,15 +35,13 @@ const ( // DefaultDetachDelay is how long an actor stays suspended on this node by // default, its latest snapshot kept locally, before atelet detaches it. DefaultDetachDelay = 30 * time.Minute - // DefaultUploadTimeout is how long one background upload attempt, - // including the one a background detach runs first, may run by default, - // on top of the time its snapshot takes at the estimated upload - // throughput, before atelet cancels it. It keeps a wedged object-store - // call from holding a background worker forever. + // DefaultUploadTimeout is how long an upload attempt, background or part + // of a detach, may run beyond its estimated upload time before atelet + // cancels it, so a wedged object-store call can't hold it forever. DefaultUploadTimeout = 2 * time.Minute // DefaultUploadEstThroughput, in bytes per second of a snapshot's - // allocated local size, is how fast atelet expects a background upload to - // run by default. Each attempt's timeout allows for the snapshot to upload + // allocated local size, is how fast atelet expects an upload to run by + // default. Each attempt's timeout allows for the snapshot to upload // at this rate, so a large snapshot is not cut off. DefaultUploadEstThroughput = 100 << 20 // maxUploadTimeoutDoublings caps how many times consecutive failures of @@ -88,10 +85,9 @@ func (k opKind) String() string { } } -// logOp logs how an upload or detach ended. Foreground failures are returned -// to the RPC caller, so only background failures are logged here: as a -// warning, or at info level once ctx has ended, as when Terminate or an atelet -// shutdown cancels the operation. +// logOp logs how an upload or detach ended. A failure logs as a warning, or at +// info level once ctx has ended, as when Terminate, an atelet shutdown, or a +// newer durable snapshot cancels the operation. func logOp(ctx context.Context, actorUID string, kind opKind, target SnapshotRef, uploaded, background bool, elapsed time.Duration, err error) { attrs := []any{ slog.String("op", kind.String()), @@ -104,7 +100,6 @@ func logOp(ctx context.Context, actorUID string, kind opKind, target SnapshotRef switch { case err == nil: slog.InfoContext(ctx, "Snapshot operation finished", append(attrs, slog.Bool("uploaded", uploaded))...) - case !background: case ctx.Err() != nil: slog.InfoContext(ctx, "Snapshot operation canceled", append(attrs, slog.Any("err", err))...) default: @@ -115,14 +110,19 @@ func logOp(ctx context.Context, actorUID string, kind opKind, target SnapshotRef // actorOp is an actor's operation in progress: an upload or a detach. An // actor has at most one, of any kind. // -// A worker runs it in the background once it is due. An atelet RPC such as -// Detach runs it in the RPC's goroutine instead, without going through the -// background queue. +// A worker runs it in the background once it is due. A Detach call begins one +// directly instead, and it runs on if the caller leaves. // // The operation holds the actor's lock only while it begins and ends, not // while it runs: -// - Terminate never waits for it. -// - Checkpoint never waits for an upload. +// - Forget cancels any operation in progress and waits for it to end before +// it drops the actor's state. No operation begins during the Terminate +// that calls it. +// - CommitDurable and CommitPromoted cancel an upload of a snapshot older +// than LatestDurable and wait for it to end; a deferred Checkpoint never +// waits for an upload. +// - No upload begins during a Checkpoint under UPLOAD_MODE_IMMEDIATE or a +// PromoteSnapshot, though one already running goes on. // - Run, Restore, and Checkpoint wait for a detach to end, so it never // deletes the actor's local snapshots underneath them. type actorOp struct { @@ -134,6 +134,9 @@ type actorOp struct { // done is closed once the operation has ended, and has finished any change // it makes to the actor's local snapshots. done chan struct{} + // state and err are a detach's result, set before done is closed. + state *ateletpb.ActorSnapshotState + err error } // Start reads each actor's snapshot state file on this node, queues the actor @@ -172,9 +175,13 @@ func (t *Tracker) Start() { } } -// Wait blocks until the workers have stopped, which they do once the tracker's -// context ends, or until ctx ends. It returns ctx's error if ctx ended first. +// Wait blocks until the workers and any detach a Detach call started have +// stopped, which they do once the tracker's context ends. It returns ctx's +// error if ctx ends first. Detach starts nothing after Wait begins. func (t *Tracker) Wait(ctx context.Context) error { + t.foregroundMu.Lock() + t.waiting = true + t.foregroundMu.Unlock() stopped := make(chan struct{}) go func() { t.workersDone.Wait() @@ -213,8 +220,10 @@ func (t *Tracker) syncActor(actorUID string, kind opKind) { // beginDueWork begins the actor's background operation of kind if one is due, // and returns the func that runs it. Otherwise it returns nil, and queues the // actor for when its next operations are due, unless an operation, Run, -// Restore, Checkpoint, or PromoteSnapshot in progress on the actor queues it -// again when it ends. +// Restore, Checkpoint, PromoteSnapshot, or Terminate in progress on the actor +// queues it again when it ends. An upload worker that finds no upload due +// clears a superseded Uploading snapshot instead, unless Terminate holds the +// actor. func (t *Tracker) beginDueWork(actorUID string, kind opKind) func() { if !t.locks.Lock(t.ctx, actorUID) { // atelet is shutting down. @@ -228,13 +237,17 @@ func (t *Tracker) beginDueWork(actorUID string, kind opKind) func() { a := t.findLocked(actorUID) if a == nil || a.op != nil { // Forgotten, or an operation is already in progress on the actor: one - // a worker of the other queue began, or one an atelet RPC such as - // Detach runs in the RPC's goroutine. When that operation ends, + // a worker of the other queue or a Detach call began. When it ends, // requeueLocked queues the actor again. return nil } + if a.hold == holdTerminate { + // A Terminate of the actor is in progress: Forget drops the actor, or + // the terminate's end queues it again if it fails first (see + // holdLocked). + return nil + } now := t.clock.Now() - var cleanup func() switch kind { case opDetach: if due, ok := t.detachDueLocked(a); ok && !due.After(now) { @@ -244,16 +257,17 @@ func (t *Tracker) beginDueWork(actorUID string, kind opKind) func() { // (see holdLocked). return nil } - if finish, err := t.beginDetachLocked(t.ctx, a, true); err == nil { + if _, run, err := t.beginDetachLocked(t.ctx, a, true); err == nil { t.instruments.recordDetachOverdue(t.ctx, now.Sub(due)) - return func() { _, _ = finish() } + return run } } case opUpload: if due, ok := t.uploadDueLocked(a); ok && !due.After(now) { - if a.hold == holdPromote { - // A PromoteSnapshot of the actor is in progress, and queues - // the actor again when it ends (see holdLocked). + if a.hold.blocksUploads() { + // A Checkpoint under UPLOAD_MODE_IMMEDIATE or a PromoteSnapshot + // of the actor is in progress, and queues the actor again when + // it ends (see holdLocked). return nil } run := t.beginUploadLocked(t.ctx, a) @@ -264,10 +278,10 @@ func (t *Tracker) beginDueWork(actorUID string, kind opKind) func() { // starting the next upload. When no upload runs (for example after an // immediate checkpoint or promote made a newer snapshot durable, // leaving no upload due), clear a superseded Uploading snapshot here. - cleanup = t.cleanupSupersededUploadingLocked(a) + t.cleanupSupersededUploadingLocked(a) } t.requeueLocked(a) - return cleanup + return nil } // nextDueLocked returns when the actor's next background detach or upload is @@ -283,13 +297,18 @@ func (t *Tracker) nextDueLocked(a *actorSnapshots) (time.Time, bool) { // requeueLocked queues the actor on each queue for when its next background // operation of that kind is due, if it has one, then drops the actor's entry // from memory unless the entry has an operation, or a Run, Restore, -// Checkpoint, or PromoteSnapshot, in progress, a retry hold, a failed upload -// kept to retry, or a change that failed to save. An entry whose Uploading is -// superseded is queued at once on the upload queue and kept in memory until an -// upload worker cleans it up. A save that finds the actor's directory gone -// leaves nothing unsaved. requeueLocked runs after a change to the actor is -// saved, so it must not change what the snapshot state file keeps. +// Checkpoint, PromoteSnapshot, or Terminate, in progress, a retry hold, a +// failed upload kept to retry, or a change that failed to save. An entry whose +// Uploading is superseded is queued at once on the upload queue and kept in +// memory until an upload worker cleans it up. A save that finds the actor's +// directory gone leaves nothing unsaved. requeueLocked runs after a change to +// the actor is saved, so it must not change what the snapshot state file keeps. func (t *Tracker) requeueLocked(a *actorSnapshots) { + if a.hold == holdTerminate { + // Forget drops the actor, or the terminate's end queues it again if it + // fails first (see holdLocked). + return + } now := t.clock.Now() // CommitDurable and CommitPromoted leave a superseded Uploading snapshot // for an upload worker to clear from the state file and delete from disk, @@ -334,63 +353,32 @@ func (t *Tracker) beginOpLocked(ctx context.Context, a *actorSnapshots, kind opK return ctx, a.op } -// opEntryLocked returns the actor's entry while op is still its operation in -// progress, or nil once Forget has abandoned op. -func (t *Tracker) opEntryLocked(actorUID string, op *actorOp) *actorSnapshots { - a := t.findLocked(actorUID) - if a == nil || a.op != op { - return nil - } - return a -} - -// abandonedError returns what an operation on the actor fails with once Forget -// has abandoned it: opErr if the operation failed already, and otherwise -// Aborted. -func abandonedError(actorUID string, opErr error) error { - if opErr != nil { - return opErr - } - return apierror.Aborted("actor %s was forgotten during the operation", actorUID) -} - -// updateOp runs fn under the actor's lock and saves the actor's state while op -// is still the actor's operation in progress. Once Forget has abandoned op, it -// leaves the actor alone and fails with abandonedError. -func (t *Tracker) updateOp(actorUID string, op *actorOp, opErr error, fn func(a *actorSnapshots)) error { +// updateOp runs fn on the actor's entry under the actor's lock, while an +// operation of the actor runs, and saves the actor's state. The operation keeps +// the entry in memory (see beginOpLocked). +func (t *Tracker) updateOp(actorUID string, fn func(a *actorSnapshots)) { t.lock(actorUID) defer t.unlock(actorUID) - a := t.opEntryLocked(actorUID, op) - if a == nil { - return abandonedError(actorUID, opErr) - } + a := t.findLocked(actorUID) fn(a) t.saveLocked(a) - return nil } -// endOpLocked ends op and returns the actor's entry, or nil once Forget has -// abandoned op. +// endOpLocked ends op, the actor's operation in progress, and returns the +// actor's entry. func (t *Tracker) endOpLocked(actorUID string, op *actorOp) *actorSnapshots { op.cancel() - a := t.opEntryLocked(actorUID, op) - if a != nil { - a.op = nil - } + a := t.findLocked(actorUID) + a.op = nil return a } // endOp ends op, queues the actor for its next background operation and any -// report due, and returns the actor's state. Once Forget has abandoned op, it -// leaves the actor alone and fails with abandonedError; otherwise it fails -// with opErr if non-nil. +// report due, and returns the actor's state, or opErr if non-nil. func (t *Tracker) endOp(actorUID string, op *actorOp, opErr error) (*ateletpb.ActorSnapshotState, error) { t.lock(actorUID) defer t.unlock(actorUID) a := t.endOpLocked(actorUID, op) - if a == nil { - return nil, abandonedError(actorUID, opErr) - } t.requeueLocked(a) t.queueReportLocked(a) if opErr != nil { @@ -486,19 +474,6 @@ func (t *Tracker) waitOpLocked(ctx context.Context, actorUID string, op *actorOp } } -// abandonOpLocked cancels the actor's operation in progress, if any, without -// waiting for it to end. -func (t *Tracker) abandonOpLocked(actorUID string) { - // An entry with an operation in progress is always in memory. - t.mu.Lock() - a := t.actors[actorUID] - t.mu.Unlock() - if a != nil && a.op != nil { - a.op.cancel() - a.op = nil - } -} - // backOffLocked records a failure of the actor's background work, and holds // that work back until the time it returns: later with each consecutive // failure. diff --git a/cmd/atelet/internal/snapshottracker/background_test.go b/cmd/atelet/internal/snapshottracker/background_test.go index cba3521552..982cadca07 100644 --- a/cmd/atelet/internal/snapshottracker/background_test.go +++ b/cmd/atelet/internal/snapshottracker/background_test.go @@ -34,12 +34,14 @@ import ( ) // useActorDir roots atelet's on-node state in temp directories and creates the -// tracked actor's directory, so the tracker writes its snapshot state file. +// tracked actor's directories, so the tracker writes its snapshot state file. func useActorDir(t *testing.T) { t.Helper() useTempNodeDirs(t) - if err := os.MkdirAll(ateletpath.ActorPath(trackedActorUID), 0o700); err != nil { - t.Fatal(err) + for _, dir := range []string{ateletpath.ActorPath(trackedActorUID), ateletpath.ActorSnapshotPath(trackedActorUID)} { + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } } } @@ -71,18 +73,13 @@ func beginOp(t *testing.T, tr *Tracker, target SnapshotRef) (context.Context, *a return ctx, op } -// endOp ends op, as a worker does, and reports whether it was still the -// tracked actor's operation in progress. It then finishes like a worker -// whose operation changed nothing, calling requeueLocked. -func endOp(tr *Tracker, op *actorOp) bool { +// endOp ends op, as a worker does whose operation changed nothing: it calls +// requeueLocked and closes op.done. +func endOp(tr *Tracker, op *actorOp) { + defer close(op.done) tr.lock(trackedActorUID) defer tr.unlock(trackedActorUID) - a := tr.endOpLocked(trackedActorUID, op) - if a == nil { - return false - } - tr.requeueLocked(a) - return true + tr.requeueLocked(tr.endOpLocked(trackedActorUID, op)) } // inMemory reports whether the tracker holds the tracked actor's entry in @@ -144,9 +141,7 @@ func TestEntryStaysInMemoryDuringOp(t *testing.T) { t.Error("entry dropped from memory during an operation, want it kept") } - if !endOp(tr, op) { - t.Fatal("endOpLocked = nil, want the operation still the actor's") - } + endOp(tr, op) if inMemory(tr) { t.Error("entry in memory after the operation ended, want it dropped") } @@ -217,40 +212,119 @@ func TestStartQueuesActorsWithSnapshotState(t *testing.T) { } } -// Forget does not wait for the actor's background operation in progress. It -// cancels the operation, which finds itself abandoned when it ends, so it -// leaves alone both the forgotten actor's state and an operation that began -// for the actor after Forget. -func TestForgetAbandonsInFlightOp(t *testing.T) { +// Forget cancels the actor's background operation in progress and waits for it +// to end, releasing the actor's lock meanwhile, before it drops the actor's +// state in memory and on disk. +func TestForgetWaitsForInFlightOp(t *testing.T) { useActorDir(t) tr := newTracker(t) s1 := testRef("s1", 1, time.Now()) tr.CommitLocal(trackedActorUID, s1) opCtx, op := beginOp(t, tr, s1) - returnsPromptly(t, "Forget during a background operation", func() { - if err := tr.Forget(trackedActorUID); err != nil { + forgetDone := make(chan error, 1) + go func() { forgetDone <- tr.Forget(trackedActorUID) }() + select { + case <-opCtx.Done(): + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for Forget to cancel the operation") + } + returnsPromptly(t, "lock while Forget waits", func() { + tr.lock(trackedActorUID) + tr.unlock(trackedActorUID) + }) + if !inMemory(tr) { + t.Error("entry dropped from memory while Forget waits, want it kept") + } + if got := uuidOf(newTracker(t).State(trackedActorUID).GetLatestLocal()); got != "s1" { + t.Errorf("latest_local a restarted atelet reads while Forget waits = %q, want %q", got, "s1") + } + select { + case err := <-forgetDone: + t.Fatalf("Forget returned (%v) before the canceled operation ended", err) + case <-time.After(50 * time.Millisecond): + } + endOp(tr, op) + select { + case err := <-forgetDone: + if err != nil { t.Errorf("Forget: %v", err) } - }) - if opCtx.Err() == nil { - t.Error("the operation's context is live after Forget, want it cancelled") + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for Forget to return") } - _, next := beginOp(t, tr, s1) - if endOp(tr, op) { - t.Error("endOpLocked = true after Forget, want the operation abandoned") + if inMemory(tr) { + t.Error("entry in memory after Forget, want it dropped") } - if got, want := tr.State(trackedActorUID), (&ateletpb.ActorSnapshotState{Uploading: s1.report()}); !proto.Equal(got, want) { - t.Errorf("state after forget = %v, want %v: only the operation begun after Forget", got, want) + if st := newTracker(t).State(trackedActorUID); !proto.Equal(st, &ateletpb.ActorSnapshotState{}) { + t.Errorf("state a restarted atelet reads after Forget = %v, want empty", st) } - if !endOp(tr, next) { - t.Error("endOpLocked = false for the operation begun after Forget, want the abandoned one's end to leave it alone") +} + +// From BeginTerminate on, no background upload or detach of the actor begins, +// though both are due. If the terminate fails before Forget, its end lets them +// begin again; once Forget has dropped the actor, its end leaves the actor +// alone and queues nothing for it. +func TestTrackerTerminateHoldsOffBackgroundWork(t *testing.T) { + for _, tc := range []struct { + name string + forget bool + }{ + {name: "terminate fails before Forget"}, + {name: "Forget drops the actor", forget: true}, + } { + t.Run(tc.name, func(t *testing.T) { + tr := newIdleTestTracker(t, newFakeSnapshotStore()) + t0 := tr.clock.Now() + tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + stepTo(tr, t0.Add(testDetachDelay)) // An upload and a detach are both due. + eventually(t, "upload and detach are queued", func() bool { + return queuedNow(tr) == 2 + }) + for _, q := range tr.queues { + uid, _ := q.Get() + q.Done(uid) + } + + end := tr.BeginTerminate(trackedActorUID) + for _, kind := range []opKind{opUpload, opDetach} { + if run := tr.beginDueWork(trackedActorUID, kind); run != nil { + t.Errorf("beginDueWork(%v) began work during Terminate, want none", kind) + run() + } + } + if tc.forget { + opCtx, op := beginOp(t, tr, testRef("s1", 1, t0)) + go func() { + <-opCtx.Done() + endOp(tr, op) + }() + if err := tr.Forget(trackedActorUID); err != nil { + t.Fatalf("Forget: %v", err) + } + } + end() + + if tc.forget { + if inMemory(tr) { + t.Error("entry in memory after Forget and the terminate's end, want it dropped") + } + if n := queuedNow(tr); n != 0 { + t.Errorf("queued = %d after Forget and the terminate's end, want nothing queued for the forgotten actor", n) + } + return + } + run := tr.beginDueWork(trackedActorUID, opUpload) + if run == nil { + t.Fatal("beginDueWork(upload) began nothing once the terminate failed, want the due upload") + } + run() + }) } } -// A Checkpoint's commits and LocalSnapshotsToKeep, PromoteSnapshot's commit, -// and State neither wait for the actor's background operation in progress nor -// cancel it. +// CommitLocal, LocalSnapshotsToKeep, and State neither wait for the actor's +// background upload in progress nor cancel it. func TestCommitsDoNotWaitForInFlightOp(t *testing.T) { useActorDir(t) tr := newTracker(t) @@ -265,8 +339,6 @@ func TestCommitsDoNotWaitForInFlightOp(t *testing.T) { }{ {"CommitLocal", func() { tr.CommitLocal(trackedActorUID, testRef("s2", 2, t0)) }}, {"LocalSnapshotsToKeep", func() { tr.LocalSnapshotsToKeep(trackedActorUID) }}, - {"CommitDurable", func() { tr.CommitDurable(trackedActorUID, testRef("s3", 3, t0)) }}, - {"CommitPromoted", func() { tr.CommitPromoted(trackedActorUID, testRef("s4", 4, t0)) }}, {"State", func() { tr.State(trackedActorUID) }}, } { returnsPromptly(t, c.name+" during a background operation", c.call) @@ -274,9 +346,7 @@ func TestCommitsDoNotWaitForInFlightOp(t *testing.T) { if err := opCtx.Err(); err != nil { t.Errorf("the operation's context ended with %v, want it still running", err) } - if !endOp(tr, op) { - t.Error("endOpLocked = false, want the operation still the actor's") - } + endOp(tr, op) } // An actor has at most one background operation in progress, and another @@ -302,9 +372,7 @@ func TestBeginOpLockedAllowsOneOpPerActor(t *testing.T) { t.Error("beginOpLocked for another actor = nil, want an operation") } - if !endOp(tr, op) { - t.Fatal("endOpLocked = false, want the operation still the actor's") - } + endOp(tr, op) _, op = beginOp(t, tr, s1) s1.Snapshot.SnapshotUuid = "changed" if got := op.target.Snapshot.GetSnapshotUuid(); got != "s1" { @@ -372,9 +440,8 @@ func (h *recordingHandler) WithAttrs([]slog.Attr) slog.Handler { return h } func (h *recordingHandler) WithGroup(string) slog.Handler { return h } -// logOp logs a finished operation at info level, a failed background one as a -// warning, and a background one canceled by its context at info level. A -// failed foreground operation goes back to its caller unlogged. +// logOp logs a finished operation at info level, a failed one as a warning, and +// one its context canceled at info level, in the background or not. func TestLogOp(t *testing.T) { canceled, cancel := context.WithCancel(context.Background()) cancel() @@ -389,7 +456,8 @@ func TestLogOp(t *testing.T) { {name: "finished", ctx: context.Background(), background: true, wantLevel: slog.LevelInfo, wantMsg: "Snapshot operation finished"}, {name: "background failure", ctx: context.Background(), background: true, err: errors.New("boom"), wantLevel: slog.LevelWarn, wantMsg: "Snapshot operation failed"}, {name: "background canceled", ctx: canceled, background: true, err: context.Canceled, wantLevel: slog.LevelInfo, wantMsg: "Snapshot operation canceled"}, - {name: "foreground failure", ctx: context.Background(), err: errors.New("boom")}, + {name: "foreground failure", ctx: context.Background(), err: errors.New("boom"), wantLevel: slog.LevelWarn, wantMsg: "Snapshot operation failed"}, + {name: "foreground canceled", ctx: canceled, err: context.Canceled, wantLevel: slog.LevelInfo, wantMsg: "Snapshot operation canceled"}, } { t.Run(tc.name, func(t *testing.T) { h := &recordingHandler{} @@ -400,12 +468,6 @@ func TestLogOp(t *testing.T) { logOp(tc.ctx, trackedActorUID, opUpload, testRef("s1", 1, time.Time{}), false, tc.background, time.Second, tc.err) h.mu.Lock() defer h.mu.Unlock() - if tc.wantMsg == "" { - if len(h.records) != 0 { - t.Errorf("logged %q, want nothing", h.records[0].Message) - } - return - } if len(h.records) != 1 || h.records[0].Message != tc.wantMsg || h.records[0].Level != tc.wantLevel { t.Errorf("logged %v, want one %v record %q", h.records, tc.wantLevel, tc.wantMsg) } diff --git a/cmd/atelet/internal/snapshottracker/detach.go b/cmd/atelet/internal/snapshottracker/detach.go index 85a2c8d59c..378311162e 100644 --- a/cmd/atelet/internal/snapshottracker/detach.go +++ b/cmd/atelet/internal/snapshottracker/detach.go @@ -22,17 +22,18 @@ import ( ) // beginDetachLocked begins a detach of the actor's latest local snapshot, -// under a context derived from ctx, and returns the func that finishes it -// once the caller releases the actor's lock. It begins nothing, and fails, +// under a context derived from ctx, and returns its op and the func that runs +// it once the caller releases the actor's lock. It begins nothing, and fails, // when no detach can make the snapshot durable (see detachTarget). -func (t *Tracker) beginDetachLocked(ctx context.Context, a *actorSnapshots, background bool) (func() (*ateletpb.ActorSnapshotState, error), error) { +func (t *Tracker) beginDetachLocked(ctx context.Context, a *actorSnapshots, background bool) (*actorOp, func(), error) { target, alreadyDurable, err := a.detachTarget() if err != nil { - return nil, err + return nil, nil, err } opCtx, op := t.beginOpLocked(ctx, a, opDetach, target) - return func() (*ateletpb.ActorSnapshotState, error) { - return t.finishDetach(opCtx, a.actorUID, op, alreadyDurable, background) + return op, func() { + defer close(op.done) + op.state, op.err = t.finishDetach(opCtx, a.actorUID, op, alreadyDurable, background) }, nil } @@ -40,10 +41,11 @@ func (t *Tracker) beginDetachLocked(ctx context.Context, a *actorSnapshots, back // actor's local snapshots. It is a no-op when atelet holds no local snapshot // for the actor. It fails with Aborted once a Run or Restore of the actor has // begun since its latest local snapshot, and with FailedPrecondition when no -// detach can make the snapshot durable. It runs in the calling goroutine -// rather than through the background queue, once any operation in progress on -// the actor has ended. Callers serialize it with the actor's other lifecycle -// RPCs (see Tracker). +// detach can make the snapshot durable. It starts once the actor's operation +// in progress ends, and runs on the tracker's context, not ctx. If ctx ends +// first, Detach returns ctx's error and the detach runs on, recorded by its +// report. After Wait begins, Detach fails with Unavailable. Callers serialize +// it with the actor's other lifecycle RPCs (see Tracker). func (t *Tracker) Detach(ctx context.Context, actorUID string) (*ateletpb.ActorSnapshotState, error) { if t.store == nil { return nil, apierror.Unimplemented("atelet has no snapshot store to detach actors with") @@ -59,12 +61,10 @@ func (t *Tracker) Detach(ctx context.Context, actorUID string) (*ateletpb.ActorS } if a == nil || a.LatestLocal == nil { // No local snapshot is stored on this node, so there is no work to be - // done. + // done. A report already due stays due, in case the caller doesn't + // record this state. st := &ateletpb.ActorSnapshotState{} if a != nil { - if a.reportDue() { - t.clearReportDueLocked(a) - } st = t.stateLocked(a) } t.unlock(actorUID) @@ -76,22 +76,45 @@ func (t *Tracker) Detach(ctx context.Context, actorUID string) (*ateletpb.ActorS t.unlock(actorUID) return nil, apierror.Aborted("actor %s is in use on this node", actorUID) } - finish, err := t.beginDetachLocked(ctx, a, false) - t.unlock(actorUID) + // Under foregroundMu, so no detach joins workersDone after Wait begins. + t.foregroundMu.Lock() + if t.waiting || t.ctx.Err() != nil { + t.foregroundMu.Unlock() + t.unlock(actorUID) + return nil, apierror.Unavailable("atelet is shutting down") + } + op, run, err := t.beginDetachLocked(t.ctx, a, false) if err != nil { + t.foregroundMu.Unlock() + t.unlock(actorUID) return nil, err } - return finish() + t.workersDone.Go(run) + t.foregroundMu.Unlock() + t.unlock(actorUID) + + select { + case <-op.done: + return op.state, t.detachCallError(ctx, op.err) + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// detachCallError returns err, or Unavailable if the tracker's shutdown ended +// the detach, so the caller doesn't mistake it for its own cancellation. +func (t *Tracker) detachCallError(ctx context.Context, err error) error { + if err != nil && t.ctx.Err() != nil && ctx.Err() == nil { + return apierror.Unavailable("atelet stopped during the detach: %v", err) + } + return err } // finishDetach runs op, a detach of the actor's latest local snapshot: it // uploads the snapshot unless already durable, then deletes the actor's local // snapshots. background reports whether a worker runs it, rather than a Detach -// call; a worker's detach makes a report of what it changed due, while a -// Detach call returns the new state and cancels any pending report. A detach -// Forget abandons leaves the actor alone, and fails. +// call. Either way it makes a report of what it changed due. func (t *Tracker) finishDetach(ctx context.Context, actorUID string, op *actorOp, alreadyDurable, background bool) (*ateletpb.ActorSnapshotState, error) { - defer close(op.done) uploaded := false target := op.target detachStart := t.clock.Now() @@ -102,8 +125,8 @@ func (t *Tracker) finishDetach(ctx context.Context, actorUID string, op *actorOp uploaded, _, err = t.uploadSnapshot(ctx, actorUID, target, background) } if err == nil { - err = t.updateOp(actorUID, op, nil, func(a *actorSnapshots) { - if t.recordUploadLocked(a, target, uploaded) && background { + t.updateOp(actorUID, func(a *actorSnapshots) { + if t.recordUploadLocked(a, target, uploaded) { t.markReportDueLocked(a) } }) @@ -120,29 +143,24 @@ func (t *Tracker) finishDetach(ctx context.Context, actorUID string, op *actorOp recordDetach := err == nil if recordDetach { detachStart = t.clock.Now() - // Forget or the caller may have ended ctx as the upload finished. + // Forget or the tracker's shutdown may have ended ctx as the upload + // finished. err = ctx.Err() } if err == nil { err = PruneLocalCheckpoints(ctx, actorUID) } - if abErr := t.updateOp(actorUID, op, err, func(a *actorSnapshots) { + t.updateOp(actorUID, func(a *actorSnapshots) { switch { case err == nil: t.clearLatestLocalLocked(a) - if background { - t.markReportDueLocked(a) - } else { - t.cancelReportLocked(a) - } + t.markReportDueLocked(a) case ctx.Err() == nil: // Detach again once the retry delay has passed, and no sooner // than the detach was due. a.DetachAt = later(a.DetachAt, t.backOffLocked(a)) } - }); abErr != nil { - err = abErr - } + }) // Log and record before endOp, which ends ctx. elapsed := t.clock.Since(detachStart) diff --git a/cmd/atelet/internal/snapshottracker/detach_test.go b/cmd/atelet/internal/snapshottracker/detach_test.go index 5c01ad25af..f9cd090aa8 100644 --- a/cmd/atelet/internal/snapshottracker/detach_test.go +++ b/cmd/atelet/internal/snapshottracker/detach_test.go @@ -17,6 +17,7 @@ package snapshottracker import ( "context" "errors" + "log/slog" "os" "slices" "testing" @@ -27,6 +28,7 @@ import ( "go.ate.dev/substrate/internal/proto/ateletpb" "google.golang.org/grpc/codes" "google.golang.org/protobuf/proto" + clocktesting "k8s.io/utils/clock/testing" ) // eventually polls cond until it holds, and fails the test if it does not @@ -334,7 +336,7 @@ func TestTrackerDetach(t *testing.T) { } } -// A local snapshot no detach can make durable, as a pause takes today, never +// A local snapshot no detach can make durable (e.g. with no upload URI) never // comes due for a background detach, rather than failing it again and again. // A checkpoint that replaces it schedules a detach again. func TestTrackerSkipsUndetachableSnapshots(t *testing.T) { @@ -402,7 +404,7 @@ func TestTrackerDetachTimer(t *testing.T) { tr.CommitLocal(trackedActorUID, testRef("s2", 2, tr.clock.Now())) due := detachDueOf(tr, trackedActorUID) - endCheckpoint, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s3", 3, time.Time{}).Snapshot) + _, endCheckpoint, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s3", 3, time.Time{}).Snapshot, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) if err != nil { t.Fatalf("beginCheckpoint: %v", err) } @@ -428,8 +430,8 @@ func TestTrackerDetachTimer(t *testing.T) { } } -// A start that cannot save that it unscheduled the actor's detach fails before -// anything starts, and leaves the detach scheduled. +// A start that cannot save that it began fails before anything starts, and +// leaves the detach scheduled and the actor not started. func TestTrackerStartFailsIfItCannotSave(t *testing.T) { tr := newIdleTestTracker(t, newFakeSnapshotStore()) stateFile := ateletpath.ActorSnapshotStateFile(trackedActorUID) @@ -438,7 +440,8 @@ func TestTrackerStartFailsIfItCannotSave(t *testing.T) { if err := os.MkdirAll(stateFile, 0o700); err != nil { t.Fatal(err) } - tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + s1 := testRef("s1", 1, tr.clock.Now()) + tr.CommitLocal(trackedActorUID, s1) due := detachDueOf(tr, trackedActorUID) if due.IsZero() { t.Fatal("no detach scheduled after the checkpoint") @@ -450,6 +453,9 @@ func TestTrackerStartFailsIfItCannotSave(t *testing.T) { if got := detachDueOf(tr, trackedActorUID); !got.Equal(due) { t.Errorf("detachDue after the failed start = %v, want %v kept", got, due) } + if !tr.NotStartedSince(trackedActorUID, s1.Snapshot) { + t.Error("NotStartedSince after the failed start = false, want true") + } if hold := tr.actors[trackedActorUID].hold; hold != holdNone { t.Errorf("hold after the failed start = %v, want none", hold) } @@ -483,7 +489,8 @@ func TestTrackerWaitsForDetach(t *testing.T) { { name: "BeginCheckpoint", begin: func(tr *Tracker, ctx context.Context, uid string) (func(), error) { - return tr.BeginCheckpoint(ctx, uid, testRef("s2", 2, time.Time{}).Snapshot) + _, end, err := tr.BeginCheckpoint(ctx, uid, testRef("s2", 2, time.Time{}).Snapshot, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + return end, err }, }, } { @@ -539,7 +546,7 @@ func TestTrackerDetachWaitChecksGeneration(t *testing.T) { stale := testRef("s0", 1, time.Time{}) returnsPromptly(t, "BeginCheckpoint of stale generation during detach", func() { - if _, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, stale.Snapshot); apierror.Code(err) != codes.Aborted { + if _, _, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, stale.Snapshot, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); apierror.Code(err) != codes.Aborted { t.Errorf("BeginCheckpoint = %v, want Aborted", err) } }) @@ -652,7 +659,7 @@ func TestTrackerHoldsFirstCheckpoint(t *testing.T) { store := newFakeSnapshotStore() tr := newIdleTestTracker(t, store) s1 := testRef("s1", 1, tr.clock.Now()) - end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, s1.Snapshot) + _, end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, s1.Snapshot, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) if err != nil { t.Fatalf("beginCheckpoint: %v", err) } @@ -710,12 +717,12 @@ func TestTrackerLiftsBackoffWithDetach(t *testing.T) { } } -// Forget does not wait for a detach in progress: it cancels the detach, which -// fails and leaves the forgotten actor's local snapshots for Terminate to -// prune. -func TestTrackerForgetAbandonsDetach(t *testing.T) { +// Forget cancels a detach in progress and waits for it to end, leaving the +// forgotten actor's local snapshots for Terminate to prune. +func TestTrackerForgetCancelsDetach(t *testing.T) { store := newFakeSnapshotStore() store.setBlocking(true) + store.setIgnoreCancel(true) tr := newTestTracker(t, store) if err := os.MkdirAll(ateletpath.LocalSnapshotDir(trackedActorUID, "s1"), 0o700); err != nil { t.Fatal(err) @@ -724,14 +731,30 @@ func TestTrackerForgetAbandonsDetach(t *testing.T) { detached := startDetach(t, tr) waitStarted(t, store) - returnsPromptly(t, "Forget during a detach", func() { - if err := tr.Forget(trackedActorUID); err != nil { - t.Errorf("Forget: %v", err) - } - }) + forgetDone := make(chan error, 1) + go func() { + forgetDone <- tr.Forget(trackedActorUID) + }() + if got := waitCanceled(t, store); got != "s1" { + t.Fatalf("canceled detach upload = %q, want s1", got) + } + select { + case err := <-forgetDone: + t.Fatalf("Forget returned (%v) before the canceled detach ended", err) + case <-time.After(20 * time.Millisecond): + } + store.release <- context.Canceled if err := <-detached; !errors.Is(err, context.Canceled) { t.Errorf("detach = %v, want %v", err, context.Canceled) } + select { + case err := <-forgetDone: + if err != nil { + t.Errorf("Forget: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for Forget to return") + } if st := tr.State(trackedActorUID); !proto.Equal(st, &ateletpb.ActorSnapshotState{}) { t.Errorf("state after forget = %v, want empty", st) } @@ -823,8 +846,122 @@ func TestTrackerRequeuesUntilDetachIsDue(t *testing.T) { } } +// A Detach whose caller leaves mid-upload returns the caller's error, and the +// detach runs on to finish and report. Wait covers it. +func TestTrackerDetachOutlivesCaller(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newIdleTestTracker(t, store) + if err := os.MkdirAll(ateletpath.LocalSnapshotDir(trackedActorUID, "s1"), 0o700); err != nil { + t.Fatal(err) + } + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + + ctx, cancel := context.WithCancel(t.Context()) + first := make(chan error, 1) + go func() { + _, err := tr.Detach(ctx, trackedActorUID) + first <- err + }() + waitStarted(t, store) + done := opDone(t, tr, trackedActorUID) + cancel() + returnsPromptly(t, "Detach after its caller left", func() { + if err := <-first; !errors.Is(err, context.Canceled) { + t.Errorf("first detach = %v, want %v", err, context.Canceled) + } + }) + + // Wait covers the detach, so atelet's shutdown waits for it. + waitCtx, cancelWait := context.WithTimeout(t.Context(), 50*time.Millisecond) + defer cancelWait() + if err := tr.Wait(waitCtx); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("Wait while the detach runs = %v, want %v", err, context.DeadlineExceeded) + } + + store.release <- nil + waitClosed(t, done, "the detach its caller left") + if st := tr.State(trackedActorUID); st.GetLatestLocal() != nil || uuidOf(st.GetLatestDurable()) != "s1" { + t.Errorf("state = %v, want s1 durable and nothing local", st) + } + if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s1")); !errors.Is(err, os.ErrNotExist) { + t.Errorf("s1 after the detach: %v, want it deleted", err) + } + if !reportDue(tr) { + t.Error("no report due after the detach its caller left") + } + if got := store.uploads(); !slices.Equal(got, []string{"s1"}) { + t.Errorf("uploads = %v, want s1 once", got) + } +} + +// A detach that fails after its caller left logs the failure. +func TestTrackerLogsDetachFailingAfterCallerLeft(t *testing.T) { + failures := recordLogLevels(t, "Snapshot operation failed") + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newIdleTestTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + + ctx, cancel := context.WithCancel(t.Context()) + first := make(chan error, 1) + go func() { + _, err := tr.Detach(ctx, trackedActorUID) + first <- err + }() + waitStarted(t, store) + done := opDone(t, tr, trackedActorUID) + cancel() + if err := <-first; !errors.Is(err, context.Canceled) { + t.Fatalf("detach = %v, want %v", err, context.Canceled) + } + store.release <- errors.New("object store unavailable") + waitClosed(t, done, "the detach its caller left") + if got := failures(); !slices.Equal(got, []slog.Level{slog.LevelWarn}) { + t.Errorf("failure logs = %v, want one at %v", got, slog.LevelWarn) + } +} + +// After Wait begins, Detach starts nothing. +func TestTrackerDetachRefusedOnceWaitBegins(t *testing.T) { + store := newFakeSnapshotStore() + tr := newIdleTestTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + if err := tr.Wait(t.Context()); err != nil { + t.Fatalf("Wait: %v", err) + } + if _, err := tr.Detach(t.Context(), trackedActorUID); apierror.Code(err) != codes.Unavailable { + t.Errorf("detach after Wait = %v, want Unavailable", err) + } + if got := store.uploads(); len(got) != 0 { + t.Errorf("uploads = %v, want none", got) + } +} + +// A detach the tracker's shutdown ends fails a live caller with Unavailable. +func TestTrackerDetachStoppedByShutdownIsUnavailable(t *testing.T) { + useActorDir(t) + store := newFakeSnapshotStore() + store.setBlocking(true) + trackerCtx, stop := context.WithCancel(t.Context()) + tr := New(trackerCtx, store, clocktesting.NewFakeClock(time.Unix(1_700_000_000, 0)), + WithUploadDelay(testUploadDelay), WithDetachDelay(testDetachDelay)) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + + detached := make(chan error, 1) + go func() { + _, err := tr.Detach(t.Context(), trackedActorUID) + detached <- err + }() + waitStarted(t, store) + stop() + if err := <-detached; apierror.Code(err) != codes.Unavailable { + t.Errorf("detach during shutdown = %v, want Unavailable", err) + } +} + // cancelingStore is a Store whose upload succeeds just after cancel ends the -// detach's context, as Forget or the Detach caller can. +// detach's context, as Forget or the tracker's shutdown can. type cancelingStore struct{ cancel context.CancelFunc } func (s *cancelingStore) UploadLocalSnapshot(context.Context, string, SnapshotRef) (bool, error) { @@ -832,18 +969,21 @@ func (s *cancelingStore) UploadLocalSnapshot(context.Context, string, SnapshotRe return true, nil } -// A detach whose context ends as its upload succeeds records the upload, but -// deletes no local snapshot, and fails with the context's error. +// A detach whose context ends as its upload succeeds records the upload but +// deletes no local snapshot. The tracker's shutdown ends it here, so the +// caller gets Unavailable. func TestTrackerDetachDoesNotPruneOnceCancelled(t *testing.T) { + useActorDir(t) ctx, cancel := context.WithCancel(t.Context()) - tr := newIdleTestTracker(t, &cancelingStore{cancel: cancel}) + tr := New(ctx, &cancelingStore{cancel: cancel}, clocktesting.NewFakeClock(time.Unix(1_700_000_000, 0)), + WithUploadDelay(testUploadDelay), WithDetachDelay(testDetachDelay)) if err := os.MkdirAll(ateletpath.LocalSnapshotDir(trackedActorUID, "s1"), 0o700); err != nil { t.Fatal(err) } tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) - if _, err := tr.Detach(ctx, trackedActorUID); !errors.Is(err, context.Canceled) { - t.Fatalf("detach = %v, want %v", err, context.Canceled) + if _, err := tr.Detach(t.Context(), trackedActorUID); apierror.Code(err) != codes.Unavailable { + t.Fatalf("detach = %v, want Unavailable", err) } if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s1")); err != nil { t.Errorf("s1 after the cancelled detach: %v, want it kept", err) @@ -894,9 +1034,8 @@ func TestTrackerDetachActsForTheActor(t *testing.T) { }) } -// A background detach whose upload runs past the upload timeout fails, backs -// off, and keeps the local snapshot to retry; a foreground Detach RPC is -// bounded by its caller's context instead. +// A detach whose upload runs past the upload timeout fails and keeps the local +// snapshot to retry, whether a worker or a Detach call runs it. func TestTrackerDetachUploadTimeout(t *testing.T) { t.Run("background", func(t *testing.T) { store := newFakeSnapshotStore() @@ -928,21 +1067,28 @@ func TestTrackerDetachUploadTimeout(t *testing.T) { } }) - t.Run("foreground ignores background upload timeout", func(t *testing.T) { + t.Run("foreground", func(t *testing.T) { store := newFakeSnapshotStore() store.setBlocking(true) - tr := newTestTracker(t, store) + tr := newIdleTestTracker(t, store) tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) - detached := startDetach(t, tr) waitStarted(t, store) - if d := store.lastDeadline(); !d.IsZero() { - t.Errorf("foreground Detach's upload has deadline %v, want none beyond its caller's", d) + if d := store.lastDeadline(); d.IsZero() { + t.Error("foreground Detach's upload has no deadline, want the upload timeout") } - store.release <- nil - if err := <-detached; err != nil { - t.Fatalf("Detach: %v", err) + stepTo(tr, tr.clock.Now().Add(testUploadTimeout)) + select { + case err := <-detached: + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Detach = %v, want %v", err, context.DeadlineExceeded) + } + case <-time.After(10 * time.Second): + t.Fatal("Detach still running past the upload timeout") + } + if got := uuidOf(tr.State(trackedActorUID).GetLatestLocal()); got != "s1" { + t.Errorf("latest_local after timeout = %q, want s1 kept", got) } }) } diff --git a/cmd/atelet/internal/snapshottracker/lifecycle.go b/cmd/atelet/internal/snapshottracker/lifecycle.go index 07c6c6dcd8..6965785486 100644 --- a/cmd/atelet/internal/snapshottracker/lifecycle.go +++ b/cmd/atelet/internal/snapshottracker/lifecycle.go @@ -43,6 +43,7 @@ func (t *Tracker) CommitLocal(actorUID string, snap SnapshotRef) *ateletpb.Actor return t.stateLocked(a) } t.setLatestLocalLocked(a, snap) + a.Started = false if a.UndurableSince.IsZero() && a.uploadNeeded(snap) { a.UndurableSince = snap.CommitTime } @@ -65,12 +66,14 @@ func (t *Tracker) CommitLocal(actorUID string, snap SnapshotRef) *ateletpb.Actor // CommitDurable records snap, just uploaded by a Checkpoint under // UPLOAD_MODE_IMMEDIATE, as the actor's latest durable snapshot, and returns // the actor's state. A commit of a snapshot already durable or older than -// LatestDurable leaves the state as it is. It drops an older LatestLocal, -// which snap supersedes, so the Checkpoint's prune deletes it even if no local -// copy of snap is kept. The Checkpoint then records its local copy with -// CommitLocal, if it manages to keep one. An older snapshot's upload in -// progress runs on, and removes that snapshot's local copy when it ends (see -// removeSupersededLocalCopy). +// LatestDurable leaves the state as it is. +// +// It drops an older LatestLocal, which snap supersedes, so the Checkpoint's +// prune deletes it even if no local copy of snap is kept. The Checkpoint then +// records its local copy with CommitLocal, if it manages to keep one. +// +// Once it records snap, it returns only after any upload of an older snapshot +// has stopped (see durableStateLocked). func (t *Tracker) CommitDurable(actorUID string, snap SnapshotRef) *ateletpb.ActorSnapshotState { t.lock(actorUID) defer t.unlock(actorUID) @@ -84,11 +87,29 @@ func (t *Tracker) CommitDurable(actorUID string, snap SnapshotRef) *ateletpb.Act return t.stateLocked(a) } a.LatestDurable = snap.clone() + a.Started = false if a.LatestLocal != nil && a.LatestLocal.generation() < snap.generation() { a.clearLatestLocal() } t.cancelReportLocked(a) t.durableCommittedLocked(a) + return t.durableStateLocked(a) +} + +// CommittedCheckpointState returns the actor's state for a Checkpoint whose +// snapshot atelet committed already, and cancels any pending background +// report: a Checkpoint returns the state to the control plane, including a +// detach, in either upload mode. +func (t *Tracker) CommittedCheckpointState(actorUID string) *ateletpb.ActorSnapshotState { + t.lock(actorUID) + defer t.unlock(actorUID) + a := t.findLocked(actorUID) + if a == nil { + return &ateletpb.ActorSnapshotState{} + } + if a.reportDue() { + t.clearReportDueLocked(a) + } return t.stateLocked(a) } @@ -115,11 +136,15 @@ func (t *Tracker) LocalSnapshotsToKeep(actorUID string) []string { } // CommitPromoted records snap, a local snapshot PromoteSnapshot made durable or -// found durable already, as the actor's latest durable snapshot, keeping its -// local copy, the time its checkpoint committed, and any UploadTime on snap. -// It leaves DetachAt as it is. A promote of a snapshot already durable, or -// older than LatestDurable, keeps what was recorded. It returns the actor's -// state. +// found durable already, as the actor's latest durable snapshot, and returns +// the actor's state. A promote of a snapshot already durable, or older than +// LatestDurable, keeps what was recorded. +// +// It keeps snap's local copy, the time its checkpoint committed, and any +// UploadTime on snap. It leaves DetachAt as it is. +// +// Once it records snap, it returns only after any upload of an older snapshot +// has stopped (see durableStateLocked). func (t *Tracker) CommitPromoted(actorUID string, snap SnapshotRef) *ateletpb.ActorSnapshotState { t.lock(actorUID) defer t.unlock(actorUID) @@ -149,19 +174,26 @@ func (t *Tracker) CommitPromoted(actorUID string, snap SnapshotRef) *ateletpb.Ac t.cancelReportLocked(a) } t.durableCommittedLocked(a) - return t.stateLocked(a) + return t.durableStateLocked(a) } // BeginStart records that a Run or Restore is starting the actor on this node, // once any background detach of the actor in progress has finished, and // returns the func to call when the start ends, whether or not it succeeded. -// It first unschedules the actor's detach (see markStarted) and saves that, so -// atelet does not detach the actor while a workload restored from its local -// snapshot may use it, even if atelet restarts or the start fails partway. A -// failed Run or Restore leaves the detach unscheduled, since atelet cannot -// tell whether the workload is running on the worker; the local snapshot stays -// until the next checkpoint or Terminate on this node. When it cannot save -// that, it fails with Unavailable, and the caller must not start the actor. +// +// It first records the start (see markStarted) and saves it, so even if +// atelet restarts or the start fails partway: +// - atelet does not detach the actor while a workload restored from its +// local snapshot may use it. +// - A retried Checkpoint does not reset the new activation's directories +// and volumes. +// +// A failed Run or Restore leaves the detach unscheduled, since atelet cannot +// tell whether the workload is running on the worker. The local snapshot stays +// until the next checkpoint or Terminate on this node. +// +// When it cannot save the start, it fails with Unavailable, and the caller +// must not start the actor. func (t *Tracker) BeginStart(ctx context.Context, actorUID string) (end func(), err error) { t.lock(actorUID) defer t.unlock(actorUID) @@ -170,14 +202,14 @@ func (t *Tracker) BeginStart(ctx context.Context, actorUID string) (end func(), return nil, err } if a != nil { - // markStarted clears DetachAt, and reports whether it was set: - // only then is there a change to save. If the save fails, put - // DetachAt and unsaved back as they were, matching the state file, - // so the detach stays scheduled. - if detachAt, unsaved := a.DetachAt, a.unsaved; a.markStarted() { + // markStarted sets Started and clears DetachAt, and reports whether + // either changed: only then is there a change to save. If the save + // fails, put them and unsaved back as they were, matching the state + // file, so the detach stays scheduled. + if detachAt, started, unsaved := a.DetachAt, a.Started, a.unsaved; a.markStarted() { if err := t.writeStateLocked(a); err != nil { - a.DetachAt, a.unsaved = detachAt, unsaved - return nil, apierror.Unavailable("while unscheduling the detach of actor %s before starting it: %v", actorUID, err) + a.DetachAt, a.Started, a.unsaved = detachAt, started, unsaved + return nil, apierror.Unavailable("while recording the start of actor %s: %v", actorUID, err) } } // No detach is scheduled now, so failed detaches have nothing left @@ -189,25 +221,60 @@ func (t *Tracker) BeginStart(ctx context.Context, actorUID string) (end func(), return t.holdLocked(actorUID, a, holdStart), nil } -// BeginCheckpoint records that a Checkpoint of snap is in progress on this -// node, once any background detach in progress has finished, and returns the -// func to call when the checkpoint ends, whether or not it succeeded. It fails -// with Aborted when snap has a stale generation for the actor: below the -// actor's newest snapshot on this node, or equal to it under another -// snapshot_uuid. atelet does not detach the actor in between, so a detach -// cannot delete the local snapshot the checkpoint writes. -func (t *Tracker) BeginCheckpoint(ctx context.Context, actorUID string, snap *ateletpb.Snapshot) (end func(), err error) { +// BeginCheckpoint records that a Checkpoint of snap under mode is in progress +// on this node, once any background detach in progress has finished, and +// returns the func to call when the checkpoint ends, whether or not it +// succeeded. +// +// atelet does not detach the actor in between, so a detach cannot delete the +// local snapshot the checkpoint writes. Under UPLOAD_MODE_IMMEDIATE, atelet +// starts no background upload in between either, since CommitDurable would +// only cancel it; one already running goes on. +// +// It reports whether snap is committed already, as the actor's latest local +// or durable snapshot. The Checkpoint is then a retry of one that committed +// snap and deleted the workload. +// +// It fails with: +// - Aborted when snap has a stale generation for the actor: below the +// actor's newest snapshot on this node, or equal to it under another +// snapshot_uuid. +// - FailedPrecondition when snap is committed under its snapshot_uuid but +// named differently. +func (t *Tracker) BeginCheckpoint(ctx context.Context, actorUID string, snap *ateletpb.Snapshot, mode ateletpb.UploadMode) (committed bool, end func(), err error) { t.lock(actorUID) defer t.unlock(actorUID) a, err := t.waitForDetachLocked(ctx, actorUID, func(a *actorSnapshots) error { return a.checkGeneration(snap) }) if err != nil { - return nil, err + return false, nil, err + } + if a != nil { + if committed, err = a.checkpointCommitted(snap); err != nil { + return false, nil, err + } + } + kind := holdCheckpoint + if mode == ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE { + kind = holdDurable } // Unlike BeginStart, leave DetachAt as it is. Either it is already // zero, because the actor's Run or Restore on this node unscheduled the - // detach, or this is a retry of a Checkpoint that already committed its - // snapshot, whose detach stays scheduled. - return t.holdLocked(actorUID, a, holdCheckpoint), nil + // detach, or snap is committed already, and its detach stays scheduled. + return committed, t.holdLocked(actorUID, a, kind), nil +} + +// NotStartedSince reports whether snap is committed, as LatestLocal or +// LatestDurable, and no Run or Restore of the actor has begun on this node +// since the latest Checkpoint commit (see Started). +func (t *Tracker) NotStartedSince(actorUID string, snap *ateletpb.Snapshot) bool { + t.lock(actorUID) + defer t.unlock(actorUID) + a := t.findLocked(actorUID) + if a == nil || a.Started { + return false + } + committed, err := a.checkpointCommitted(snap) + return err == nil && committed } // BeginPromote records that a PromoteSnapshot of snap is in progress on this @@ -230,25 +297,47 @@ func (t *Tracker) BeginPromote(ctx context.Context, actorUID string, snap Snapsh return false, nil, err } } - return alreadyDurable, t.holdLocked(actorUID, a, holdPromote), nil + return alreadyDurable, t.holdLocked(actorUID, a, holdDurable), nil +} + +// BeginTerminate records that a Terminate of the actor is in progress on this +// node, and returns the func to call when the terminate ends, whether or not it +// succeeded. Until then, atelet starts no background work on the actor; Forget +// cancels an upload or detach already in progress. If the terminate fails +// before Forget, its end lets the actor's background work resume. +func (t *Tracker) BeginTerminate(actorUID string) (end func()) { + t.lock(actorUID) + defer t.unlock(actorUID) + return t.holdLocked(actorUID, t.findLocked(actorUID), holdTerminate) } -// holdLocked records a Run, Restore, Checkpoint, or PromoteSnapshot of the -// actor in progress on its entry a, which holds detaches and state reports of -// the actor off (and background uploads during a promote), and returns the -// func that ends it and queues the actor again. An actor atelet tracks nothing -// of yet (a nil a) is held too, so no detach or report can begin on a snapshot -// its first Checkpoint commits before it ends. +// holdLocked records a Run, Restore, Checkpoint, PromoteSnapshot, or Terminate +// of the actor in progress on its entry a, which holds detaches and state +// reports of the actor off (and background uploads when kind.blocksUploads), +// and returns the func that ends it and queues the actor again, unless Forget +// has dropped the actor meanwhile. An actor atelet tracks nothing of yet (a +// nil a) is held too, so no detach or report can begin on a snapshot its first +// Checkpoint commits before it ends. func (t *Tracker) holdLocked(actorUID string, a *actorSnapshots, kind holdKind) (end func()) { if a == nil { a = &actorSnapshots{actorUID: actorUID} } t.addLocked(a) + prev := a.hold a.hold = kind return func() { t.lock(actorUID) defer t.unlock(actorUID) - a.hold = holdNone + if t.findLocked(actorUID) != a { + // Forget dropped the actor, as a Terminate does while it holds + // the actor. + return + } + a.hold = prev + if a.hold != holdNone { + // The outer Checkpoint's end queues the actor. + return + } if a.reportDeferred { t.queueReportLocked(a) } @@ -256,14 +345,23 @@ func (t *Tracker) holdLocked(actorUID string, a *actorSnapshots, kind holdKind) } } -// Forget drops the actor's state, in memory and on disk. It does not wait for -// the actor's operation in progress, an upload or a detach: it cancels it, and -// the operation finds itself abandoned when it ends. +// Forget drops the actor's state, in memory and on disk. It first cancels any +// background upload or detach of the actor in progress and waits for it to +// end, releasing the actor's lock meanwhile. func (t *Tracker) Forget(actorUID string) error { t.lock(actorUID) defer t.unlock(actorUID) - t.abandonOpLocked(actorUID) - // Start the actor's backoff over; its entry, and any hold, goes below. + if a := t.findLocked(actorUID); a != nil { + t.addLocked(a) + a.hold = holdTerminate + for a.op != nil { + op := a.op + op.cancel() + t.unlock(actorUID) + <-op.done + t.lock(actorUID) + } + } t.retries.Forget(actorUID) t.reporting.retries.Forget(actorUID) t.dropLocked(actorUID) @@ -302,6 +400,24 @@ func (t *Tracker) durableCommittedLocked(a *actorSnapshots) { t.requeueLocked(a) } +// durableStateLocked returns the actor's state for CommitDurable or +// CommitPromoted to return once they record a new LatestDurable. Once the +// control plane learns of it, it may delete the objects of older snapshots, so +// this first cancels any upload of an older snapshot in progress and waits for +// it to end, releasing the actor's lock meanwhile. The object-store plugin may +// still finish an object that upload had under way; deleting the actor +// collects it. +func (t *Tracker) durableStateLocked(a *actorSnapshots) *ateletpb.ActorSnapshotState { + for op := a.supersededUpload(); op != nil; op = a.supersededUpload() { + op.cancel() + t.unlock(a.actorUID) + // The upload ends by updating a, the entry it began on. + <-op.done + t.lock(a.actorUID) + } + return t.stateLocked(a) +} + // succeededLocked finishes a change that leaves the actor in a successful end // state, such as a background upload or detach: it starts the actor's backoff // over, saves the actor's state, and queues the actor for its next background diff --git a/cmd/atelet/internal/snapshottracker/metrics.go b/cmd/atelet/internal/snapshottracker/metrics.go index a2db45a833..736003fb15 100644 --- a/cmd/atelet/internal/snapshottracker/metrics.go +++ b/cmd/atelet/internal/snapshottracker/metrics.go @@ -250,9 +250,10 @@ func outcomeAttrs(ctx context.Context, target SnapshotRef, err error) []attribut } // errorType returns the error.type of a failed upload or detach: Canceled or -// DeadlineExceeded once ctx has ended (when Terminate, an atelet shutdown, or -// the background upload timeout ends it), and otherwise the code an apierror -// or a plugin's gRPC status carries, or Internal. +// DeadlineExceeded once ctx has ended (when Terminate, an atelet shutdown, a +// newer durable snapshot, or the background upload timeout ends it), and +// otherwise the code an apierror or a plugin's gRPC status carries, or +// Internal. func errorType(ctx context.Context, err error) string { if ctxErr := ctx.Err(); ctxErr != nil { return status.FromContextError(ctxErr).Code().String() diff --git a/cmd/atelet/internal/snapshottracker/metrics_test.go b/cmd/atelet/internal/snapshottracker/metrics_test.go index 0accc34b07..3bbc97cbaa 100644 --- a/cmd/atelet/internal/snapshottracker/metrics_test.go +++ b/cmd/atelet/internal/snapshottracker/metrics_test.go @@ -421,20 +421,29 @@ func TestDetachDurationOutcomes(t *testing.T) { t.Run("canceled", func(t *testing.T) { inst, reader := newTestInstruments(t) store := newFakeSnapshotStore() + store.setBlocking(true) + store.setIgnoreCancel(true) useActorDir(t) tr := newStoreTracker(t, store, WithInstruments(inst)) - ref := testRef("s1", 1, tr.clock.Now()) - tr.CommitDurable(trackedActorUID, ref) - tr.CommitLocal(trackedActorUID, ref) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) stepTo(tr, detachDueOf(tr, trackedActorUID)) run := tr.beginDueWork(trackedActorUID, opDetach) if run == nil { t.Fatal("beginDueWork returned nil, want due detach") } - if err := tr.Forget(trackedActorUID); err != nil { - t.Fatalf("Forget: %v", err) - } - run() + go run() + waitStarted(t, store) + // Forget cancels the detach as its upload finishes, and waits for it + // to end. + forgetDone := make(chan error, 1) + go func() { forgetDone <- tr.Forget(trackedActorUID) }() + waitCanceled(t, store) + store.release <- nil + returnsPromptly(t, "Forget", func() { + if err := <-forgetDone; err != nil { + t.Errorf("Forget: %v", err) + } + }) if got, want := errorTypesOf(t, reader, detachDurationMetric), []string{codes.Canceled.String()}; !slices.Equal(got, want) { t.Errorf("error types = %v, want %v", got, want) } diff --git a/cmd/atelet/internal/snapshottracker/queues_test.go b/cmd/atelet/internal/snapshottracker/queues_test.go index e144f64ab8..286ca5e770 100644 --- a/cmd/atelet/internal/snapshottracker/queues_test.go +++ b/cmd/atelet/internal/snapshottracker/queues_test.go @@ -21,6 +21,7 @@ import ( "time" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/internal/proto/ateletpb" ) // Uploads and detaches have queues and workers of their own, so a due detach @@ -183,7 +184,7 @@ func TestTrackerUploadRunsDuringCheckpointHoldWhileDetachWaits(t *testing.T) { t0 := tr.clock.Now() tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) - endCheckpoint, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s2", 2, time.Time{}).Snapshot) + _, endCheckpoint, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s2", 2, time.Time{}).Snapshot, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) if err != nil { t.Fatalf("BeginCheckpoint: %v", err) } diff --git a/cmd/atelet/internal/snapshottracker/report.go b/cmd/atelet/internal/snapshottracker/report.go index 6778908e3f..7831344872 100644 --- a/cmd/atelet/internal/snapshottracker/report.go +++ b/cmd/atelet/internal/snapshottracker/report.go @@ -118,8 +118,8 @@ func (t *Tracker) cancelReportLocked(a *actorSnapshots) { } // queueReportLocked queues the actor's report, if one is due. While a Run, -// Restore, Checkpoint, or PromoteSnapshot holds the actor, it defers the -// report until the hold ends. +// Restore, Checkpoint, PromoteSnapshot, or Terminate holds the actor, it +// defers the report until the hold ends. func (t *Tracker) queueReportLocked(a *actorSnapshots) { if t.reporting.reporter == nil || !a.reportDue() { return @@ -182,7 +182,7 @@ func (t *Tracker) waitReportQueue(delay time.Duration) bool { // returns how long the report queue should pause before the next report. A // report the control plane acknowledges or refuses is no longer due, unless a // background change was made while it ran. Any other failure is retried after -// a backoff, until a commit or detach cancels it or giveUpAfter has passed +// a backoff, until a commit cancels it or giveUpAfter has passed // since the report became due; a failure other than Aborted also backs off // the whole report queue. func (t *Tracker) report(actorUID string) time.Duration { @@ -243,7 +243,7 @@ func jitteredReportDelay(d time.Duration) time.Duration { // beginReport returns the actor, the state to report for it, and when the // report became due, or false when no report is due or a Run, Restore, -// Checkpoint, or PromoteSnapshot of the actor is in progress. +// Checkpoint, PromoteSnapshot, or Terminate of the actor is in progress. func (t *Tracker) beginReport(actorUID string) (resources.ActorRef, *ateletpb.ActorSnapshotState, time.Time, bool) { if !t.locks.Lock(t.ctx, actorUID) { // atelet is shutting down. @@ -256,8 +256,9 @@ func (t *Tracker) beginReport(actorUID string) (resources.ActorRef, *ateletpb.Ac return resources.ActorRef{}, nil, time.Time{}, false } if a.hold != holdNone { - // A Run, Restore, Checkpoint, or PromoteSnapshot of the actor is in - // progress, and queues the report when it ends (see holdLocked). + // A Run, Restore, Checkpoint, PromoteSnapshot, or Terminate of the + // actor is in progress, and queues the report when it ends (see + // holdLocked). a.reportDeferred = true return resources.ActorRef{}, nil, time.Time{}, false } @@ -284,7 +285,7 @@ func (t *Tracker) endReport(actorUID string, sent *ateletpb.ActorSnapshotState) } // retryReport schedules another attempt to send the report that became due at -// dueSince after err, unless a commit or detach canceled it, a newer +// dueSince after err, unless a commit canceled it, a newer // background change replaced it, or giveUpAfter has passed. func (t *Tracker) retryReport(actorUID string, dueSince time.Time, attrs []any, err error) { t.lock(actorUID) diff --git a/cmd/atelet/internal/snapshottracker/report_test.go b/cmd/atelet/internal/snapshottracker/report_test.go index f82b8c3f8a..c8c63e76d5 100644 --- a/cmd/atelet/internal/snapshottracker/report_test.go +++ b/cmd/atelet/internal/snapshottracker/report_test.go @@ -275,17 +275,58 @@ func TestTrackerReportsBackgroundDetach(t *testing.T) { }) } -// A Detach call makes no report due: its caller learns the actor's state from -// the response. -func TestTrackerDoesNotReportDetachCall(t *testing.T) { +// A Detach call makes a report of the detach due, as a background detach does. +func TestTrackerReportsDetachCall(t *testing.T) { store := newFakeSnapshotStore() tr := newReportingTracker(t, store, newFakeReporter()) tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) if _, err := tr.Detach(t.Context(), trackedActorUID); err != nil { t.Fatalf("detach: %v", err) } - if reportDue(tr) || tr.reporting.queue.Len() != 0 { - t.Errorf("report due = %v, queued = %d after a Detach call, want neither", reportDue(tr), tr.reporting.queue.Len()) + if !reportDue(tr) || !reportDueOnDisk() || tr.reporting.queue.Len() != 1 { + t.Errorf("report due = %v, on disk = %v, queued = %d after a Detach call, want due, saved, and queued", + reportDue(tr), reportDueOnDisk(), tr.reporting.queue.Len()) + } +} + +// A Detach call whose upload succeeds and prune fails still reports the upload. +func TestTrackerReportsDetachCallUploadWhenPruneFails(t *testing.T) { + store := newFakeSnapshotStore() + tr := newReportingTracker(t, store, newFakeReporter()) + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + // A file where the local checkpoint directory belongs fails the prune. + if err := os.WriteFile(ateletpath.LocalCheckpointsDir(trackedActorUID), nil, 0o600); err != nil { + t.Fatal(err) + } + if _, err := tr.Detach(t.Context(), trackedActorUID); err == nil { + t.Fatal("detach succeeded, want the prune's error") + } + if st := tr.State(trackedActorUID); uuidOf(st.GetLatestDurable()) != "s1" { + t.Fatalf("state = %v, want s1 durable", st) + } + if !reportDue(tr) || !reportDueOnDisk() { + t.Error("no report due for the upload after the prune failed") + } +} + +// A Detach call with nothing left to detach keeps a report already due. +func TestTrackerDetachCallKeepsReportDue(t *testing.T) { + store := newFakeSnapshotStore() + tr := newReportingTracker(t, store, newFakeReporter()) + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + detachNow(t, tr) + if !reportDue(tr) { + t.Fatal("report not due after the background detach") + } + st, err := tr.Detach(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("detach: %v", err) + } + if st.GetLatestLocal() != nil || uuidOf(st.GetLatestDurable()) != "s1" { + t.Errorf("state = %v, want s1 durable and nothing local", st) + } + if !reportDue(tr) || !reportDueOnDisk() { + t.Error("report no longer due after the Detach call") } } @@ -295,9 +336,12 @@ func TestTrackerDoesNotReportWhenNothingChanged(t *testing.T) { for _, tc := range []struct { name string snap SnapshotRef + // cancels is set when snap is newer than the upload's snapshot, so + // CommitDurable cancels the upload and waits for it to stop. + cancels bool }{ - {"newer durable snapshot", committedRef("s2", 2, time.Time{})}, - {"same generation already durable", committedRef("s1", 1, time.Time{})}, + {"newer durable snapshot", committedRef("s2", 2, time.Time{}), true}, + {"same generation already durable", committedRef("s1", 1, time.Time{}), false}, } { t.Run("upload overtaken by "+tc.name, func(t *testing.T) { store := newFakeSnapshotStore() @@ -311,7 +355,9 @@ func TestTrackerDoesNotReportWhenNothingChanged(t *testing.T) { durable.UploadTime = durable.CommitTime.Add(time.Second) tr.CommitDurable(trackedActorUID, durable) - store.release <- nil + if !tc.cancels { + store.release <- nil + } waitClosed(t, done, "the upload of s1") if reportDue(tr) || tr.reporting.queue.Len() != 0 { t.Errorf("report due = %v, queued = %d after an overtaken upload, want neither", reportDue(tr), tr.reporting.queue.Len()) @@ -911,20 +957,16 @@ func TestTrackerCancelsReportOnForegroundCall(t *testing.T) { }, }, { - name: "Detach with local snapshot", - call: func(t *testing.T, tr *Tracker) { - if _, err := tr.Detach(t.Context(), trackedActorUID); err != nil { - t.Fatalf("Detach: %v", err) - } + name: "CommittedCheckpointState with local snapshot", + call: func(_ *testing.T, tr *Tracker) { + tr.CommittedCheckpointState(trackedActorUID) }, }, { - name: "Detach when already detached", + name: "CommittedCheckpointState when already detached", alreadyDetached: true, - call: func(t *testing.T, tr *Tracker) { - if _, err := tr.Detach(t.Context(), trackedActorUID); err != nil { - t.Fatalf("Detach: %v", err) - } + call: func(_ *testing.T, tr *Tracker) { + tr.CommittedCheckpointState(trackedActorUID) }, }, } { @@ -1112,10 +1154,11 @@ func TestTrackerReportStaysDueAtShutdown(t *testing.T) { } } -// A report waits while a Run, Restore, Checkpoint, or PromoteSnapshot holds -// the actor, since the control plane holds the actor's lease throughout that -// call and would refuse the report with Aborted. Once the hold ends without a -// commit canceling the report, the report is queued. +// A report waits while a Run, Restore, Checkpoint, PromoteSnapshot, or +// Terminate holds the actor, since the control plane holds the actor's lease +// throughout that call and would refuse the report with Aborted. Once the hold +// ends without a commit canceling the report, or a Terminate forgetting the +// actor, the report is queued. func TestTrackerDefersReportDuringHold(t *testing.T) { holds := []struct { name string @@ -1134,7 +1177,17 @@ func TestTrackerDefersReportDuringHold(t *testing.T) { { name: "BeginCheckpoint", begin: func(t *testing.T, tr *Tracker) func() { - end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, committedRef("s2", 2, tr.clock.Now()).Snapshot) + _, end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, committedRef("s2", 2, tr.clock.Now()).Snapshot, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + if err != nil { + t.Fatalf("BeginCheckpoint: %v", err) + } + return end + }, + }, + { + name: "BeginCheckpoint immediate", + begin: func(t *testing.T, tr *Tracker) func() { + _, end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, committedRef("s2", 2, tr.clock.Now()).Snapshot, ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE) if err != nil { t.Fatalf("BeginCheckpoint: %v", err) } @@ -1153,6 +1206,13 @@ func TestTrackerDefersReportDuringHold(t *testing.T) { return end }, }, + { + // A Terminate that fails before Forget. + name: "BeginTerminate", + begin: func(t *testing.T, tr *Tracker) func() { + return tr.BeginTerminate(trackedActorUID) + }, + }, } for _, h := range holds { @@ -1224,7 +1284,7 @@ func TestTrackerDefersReportDuringHold(t *testing.T) { uploadNow(t, tr, store, nil) s2 := committedRef("s2", 2, tr.clock.Now()) - end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, s2.Snapshot) + _, end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, s2.Snapshot, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) if err != nil { t.Fatalf("BeginCheckpoint: %v", err) } diff --git a/cmd/atelet/internal/snapshottracker/tracker.go b/cmd/atelet/internal/snapshottracker/tracker.go index 3bf91e6291..1109db0046 100644 --- a/cmd/atelet/internal/snapshottracker/tracker.go +++ b/cmd/atelet/internal/snapshottracker/tracker.go @@ -83,31 +83,61 @@ type Store interface { // does not detach an actor whose start has begun, even after a restart or // a failed start; only the actor's next checkpoint schedules a detach // again. A background detach that comes due while a Run, Restore, -// Checkpoint, or PromoteSnapshot is in progress (BeginStart, -// BeginCheckpoint, or BeginPromote) waits for it to end, which queues the -// actor again. +// Checkpoint, PromoteSnapshot, or Terminate is in progress (BeginStart, +// BeginCheckpoint, BeginPromote, or BeginTerminate) waits for it to end, +// which queues the actor again unless Terminate forgot it. // 7. A background upload that records a new LatestDurable, or a background // detach that records one or clears LatestLocal, makes a report of the // actor's state due (ReportDueSince), saved with the change so that it -// survives a restart. A Checkpoint, a PromoteSnapshot while a local -// snapshot is kept, or a Detach call returns the latest state to the -// control plane, so it cancels any pending report. Report workers, apart -// from the upload and detach workers, spread saved reports across a +// survives a restart. A Detach call does too, so the control plane records +// the detach even if the caller leaves or fails to. A Checkpoint or a +// PromoteSnapshot while a local snapshot is kept returns the latest state +// to the control plane, so it cancels any pending report. Report workers, +// apart from the upload and detach workers, spread saved reports across a // startup window, send the actor's whole state through the Reporter, and // retry with per-actor and queue-wide backoff until the control plane // acknowledges or refuses it, a call cancels it, or reporting.giveUpAfter -// passes. A report waits while a Run, Restore, Checkpoint, or -// PromoteSnapshot is in progress, and a background change made while a -// report ran is reported again. +// passes. A report waits while a Run, Restore, Checkpoint, +// PromoteSnapshot, or Terminate is in progress, and a background change +// made while a report ran is reported again. // -// Detach runs in the calling goroutine rather than on a worker, so an explicit -// control-plane call does not wait behind other actors' background work. It -// does wait for the actor's own background operation in progress, even an -// upload of an older snapshot; Run, Restore, and Checkpoint wait for a -// background detach in progress; and PromoteSnapshot waits for a background -// detach or a background upload of the same snapshot. A worker leaves an actor -// with an operation in progress to that operation's end, which queues the -// actor again. +// Detach runs under the tracker's context rather than on a worker, so an +// explicit control-plane call does not wait behind other actors' background +// work, and its detach finishes even if the caller leaves. +// +// The first table shows how each RPC on an actor treats the actor's upload or +// detach in progress, and its report due. The RPC waits for the operation to +// end as it begins (BeginStart, BeginCheckpoint, BeginPromote, or Detach), +// cancels it and waits for it to stop, or leaves it alone (-). A Checkpoint or +// PromoteSnapshot cancels once it commits its snapshot, and Terminate cancels +// in Forget, once the workload is gone. A canceled report is neither sent nor +// retried, though an attempt under way finishes. +// +// RPC upload in progress detach in progress report due +// Run, Restore - waits - +// deferred Checkpoint - waits cancels +// immediate Checkpoint cancels waits cancels +// PromoteSnapshot waits if the same waits cancels if a +// snapshot, else local snapshot +// cancels is kept +// DetachActor waits waits keeps, and its +// detach makes +// one due +// Terminate cancels cancels cancels +// +// The second table shows whether an upload, detach, or report of the actor +// that comes due while an RPC or background work of the actor runs begins, or +// waits for it to end. An upload or detach is the actor's one operation in +// progress, whether a worker or DetachActor runs it. +// +// running upload due detach due report due +// Run, Restore begins waits waits +// deferred Checkpoint begins waits waits +// immediate Checkpoint waits waits waits +// PromoteSnapshot waits waits waits +// Terminate waits waits waits +// upload or detach waits waits begins +// report begins begins waits type Tracker struct { // ctx bounds background work, and shuts the queues down when it ends. ctx context.Context @@ -124,18 +154,18 @@ type Tracker struct { // this node before atelet detaches it: DefaultDetachDelay unless // WithDetachDelay sets it. detachDelay time.Duration - // uploadTimeout is how long one background upload attempt may run, on + // uploadTimeout is how long one upload attempt may run, on // top of the time its snapshot takes at uploadEstThroughput (see // uploadTimeoutFor), before it is canceled and retried: // DefaultUploadTimeout unless WithUploadTimeout sets it. Zero or less // turns the timeout off. uploadTimeout time.Duration // uploadEstThroughput, in bytes per second of a snapshot's allocated - // local size, is how fast atelet expects a background upload to run (see + // local size, is how fast atelet expects an upload to run (see // uploadTimeoutFor): DefaultUploadEstThroughput unless // WithUploadEstThroughput sets it. uploadEstThroughput int64 - // withTimeout derives the context a background upload attempt runs under: + // withTimeout derives the context an upload attempt runs under: // context.WithTimeout, unless a test on a fake clock replaces it with one // that times out on that clock. withTimeout func(context.Context, time.Duration) (context.Context, context.CancelFunc) @@ -162,8 +192,14 @@ type Tracker struct { // reporting holds the queue and settings for reporting snapshot state // changes to the control plane. reporting reportQueue - // workersDone counts the workers still running. + // workersDone counts the workers, and the detaches Detach calls started, + // still running. workersDone sync.WaitGroup + // foregroundMu guards waiting, and Detach holds it while adding to + // workersDone. + foregroundMu sync.Mutex + // waiting reports that Wait has begun. + waiting bool // locks holds each actor's state lock. Short critical sections on an // actor, including its snapshot state file I/O, hold that actor's lock, so @@ -175,7 +211,7 @@ type Tracker struct { // actors maps actorUID to the entries requeueLocked keeps in memory (see // requeueLocked). requeueLocked drops any other entry, and findLocked then // reads the state from the file. Read and change it through findLocked, - // findOrAddLocked, addLocked, dropLocked, and heldLocked. + // findOrAddLocked, addLocked, dropLocked, and localInUseLocked. actors map[string]*actorSnapshots } @@ -198,7 +234,7 @@ func WithDetachDelay(d time.Duration) Option { } } -// WithUploadTimeout sets how long one background upload attempt may run, on +// WithUploadTimeout sets how long one upload attempt may run, on // top of the time its snapshot takes at the estimated upload throughput (see // WithUploadEstThroughput), before atelet cancels and retries it. Each // consecutive failure doubles the next attempt's timeout, to at most 8 times @@ -357,14 +393,14 @@ func (t *Tracker) dropLocked(actorUID string) { delete(t.actors, actorUID) } -// heldLocked reports whether a Run, Restore, Checkpoint, or PromoteSnapshot of -// the actor is in progress. An entry with one stays in memory, so this never -// reads the snapshot state file. -func (t *Tracker) heldLocked(actorUID string) bool { +// localInUseLocked reports whether a Run or Restore of the actor in progress +// may use a superseded local snapshot (see actorSnapshots.localInUse). An +// entry with one stays in memory, so this never reads the snapshot state file. +func (t *Tracker) localInUseLocked(actorUID string) bool { t.mu.Lock() defer t.mu.Unlock() a := t.actors[actorUID] - return a != nil && a.hold != holdNone + return a != nil && a.localInUse() } // uploadDueLocked returns when the actor's background upload is due, as diff --git a/cmd/atelet/internal/snapshottracker/tracker_test.go b/cmd/atelet/internal/snapshottracker/tracker_test.go index cb44505c7d..30114f1e20 100644 --- a/cmd/atelet/internal/snapshottracker/tracker_test.go +++ b/cmd/atelet/internal/snapshottracker/tracker_test.go @@ -47,10 +47,11 @@ const trackedActorUID = "actor-uid-1" func useTempNodeDirs(t *testing.T) { t.Helper() root := t.TempDir() - origActors := nodepath.ActorsDir + origActors, origSnapshots := nodepath.ActorsDir, nodepath.SnapshotsDir nodepath.ActorsDir = filepath.Join(root, "actors") + nodepath.SnapshotsDir = filepath.Join(root, "snapshots") t.Cleanup(func() { - nodepath.ActorsDir = origActors + nodepath.ActorsDir, nodepath.SnapshotsDir = origActors, origSnapshots }) } @@ -121,22 +122,31 @@ const ( type fakeSnapshotStore struct { // started receives each call's snapshot uuid as the call starts. started chan string + // canceled receives each call's snapshot uuid once its ctx ends while + // ignoreCancel is set. + canceled chan string // release ends a blocked call with the error it receives. release chan error - mu sync.Mutex - calls []string - actors map[string]resources.ActorRef - uploaded map[string]bool - blocking bool - err error + mu sync.Mutex + calls []string + actors map[string]resources.ActorRef + uploaded map[string]bool + blocking bool + ignoreCancel bool + err error // deadline is the deadline of the latest call's context: zero if it has // none. deadline time.Time } func newFakeSnapshotStore() *fakeSnapshotStore { - return &fakeSnapshotStore{started: make(chan string, 100), release: make(chan error), actors: map[string]resources.ActorRef{}} + return &fakeSnapshotStore{ + started: make(chan string, 100), + canceled: make(chan string, 100), + release: make(chan error), + actors: map[string]resources.ActorRef{}, + } } // setBlocking makes later calls wait for release, or their context. @@ -146,6 +156,14 @@ func (f *fakeSnapshotStore) setBlocking(blocking bool) { f.blocking = blocking } +// setIgnoreCancel makes blocked calls wait for release even after their +// context ends, sending their snapshot uuid on canceled first. +func (f *fakeSnapshotStore) setIgnoreCancel(ignore bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.ignoreCancel = ignore +} + // setErr makes later calls fail with err, or succeed when it is nil. func (f *fakeSnapshotStore) setErr(err error) { f.mu.Lock() @@ -179,14 +197,18 @@ func (f *fakeSnapshotStore) UploadLocalSnapshot(ctx context.Context, _ string, r f.calls = append(f.calls, ref.uuid()) f.actors[ref.uuid()] = ref.Actor f.deadline, _ = ctx.Deadline() - blocking, err := f.blocking, f.err + blocking, ignoreCancel, err := f.blocking, f.ignoreCancel, f.err f.mu.Unlock() f.started <- ref.uuid() if blocking { select { case err = <-f.release: case <-ctx.Done(): - return false, ctx.Err() + if !ignoreCancel { + return false, ctx.Err() + } + f.canceled <- ref.uuid() + err = <-f.release } } if err != nil { @@ -325,6 +347,20 @@ func waitStarted(t *testing.T, store *fakeSnapshotStore) string { } } +// waitCanceled waits for the next blocked store call with ignoreCancel set to +// see its context canceled, and returns the uuid of the snapshot it was +// uploading. +func waitCanceled(t *testing.T, store *fakeSnapshotStore) string { + t.Helper() + select { + case uuid := <-store.canceled: + return uuid + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for an upload to be canceled") + return "" + } +} + // opDone returns a channel closed once the actor's operation in progress has // ended, deleting the local snapshots it no longer needs included. func opDone(t *testing.T, tr *Tracker, actorUID string) <-chan struct{} { @@ -531,7 +567,7 @@ func TestTrackerCommitPromoted(t *testing.T) { } } -// A promote of a paused actor's local snapshot leaves its detach schedule as +// A promote of an actor's local snapshot leaves its detach schedule as // it is: the detach its checkpoint scheduled, or none for an actor whose Run // or Restore has begun since the snapshot was taken. func TestTrackerCommitPromotedKeepsDetachSchedule(t *testing.T) { @@ -753,14 +789,16 @@ func TestTrackerCheckGeneration(t *testing.T) { } checkBegin := func(t *testing.T, tr *Tracker, r SnapshotRef, wantCode codes.Code) { t.Helper() - end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, r.Snapshot) - if err == nil { - end() - } - if got := apierror.Code(err); got != wantCode { - t.Errorf("BeginCheckpoint(%v) code = %v, want %v", r.Snapshot, got, wantCode) + for _, mode := range []ateletpb.UploadMode{ateletpb.UploadMode_UPLOAD_MODE_DEFERRED, ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE} { + _, end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, r.Snapshot, mode) + if err == nil { + end() + } + if got := apierror.Code(err); got != wantCode { + t.Errorf("BeginCheckpoint(%v, %v) code = %v, want %v", r.Snapshot, mode, got, wantCode) + } } - _, end, err = tr.BeginPromote(t.Context(), trackedActorUID, r) + _, end, err := tr.BeginPromote(t.Context(), trackedActorUID, r) if err == nil { end() } @@ -812,6 +850,7 @@ func TestTrackerCheckGeneration(t *testing.T) { tr.CommitLocal(trackedActorUID, ref("s3", 6)) checkBegin(t, tr, ref("s2", 4), codes.Aborted) checkBegin(t, tr, ref("s3", 6), codes.OK) + // If durable has a higher generation than local, durable is newest. a := &actorSnapshots{ actorUID: trackedActorUID, @@ -855,6 +894,148 @@ func TestTrackerCheckGeneration(t *testing.T) { } } +// BeginCheckpoint reports whether its snapshot is committed already, as +// LatestLocal or LatestDurable, and refuses a snapshot committed under the +// same UUID but named differently. +func TestTrackerBeginCheckpointReportsCommitted(t *testing.T) { + s1 := testRef("s1", 1, time.Time{}) + commitLocal := func(tr *Tracker) { tr.CommitLocal(trackedActorUID, s1) } + commitDurable := func(tr *Tracker) { tr.CommitDurable(trackedActorUID, s1) } + renamed := func(rename func(*ateletpb.Snapshot)) *ateletpb.Snapshot { + snap := proto.CloneOf(s1.Snapshot) + rename(snap) + return snap + } + for _, tc := range []struct { + name string + commit func(*Tracker) + snap *ateletpb.Snapshot + wantCommitted bool + wantCode codes.Code + }{ + {name: "not committed", snap: s1.Snapshot}, + {name: "committed locally", commit: commitLocal, snap: s1.Snapshot, wantCommitted: true}, + {name: "durable only", commit: commitDurable, snap: s1.Snapshot, wantCommitted: true}, + { + name: "committed with another generation", + commit: commitDurable, + snap: renamed(func(s *ateletpb.Snapshot) { s.Generation = 2 }), + wantCode: codes.FailedPrecondition, + }, + { + name: "committed with another fidelity", + commit: commitDurable, + snap: renamed(func(s *ateletpb.Snapshot) { s.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES }), + wantCode: codes.FailedPrecondition, + }, + { + name: "committed with another uri", + commit: commitDurable, + snap: renamed(func(s *ateletpb.Snapshot) { s.Object.SnapshotUri += "-other" }), + wantCode: codes.FailedPrecondition, + }, + } { + t.Run(tc.name, func(t *testing.T) { + useActorDir(t) + tr := newTracker(t) + if tc.commit != nil { + tc.commit(tr) + } + committed, end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, tc.snap, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + if err == nil { + end() + } + if committed != tc.wantCommitted || apierror.Code(err) != tc.wantCode { + t.Errorf("BeginCheckpoint = (%v, %v), want committed = %v with code %v", committed, err, tc.wantCommitted, tc.wantCode) + } + }) + } +} + +// NotStartedSince holds for a committed snapshot, local or durable only, until +// a Run or Restore of the actor begins, and a fresh tracker reads the same +// from the state file. +func TestTrackerNotStartedSince(t *testing.T) { + type step func(*testing.T, *Tracker) + s1 := testRef("s1", 1, time.Time{}) + s2 := testRef("s2", 2, time.Time{}) + commitLocal := func(r SnapshotRef) step { + return func(_ *testing.T, tr *Tracker) { tr.CommitLocal(trackedActorUID, r) } + } + commitDurable := func(r SnapshotRef) step { + return func(_ *testing.T, tr *Tracker) { tr.CommitDurable(trackedActorUID, r) } + } + start := func(t *testing.T, tr *Tracker) { + end, err := tr.BeginStart(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("BeginStart: %v", err) + } + end() + } + for _, tc := range []struct { + name string + steps []step + snap *ateletpb.Snapshot + want bool + }{ + {name: "not committed", snap: s1.Snapshot}, + {name: "committed locally", steps: []step{commitLocal(s1)}, snap: s1.Snapshot, want: true}, + {name: "another snapshot", steps: []step{commitLocal(s1)}, snap: testRef("other", 1, time.Time{}).Snapshot}, + {name: "durable only", steps: []step{commitDurable(s1)}, snap: s1.Snapshot, want: true}, + {name: "started since a local commit", steps: []step{commitLocal(s1), start}, snap: s1.Snapshot}, + {name: "started since a durable commit", steps: []step{commitDurable(s1), start}, snap: s1.Snapshot}, + {name: "committed locally since a start", steps: []step{commitLocal(s1), start, commitLocal(s2)}, snap: s2.Snapshot, want: true}, + {name: "committed durably since a start", steps: []step{commitLocal(s1), start, commitDurable(s2)}, snap: s2.Snapshot, want: true}, + } { + t.Run(tc.name, func(t *testing.T) { + useActorDir(t) + tr := newTracker(t) + for _, run := range tc.steps { + run(t, tr) + } + if got := tr.NotStartedSince(trackedActorUID, tc.snap); got != tc.want { + t.Errorf("NotStartedSince = %v, want %v", got, tc.want) + } + if got := newTracker(t).NotStartedSince(trackedActorUID, tc.snap); got != tc.want { + t.Errorf("NotStartedSince on a fresh tracker = %v, want %v from the state file", got, tc.want) + } + }) + } +} + +// When a retried immediate Checkpoint promotes inside its checkpoint hold, +// ending the promote restores the checkpoint hold so detaches stay blocked +// until the checkpoint ends. +func TestTrackerPromoteInsideCheckpointKeepsHold(t *testing.T) { + store := newFakeSnapshotStore() + tr := newIdleTestTracker(t, store) + s1 := testRef("s1", 1, tr.clock.Now()) + tr.CommitLocal(trackedActorUID, s1) + + _, endCheckpoint, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, s1.Snapshot, ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE) + if err != nil { + t.Fatalf("BeginCheckpoint: %v", err) + } + _, endPromote, err := tr.BeginPromote(t.Context(), trackedActorUID, s1) + if err != nil { + t.Fatalf("BeginPromote: %v", err) + } + tr.CommitPromoted(trackedActorUID, s1) + endPromote() + + stepTo(tr, detachDueOf(tr, trackedActorUID)) + if run := tr.beginDueWork(trackedActorUID, opDetach); run != nil { + t.Error("a due detach began after the inner endPromote, want it held off while the Checkpoint holds the actor") + run() + } + endCheckpoint() + run := tr.beginDueWork(trackedActorUID, opDetach) + if run == nil { + t.Fatal("no detach began after endCheckpoint, want the due detach") + } + run() +} + // CommitLocal and CommitDurable skip older generations, and an older // CommitDurable leaves a newer LatestLocal's upload schedule and backoff intact. func TestTrackerCommitsSkipOlderGenerations(t *testing.T) { diff --git a/cmd/atelet/internal/snapshottracker/upload.go b/cmd/atelet/internal/snapshottracker/upload.go index 3fad1c546b..93bdaefadc 100644 --- a/cmd/atelet/internal/snapshottracker/upload.go +++ b/cmd/atelet/internal/snapshottracker/upload.go @@ -25,7 +25,7 @@ import ( // under a context derived from ctx, and returns the func that runs it once the // caller releases the actor's lock. Any superseded snapshot in a.Uploading is // replaced in the same state-file save, and its local copy removed (see -// removeSupersededLocalCopy) before the upload runs. +// removeSupersededLocalCopyLocked). func (t *Tracker) beginUploadLocked(ctx context.Context, a *actorSnapshots) func() { superseded, _ := a.clearSupersededUploading() target := *a.nextUploadTarget() @@ -37,40 +37,37 @@ func (t *Tracker) beginUploadLocked(ctx context.Context, a *actorSnapshots) func if !wasUploading { t.saveLocked(a) } + if superseded != nil { + t.removeSupersededLocalCopyLocked(a.actorUID, *superseded) + } actorUID := a.actorUID return func() { - if superseded != nil { - t.removeSupersededLocalCopy(actorUID, *superseded) - } t.backgroundUpload(opCtx, actorUID, op) } } // cleanupSupersededUploadingLocked clears a.Uploading when no upload is // running and a newer commit has superseded it, so backgroundUpload will never -// run for it. It saves the change and returns a func that removes the local -// copy kept for that upload (see removeSupersededLocalCopy), or nil. -func (t *Tracker) cleanupSupersededUploadingLocked(a *actorSnapshots) func() { +// run for it. It saves the change and removes the local copy kept for that +// upload (see removeSupersededLocalCopyLocked) without releasing the actor's +// lock, so no Terminate can begin in between. +func (t *Tracker) cleanupSupersededUploadingLocked(a *actorSnapshots) { superseded, cleared := a.clearSupersededUploading() if !cleared { - return nil + return } t.saveLocked(a) - if superseded == nil { - return nil - } - actorUID := a.actorUID - return func() { - t.removeSupersededLocalCopy(actorUID, *superseded) + if superseded != nil { + t.removeSupersededLocalCopyLocked(a.actorUID, *superseded) } } -// uploadSnapshot uploads target from its local copy to object storage and, -// when background is true, bounds the attempt by uploadTimeoutFor, unless -// uploadTimeout is zero or less, and records the background upload metric. +// uploadSnapshot uploads target from its local copy to object storage, bounded +// by uploadTimeoutFor unless uploadTimeout is zero or less. When background is +// true, it records the background upload metric. func (t *Tracker) uploadSnapshot(ctx context.Context, actorUID string, target SnapshotRef, background bool) (uploaded bool, elapsed time.Duration, err error) { uploadCtx := ctx - if background && t.uploadTimeout > 0 { + if t.uploadTimeout > 0 { var cancel context.CancelFunc uploadCtx, cancel = t.withTimeout(ctx, t.uploadTimeoutFor(ctx, actorUID, target)) defer cancel() @@ -84,7 +81,7 @@ func (t *Tracker) uploadSnapshot(ctx context.Context, actorUID string, target Sn return uploaded, elapsed, err } -// uploadTimeoutFor returns how long a background upload attempt of target may +// uploadTimeoutFor returns how long an upload attempt of target may // run: uploadTimeout, plus the time target's local copy takes to upload at // uploadEstThroughput, doubled for each consecutive failure of the actor's // background work, up to maxUploadTimeoutDoublings times. A copy that is @@ -118,9 +115,8 @@ func uploadAllowance(size, bytesPerSecond int64) time.Duration { // backgroundUpload runs op, a background upload of the actor's nextUploadTarget // when it began, on a worker. Once the upload succeeds or a newer snapshot // replaces target, it removes target's local copy if LatestLocal moved on (see -// removeSupersededLocalCopy); when the upload fails and LatestLocal cannot -// replace target, it keeps target in Uploading to retry. An upload Forget -// abandoned leaves the actor alone. +// removeSupersededLocalCopyLocked); when the upload fails and LatestLocal +// cannot replace target, it keeps target in Uploading to retry. func (t *Tracker) backgroundUpload(ctx context.Context, actorUID string, op *actorOp) { defer close(op.done) target := op.target @@ -129,10 +125,6 @@ func (t *Tracker) backgroundUpload(ctx context.Context, actorUID string, op *act t.lock(actorUID) a := t.endOpLocked(actorUID, op) - if a == nil { - t.unlock(actorUID) - return - } removeOld := false switch { case err == nil: @@ -166,10 +158,10 @@ func (t *Tracker) backgroundUpload(ctx context.Context, actorUID string, op *act } t.requeueLocked(a) } - t.unlock(actorUID) if removeOld { - t.removeSupersededLocalCopy(actorUID, target) + t.removeSupersededLocalCopyLocked(actorUID, target) } + t.unlock(actorUID) } // recordUploadLocked records that an upload of target succeeded, and reports @@ -189,17 +181,14 @@ func (t *Tracker) recordUploadLocked(a *actorSnapshots, target SnapshotRef, uplo return true } -// removeSupersededLocalCopy deletes the local copy of target, an upload's +// removeSupersededLocalCopyLocked deletes the local copy of target, an upload's // snapshot that a newer snapshot superseded, once no upload needs it: the copy -// was kept only for the upload. It leaves the copy while a Run, Restore, -// Checkpoint, or PromoteSnapshot of the actor is in progress, since a Restore -// or PromoteSnapshot may be reading it, and deletes it under the actor's lock +// was kept only for the upload. It leaves the copy while a Run or Restore in +// progress may use it (see localInUse), and deletes it under the actor's lock // so that none begins reading it meanwhile. Best-effort: the actor's next // checkpoint, detach, or terminate prunes a copy left behind. -func (t *Tracker) removeSupersededLocalCopy(actorUID string, target SnapshotRef) { - t.lock(actorUID) - defer t.unlock(actorUID) - if t.heldLocked(actorUID) { +func (t *Tracker) removeSupersededLocalCopyLocked(actorUID string, target SnapshotRef) { + if t.localInUseLocked(actorUID) { return } if err := removeLocalCheckpoint(t.ctx, actorUID, target.uuid()); err != nil { diff --git a/cmd/atelet/internal/snapshottracker/upload_test.go b/cmd/atelet/internal/snapshottracker/upload_test.go index 698993091f..863dd8dcfd 100644 --- a/cmd/atelet/internal/snapshottracker/upload_test.go +++ b/cmd/atelet/internal/snapshottracker/upload_test.go @@ -150,54 +150,73 @@ func TestTrackerCheckpointDuringUpload(t *testing.T) { } } -// An immediate checkpoint makes the actor's newest snapshot durable, which -// drops the older local snapshot, then records its own local copy. An older -// snapshot's upload in progress runs on: LocalSnapshotsToKeep keeps that -// snapshot, and the upload deletes it when it ends without holding the actor -// back when it fails, though the new local copy has a detach scheduled. +// An immediate checkpoint makes the actor's newest snapshot durable, drops the +// older local snapshot, cancels any older upload in progress, and waits for it +// to stop before returning. The upload then deletes the older snapshot's local +// copy, though the checkpoint holds the actor. func TestTrackerImmediateCheckpointDuringUpload(t *testing.T) { for _, tc := range []struct { name string uploadErr error }{ - {name: "older upload succeeds"}, - {name: "older upload fails", uploadErr: errors.New("boom")}, + {name: "older upload canceled", uploadErr: context.Canceled}, + {name: "older upload finished before cancel", uploadErr: nil}, + {name: "older upload failed", uploadErr: errors.New("boom")}, } { t.Run(tc.name, func(t *testing.T) { store := newFakeSnapshotStore() store.setBlocking(true) + store.setIgnoreCancel(true) tr := newTestTracker(t, store) if err := os.MkdirAll(ateletpath.LocalSnapshotDir(trackedActorUID, "s1"), 0o700); err != nil { t.Fatal(err) } t0 := tr.clock.Now() tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) - done := startUpload(t, tr, store) + startUpload(t, tr, store) s2 := testRef("s2", 2, t0.Add(time.Minute)) + _, endCheckpoint, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, s2.Snapshot, ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE) + if err != nil { + t.Fatalf("BeginCheckpoint: %v", err) + } s2.UploadTime = s2.CommitTime.Add(time.Second) - st := tr.CommitDurable(trackedActorUID, s2) - if st.GetLatestLocal() != nil || st.GetUploadDueTime() != nil || uuidOf(st.GetUploading()) != "s1" { - t.Errorf("state = %v, want nothing local or due, and s1 uploading", st) + stateCh := make(chan *ateletpb.ActorSnapshotState, 1) + go func() { + stateCh <- tr.CommitDurable(trackedActorUID, s2) + }() + if got := waitCanceled(t, store); got != "s1" { + t.Fatalf("canceled upload = %q, want s1", got) + } + select { + case st := <-stateCh: + t.Fatalf("CommitDurable returned %v before the older upload stopped", st) + case <-time.After(20 * time.Millisecond): + } + + store.release <- tc.uploadErr + var st *ateletpb.ActorSnapshotState + select { + case st = <-stateCh: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for CommitDurable to return") + } + if st.GetLatestLocal() != nil || st.GetUploadDueTime() != nil || st.GetUploading() != nil { + t.Errorf("state = %v, want nothing local, due, or uploading", st) } - if keep := tr.LocalSnapshotsToKeep(trackedActorUID); !slices.Equal(keep, []string{"s1"}) { - t.Errorf("LocalSnapshotsToKeep = %v, want [s1]: s1 is being uploaded", keep) + if keep := tr.LocalSnapshotsToKeep(trackedActorUID); len(keep) != 0 { + t.Errorf("LocalSnapshotsToKeep = %v, want none", keep) } if got := st.GetLatestDurable(); uuidOf(got) != "s2" || !timeOf(got.GetDurableCompleteTime()).Equal(s2.UploadTime) { t.Errorf("latest_durable = %v, want s2 uploaded at %v", got, s2.UploadTime) } - tr.CommitLocal(trackedActorUID, testRef("s2", 2, s2.CommitTime)) - - store.release <- tc.uploadErr - waitClosed(t, done, "the upload of s1") if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s1")); !os.IsNotExist(err) { - t.Errorf("s1 after its upload: stat = %v, want it deleted", err) - } - if got := uuidOf(tr.State(trackedActorUID).GetLatestDurable()); got != "s2" { - t.Errorf("latest_durable = %q after the upload of s1, want s2", got) + t.Errorf("s1 once CommitDurable returned: stat = %v, want it deleted", err) } + tr.CommitLocal(trackedActorUID, testRef("s2", 2, s2.CommitTime)) + endCheckpoint() if inMemory(tr) { - t.Error("entry stayed in memory after the older upload ended, want it dropped") + t.Error("entry stayed in memory after the checkpoint ended, want it dropped") } // The next deferred checkpoint is due a full delay after it was taken. @@ -210,33 +229,56 @@ func TestTrackerImmediateCheckpointDuringUpload(t *testing.T) { } } -// An older upload finishing after a newer snapshot became durable and another -// local checkpoint was taken does not push the new local checkpoint's upload -// due time back. -func TestTrackerLateUploadDoesNotPushBackNextUpload(t *testing.T) { +// CommitPromoted cancels an older snapshot's upload in progress and waits for +// it to stop before returning. The upload deletes the older snapshot's local +// copy before CommitPromoted returns, though the promote holds the actor. +func TestTrackerCommitPromotedCancelsOlderUpload(t *testing.T) { store := newFakeSnapshotStore() store.setBlocking(true) + store.setIgnoreCancel(true) tr := newTestTracker(t, store) + for _, name := range []string{"s1", "s2"} { + if err := os.MkdirAll(ateletpath.LocalSnapshotDir(trackedActorUID, name), 0o700); err != nil { + t.Fatal(err) + } + } t0 := tr.clock.Now() tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) - done := startUpload(t, tr, store) - + startUpload(t, tr, store) s2 := testRef("s2", 2, t0.Add(time.Minute)) + tr.CommitLocal(trackedActorUID, s2) + alreadyDurable, endPromote, err := tr.BeginPromote(t.Context(), trackedActorUID, s2) + if err != nil || alreadyDurable { + t.Fatalf("BeginPromote = (%v, %v), want (false, nil)", alreadyDurable, err) + } + defer endPromote() s2.UploadTime = s2.CommitTime.Add(time.Second) - tr.CommitDurable(trackedActorUID, s2) - t3 := t0.Add(2 * time.Minute) - tr.CommitLocal(trackedActorUID, testRef("s3", 3, t3)) - stepTo(tr, t3.Add(10*time.Minute)) - store.release <- nil - waitClosed(t, done, "the upload of s1") + stateCh := make(chan *ateletpb.ActorSnapshotState, 1) + go func() { + stateCh <- tr.CommitPromoted(trackedActorUID, s2) + }() + if got := waitCanceled(t, store); got != "s1" { + t.Fatalf("canceled upload = %q, want s1", got) + } + select { + case st := <-stateCh: + t.Fatalf("CommitPromoted returned %v before the older upload stopped", st) + case <-time.After(20 * time.Millisecond): + } - st := tr.State(trackedActorUID) - if got := uuidOf(st.GetLatestDurable()); got != "s2" { - t.Errorf("latest_durable = %q, want s2", got) + store.release <- context.Canceled + var st *ateletpb.ActorSnapshotState + select { + case st = <-stateCh: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for CommitPromoted to return") } - if got, want := timeOf(st.GetUploadDueTime()), t3.Add(testUploadDelay); !got.Equal(want) { - t.Errorf("upload_due_time = %v, want %v: an upload delay after s3 was taken", got, want) + if uuidOf(st.GetLatestDurable()) != "s2" || uuidOf(st.GetLatestLocal()) != "s2" || st.GetUploading() != nil || st.GetUploadDueTime() != nil { + t.Errorf("state = %v, want s2 durable and local with nothing uploading or due", st) + } + if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s1")); !os.IsNotExist(err) { + t.Errorf("s1 once CommitPromoted returned: stat = %v, want it deleted", err) } } @@ -285,14 +327,14 @@ func TestTrackerFailedUploadKeepsUndurableSince(t *testing.T) { } } -// Forget does not wait for a background upload in progress: it cancels the -// upload, which then leaves the forgotten actor alone. The upload records no -// state for the actor, and deletes none of its local snapshots, not even one -// a newer checkpoint superseded: Terminate prunes them. Its metric records -// error.type Canceled. +// Forget cancels a background upload in progress and waits for it to end +// before it drops the actor's state. The upload ends as a canceled one does, +// deleting the local copy of s1, which a newer checkpoint superseded. Its +// metric records error.type Canceled. func TestTrackerForgetCancelsUpload(t *testing.T) { store := newFakeSnapshotStore() store.setBlocking(true) + store.setIgnoreCancel(true) tr := newIdleTestTracker(t, store) inst, reader := newTestInstruments(t) tr.instruments = inst @@ -304,20 +346,38 @@ func TestTrackerForgetCancelsUpload(t *testing.T) { done := startUpload(t, tr, store) tr.CommitLocal(trackedActorUID, testRef("s2", 2, tr.clock.Now())) - returnsPromptly(t, "Forget during an upload", func() { - if err := tr.Forget(trackedActorUID); err != nil { + forgetDone := make(chan error, 1) + go func() { forgetDone <- tr.Forget(trackedActorUID) }() + if got := waitCanceled(t, store); got != "s1" { + t.Fatalf("canceled upload = %q, want s1", got) + } + select { + case err := <-forgetDone: + t.Fatalf("Forget returned (%v) before the canceled upload ended", err) + case <-time.After(50 * time.Millisecond): + } + store.release <- context.Canceled + select { + case err := <-forgetDone: + if err != nil { t.Errorf("Forget: %v", err) } - }) - waitClosed(t, done, "the cancelled upload") + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for Forget to return") + } + select { + case <-done: + default: + t.Error("Forget returned before the canceled upload ended") + } if st := tr.State(trackedActorUID); !proto.Equal(st, &ateletpb.ActorSnapshotState{}) { t.Errorf("state after forget = %v, want empty", st) } if _, err := os.Stat(ateletpath.ActorSnapshotStateFile(trackedActorUID)); !os.IsNotExist(err) { - t.Errorf("state file once the cancelled upload ended: stat = %v, want none", err) + t.Errorf("state file after Forget: stat = %v, want none", err) } - if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s1")); err != nil { - t.Errorf("s1 once the cancelled upload ended: %v, want it left for Terminate to prune", err) + if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s1")); !os.IsNotExist(err) { + t.Errorf("s1 after Forget: stat = %v, want the canceled upload to delete it", err) } if got, want := recordedErrorTypes(t, reader), []string{codes.Canceled.String()}; !slices.Equal(got, want) { t.Errorf("recorded upload error types = %v, want %v", got, want) @@ -452,7 +512,7 @@ func TestTrackerWithoutStoreSchedulesNoUploads(t *testing.T) { } } -// A local snapshot no upload can make durable, as a pause takes today, never +// A local snapshot no upload can make durable (e.g. with no upload URI) never // comes due for a background upload, rather than failing it again and again. // A checkpoint that replaces it schedules an upload again. func TestTrackerSkipsUnuploadableSnapshots(t *testing.T) { @@ -522,8 +582,9 @@ func TestTrackerStartKeepsUploadBackoff(t *testing.T) { } } -// BeginStart and BeginCheckpoint wait only for a detach in progress, not for a -// background upload. +// BeginStart, and BeginCheckpoint in either upload mode, wait only for a detach +// in progress, not for a background upload, so a Checkpoint never waits for +// one before it checkpoints the worker. func TestTrackerStartAndCheckpointDoNotWaitForUpload(t *testing.T) { store := newFakeSnapshotStore() store.setBlocking(true) @@ -531,14 +592,19 @@ func TestTrackerStartAndCheckpointDoNotWaitForUpload(t *testing.T) { tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) done := startUpload(t, tr, store) + beginCheckpoint := func(mode ateletpb.UploadMode) func(context.Context, string) (func(), error) { + return func(ctx context.Context, uid string) (func(), error) { + _, end, err := tr.BeginCheckpoint(ctx, uid, testRef("s2", 2, time.Time{}).Snapshot, mode) + return end, err + } + } for _, tc := range []struct { name string begin func(context.Context, string) (func(), error) }{ {"BeginStart", tr.BeginStart}, - {"BeginCheckpoint", func(ctx context.Context, uid string) (func(), error) { - return tr.BeginCheckpoint(ctx, uid, testRef("s2", 2, time.Time{}).Snapshot) - }}, + {"BeginCheckpoint deferred", beginCheckpoint(ateletpb.UploadMode_UPLOAD_MODE_DEFERRED)}, + {"BeginCheckpoint immediate", beginCheckpoint(ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE)}, } { returnsPromptly(t, tc.name+" during a background upload", func() { end, err := tc.begin(t.Context(), trackedActorUID) @@ -555,7 +621,7 @@ func TestTrackerStartAndCheckpointDoNotWaitForUpload(t *testing.T) { } // A background upload runs even while the actor is held by a Run, Restore, or -// Checkpoint, since an upload keeps the local snapshot intact. +// deferred Checkpoint, since an upload keeps the local snapshot intact. func TestTrackerUploadRunsWhileActorHeld(t *testing.T) { store := newFakeSnapshotStore() store.setBlocking(true) @@ -563,7 +629,7 @@ func TestTrackerUploadRunsWhileActorHeld(t *testing.T) { t0 := tr.clock.Now() tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) - end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s2", 2, t0).Snapshot) + _, end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s2", 2, t0).Snapshot, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) if err != nil { t.Fatalf("BeginCheckpoint: %v", err) } @@ -737,9 +803,9 @@ func TestTrackerBeginPromote(t *testing.T) { } } -// When a background upload of s1 fails after a pause replaced LatestLocal with -// s2 (which has no object storage location), s1 stays in Uploading on disk and -// is retried. Further pause checkpoints keep s1 on disk and preserve its +// When a background upload of s1 fails after a checkpoint replaced LatestLocal +// with s2 (which has no object storage location), s1 stays in Uploading on disk +// and is retried. Further local checkpoints keep s1 on disk and preserve its // backoff, and once s1's retry succeeds, s1's local copy is deleted. func TestTrackerRetriesUploadingWhenLatestLocalHasNoObject(t *testing.T) { store := newFakeSnapshotStore() @@ -772,7 +838,7 @@ func TestTrackerRetriesUploadingWhenLatestLocalHasNoObject(t *testing.T) { t.Fatalf("state after failed upload = %v, want s2 local, s1 uploading, due at %v", st, retryAt) } - // Another pause checkpoint during backoff still keeps s1 and its backoff. + // Another local checkpoint during backoff still keeps s1 and its backoff. s3 := testRef("s3", 3, tr.clock.Now()) s3.Snapshot.Object = nil st = tr.CommitLocal(trackedActorUID, s3) @@ -888,9 +954,9 @@ func TestTrackerCommitLocalDropsFailedUploading(t *testing.T) { } } -// A superseded local copy stays on disk while a Run, Restore, Checkpoint, or -// PromoteSnapshot of the actor is in progress, since a Restore may be reading -// it in place. It is left for the actor's next checkpoint to prune. +// A superseded local copy stays on disk while a Run or Restore of the actor is +// in progress, since a Restore may be reading it in place. It is left for the +// actor's next checkpoint to prune. func TestTrackerKeepsSupersededCopyWhileHeld(t *testing.T) { t.Run("upload ends", func(t *testing.T) { store := newFakeSnapshotStore() @@ -947,6 +1013,88 @@ func TestTrackerKeepsSupersededCopyWhileHeld(t *testing.T) { }) } +// While Terminate holds the actor, no upload worker clears a superseded +// Uploading snapshot or deletes its local copy: Terminate prunes the actor's +// local snapshots once Forget drops it. If the terminate fails before Forget, +// its end lets a worker clear the snapshot, which deletes the copy before +// releasing the actor's lock. +func TestTrackerTerminateHoldsOffSupersededUploadingCleanup(t *testing.T) { + for _, tc := range []struct { + name string + forget bool + }{ + {name: "terminate fails before Forget"}, + {name: "Forget drops the actor", forget: true}, + } { + t.Run(tc.name, func(t *testing.T) { + store := newFakeSnapshotStore() + store.setErr(errors.New("boom")) + tr := newIdleTestTracker(t, store) + s1Dir := ateletpath.LocalSnapshotDir(trackedActorUID, "s1") + if err := os.MkdirAll(s1Dir, 0o700); err != nil { + t.Fatal(err) + } + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now().Add(-testUploadDelay))) + processNext(t, tr, opUpload) // s1's upload fails, and s1 waits in Uploading to retry. + tr.CommitDurable(trackedActorUID, testRef("s2", 2, tr.clock.Now())) + + end := tr.BeginTerminate(trackedActorUID) + processNext(t, tr, opUpload) + if rec := readSnapshotState(trackedActorUID); rec == nil || rec.Uploading == nil || rec.Uploading.uuid() != "s1" { + t.Errorf("state file during Terminate = %+v, want s1 still Uploading", rec) + } + if _, err := os.Stat(s1Dir); err != nil { + t.Errorf("s1 during Terminate: stat = %v, want it left for Terminate to prune", err) + } + + if tc.forget { + if err := tr.Forget(trackedActorUID); err != nil { + t.Fatalf("Forget: %v", err) + } + end() + if n := queuedNow(tr); n != 0 { + t.Errorf("queued = %d after Forget and the terminate's end, want nothing queued for the forgotten actor", n) + } + return + } + end() + if run := tr.beginDueWork(trackedActorUID, opUpload); run != nil { + t.Error("beginDueWork left work to run after releasing the actor's lock, want s1 deleted under it") + run() + } + if rec := readSnapshotState(trackedActorUID); rec == nil || rec.Uploading != nil { + t.Errorf("state file once the terminate failed = %+v, want Uploading cleared", rec) + } + if _, err := os.Stat(s1Dir); !os.IsNotExist(err) { + t.Errorf("s1 once the terminate failed: stat = %v, want it deleted", err) + } + }) + } +} + +// A Run or Restore in progress may read an older local copy in place; +// Checkpoint and PromoteSnapshot do not. +func TestLocalInUse(t *testing.T) { + for _, tc := range []struct { + name string + hold holdKind + want bool + }{ + {name: "nothing in progress", hold: holdNone, want: false}, + {name: "Run or Restore", hold: holdStart, want: true}, + {name: "deferred Checkpoint", hold: holdCheckpoint, want: false}, + {name: "immediate Checkpoint or PromoteSnapshot", hold: holdDurable, want: false}, + {name: "Terminate", hold: holdTerminate, want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + a := &actorSnapshots{actorUID: trackedActorUID, hold: tc.hold} + if got := a.localInUse(); got != tc.want { + t.Errorf("localInUse() = %v, want %v", got, tc.want) + } + }) + } +} + // Uploading is saved to the snapshot state file, so an atelet restart mid-upload // resumes s1 unless a newer snapshot was already durable before the restart. func TestTrackerResumesUploadingAfterRestart(t *testing.T) { @@ -1040,6 +1188,7 @@ func TestTrackerResumesUploadingAfterRestart(t *testing.T) { } store := newFakeSnapshotStore() store.setBlocking(true) + store.setIgnoreCancel(true) ctx, cancel := context.WithCancel(t.Context()) before := New(ctx, store, newStoreTracker(t, nil).clock, WithUploadDelay(testUploadDelay), WithDetachDelay(testDetachDelay)) before.Start() @@ -1049,13 +1198,22 @@ func TestTrackerResumesUploadingAfterRestart(t *testing.T) { if got := waitStarted(t, store); got != "s1" { t.Fatalf("before started = %q, want s1", got) } - before.CommitDurable(trackedActorUID, testRef("s2", 2, t0.Add(time.Minute))) - // Capture the state file as a hard crash mid-upload leaves it. + commitDone := make(chan struct{}) + go func() { + defer close(commitDone) + before.CommitDurable(trackedActorUID, testRef("s2", 2, t0.Add(time.Minute))) + }() + waitCanceled(t, store) + // Capture the state file as a hard crash mid-upload leaves it: + // CommitDurable saved LatestDurable=s2, and s1's upload has not yet + // cleared Uploading. stateFile := ateletpath.ActorSnapshotStateFile(trackedActorUID) crashedState, err := os.ReadFile(stateFile) if err != nil { t.Fatal(err) } + store.release <- context.Canceled + waitClosed(t, commitDone, "CommitDurable") cancel() if err := before.Wait(t.Context()); err != nil { t.Fatalf("before.Wait: %v", err) @@ -1120,7 +1278,7 @@ func TestTrackerUploadActsForTheActor(t *testing.T) { if got := waitStarted(t, store); got != "s1" { t.Fatalf("before started = %q, want s1", got) } - // A pause snapshot no upload can make durable leaves s1 in Uploading, + // A local snapshot no upload can make durable leaves s1 in Uploading, // so the resumed upload reads its actor from there. s2 := testRef("s2", 2, t0.Add(time.Minute)) s2.Snapshot.Object = nil @@ -1200,6 +1358,54 @@ func TestTrackerUploadWaitsForPromote(t *testing.T) { }) } +// A background upload that comes due while a Checkpoint under +// UPLOAD_MODE_IMMEDIATE holds the actor waits until the checkpoint ends, then +// skips if the checkpoint made a newer snapshot durable or runs if the +// checkpoint failed. +func TestTrackerUploadWaitsForImmediateCheckpoint(t *testing.T) { + for _, tc := range []struct { + name string + // commit is set when the checkpoint makes s2 durable before it ends. + commit bool + want []string + }{ + {name: "checkpoint succeeds", commit: true}, + {name: "checkpoint fails", want: []string{"s1"}}, + } { + t.Run(tc.name, func(t *testing.T) { + store := newFakeSnapshotStore() + tr := newIdleTestTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + + s2 := testRef("s2", 2, tr.clock.Now()) + _, endCheckpoint, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, s2.Snapshot, ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE) + if err != nil { + t.Fatalf("BeginCheckpoint: %v", err) + } + stepTo(tr, tr.clock.Now().Add(testUploadDelay)) + processNext(t, tr, opUpload) + if got := store.uploads(); len(got) != 0 { + t.Fatalf("uploads during the checkpoint = %v, want none", got) + } + if n := tr.queues[opUpload].Len(); n != 0 { + t.Fatalf("queued uploads during the checkpoint = %d, want none: the checkpoint's end queues it", n) + } + + if tc.commit { + s2.UploadTime = tr.clock.Now() + tr.CommitDurable(trackedActorUID, s2) + } + endCheckpoint() + if n := tr.queues[opUpload].Len(); n != 0 { + processNext(t, tr, opUpload) + } + if got := store.uploads(); !slices.Equal(got, tc.want) { + t.Errorf("uploads after the checkpoint ended = %v, want %v", got, tc.want) + } + }) + } +} + // An out-of-order CommitLocal with a lower generation than Uploading does not // replace a failed Uploading target, reset its backoff, or delete Uploading's // local copy when it finishes. diff --git a/cmd/atelet/lifecycle_linux_test.go b/cmd/atelet/lifecycle_linux_test.go new file mode 100644 index 0000000000..a1eb442ee1 --- /dev/null +++ b/cmd/atelet/lifecycle_linux_test.go @@ -0,0 +1,51 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "errors" + "path/filepath" + "syscall" + "testing" + + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/proto/ateletpb" +) + +// TestLocalSnapshotGCWithSeparateSnapshotsFilesystem runs TestLocalSnapshotGC +// with SnapshotsDir on a different filesystem from ActorsDir. +func TestLocalSnapshotGCWithSeparateSnapshotsFilesystem(t *testing.T) { + for _, fidelity := range []ateletpb.SnapshotFidelity{ + ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, + ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + } { + t.Run(fidelity.String(), func(t *testing.T) { + runLocalSnapshotGC(t, fidelity, useSeparateSnapshotsFilesystem) + }) + } +} + +// useSeparateSnapshotsFilesystem is useTempNodeDirs with SnapshotsDir on another mount. +func useSeparateSnapshotsFilesystem(t *testing.T) { + t.Helper() + useTempNodeDirs(t) + shm := otherMount(t) + root := filepath.Dir(nodepath.ActorsDir) + if err := canHardLink(root, shm); !errors.Is(err, syscall.EXDEV) { + t.Fatalf("canHardLink(%s, %s) = %v, want EXDEV: SnapshotsDir must be on another mount than ActorsDir", root, shm, err) + } + nodepath.SnapshotsDir = filepath.Join(shm, "snapshots") +} diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index e813929a54..d94bb2fe49 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -16,6 +16,7 @@ package main import ( "context" + "crypto/sha256" "errors" "fmt" "net" @@ -30,6 +31,7 @@ import ( "time" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/cmd/atelet/internal/filecache" "go.ate.dev/substrate/cmd/atelet/internal/snapshottracker" "go.ate.dev/substrate/cmd/atelet/internal/trustbundle" "go.ate.dev/substrate/internal/apierror" @@ -41,6 +43,7 @@ import ( "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.opentelemetry.io/otel/sdk/metric/metricdata" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -55,10 +58,11 @@ import ( func useTempNodeDirs(t *testing.T) { t.Helper() root := t.TempDir() - origActors := nodepath.ActorsDir + origActors, origSnapshots := nodepath.ActorsDir, nodepath.SnapshotsDir nodepath.ActorsDir = filepath.Join(root, "actors") + nodepath.SnapshotsDir = filepath.Join(root, "snapshots") t.Cleanup(func() { - nodepath.ActorsDir = origActors + nodepath.ActorsDir, nodepath.SnapshotsDir = origActors, origSnapshots }) } @@ -76,6 +80,9 @@ type fakeWorker struct { // restored holds the file contents staged into the restore dir by the // most recent RestoreWorkload. restored map[string]string + // restoreErr, if set, fails every RestoreWorkload after it recorded the + // request, as a worker that could not start the sandbox would. + restoreErr error // actorDirs records the ActorDirs each RPC arrived with, by RPC name. actorDirs map[string]*ateworkerpb.ActorDirs // preserveRestoreDir records the PreserveRestoreDir flag from the most @@ -87,6 +94,9 @@ type fakeWorker struct { // onCheckpoint, if non-nil, runs inside CheckpointWorkload after snapshot // files are written. onCheckpoint func() + // onTerminate, if non-nil, runs inside TerminateWorkload, which fails with + // the error it returns. + onTerminate func() error } func (f *fakeWorker) recordFidelity(rpc string, fidelity ateworkerpb.SnapshotFidelity) { @@ -133,6 +143,9 @@ func (f *fakeWorker) RestoreWorkload(_ context.Context, req *ateworkerpb.Restore f.recordActorDirs("RestoreWorkload", req.GetActorDirs()) f.recordFidelity("RestoreWorkload", req.GetFidelity()) f.preserveRestoreDir = req.GetPreserveRestoreDir() + if f.restoreErr != nil { + return nil, f.restoreErr + } dir := req.GetActorDirs().GetRestoreDir() f.restored = map[string]string{} for name := range f.snapshotFiles { @@ -147,6 +160,11 @@ func (f *fakeWorker) RestoreWorkload(_ context.Context, req *ateworkerpb.Restore func (f *fakeWorker) TerminateWorkload(_ context.Context, req *ateworkerpb.TerminateWorkloadRequest) (*ateworkerpb.TerminateWorkloadResponse, error) { f.recordActorDirs("TerminateWorkload", req.GetActorDirs()) + if f.onTerminate != nil { + if err := f.onTerminate(); err != nil { + return nil, err + } + } return &ateworkerpb.TerminateWorkloadResponse{}, nil } @@ -187,23 +205,26 @@ func lifecycleSnapshot(actorUID, name string, generation int32, fidelity ateletp } // TestLocalSnapshotGC walks an actor through -// run -> pause -> resume -> pause -> terminate over atelet's RPC surface and -// ensures that superseded local snapshots are pruned by the next checkpoint -// and all remaining state is garbage collected after the actor is terminated. -// It runs at every fidelity atelet forwards, pinning that the pause and the -// resume hand worker the fidelity the control plane asked for, unchanged. +// run -> deferred checkpoint -> restore -> deferred checkpoint -> terminate +// over atelet's RPC surface and ensures that superseded local snapshots are +// pruned by the next checkpoint and all remaining state is garbage collected +// after the actor is terminated. It runs at every fidelity atelet forwards, +// pinning that the checkpoint and the restore hand worker the fidelity the +// control plane asked for, unchanged. func TestLocalSnapshotGC(t *testing.T) { for _, fidelity := range []ateletpb.SnapshotFidelity{ ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, } { - t.Run(fidelity.String(), func(t *testing.T) { runLocalSnapshotGC(t, fidelity) }) + t.Run(fidelity.String(), func(t *testing.T) { runLocalSnapshotGC(t, fidelity, useTempNodeDirs) }) } } -func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) { - useTempNodeDirs(t) +// runLocalSnapshotGC runs the TestLocalSnapshotGC flow on the node dirs useDirs +// sets up. +func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity, useDirs func(*testing.T)) { + useDirs(t) ctx := t.Context() const ( @@ -231,7 +252,7 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) { systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } - sandboxRec := testSandboxRecord(release, pinnedImage) + sandboxRec := testSandboxRecord(release) spec := &ateletpb.WorkloadSpec{ Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, } @@ -250,7 +271,7 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) { t.Fatalf("Run: %v", err) } - // Pause: a deferred checkpoint, which leaves the snapshot on this node. + // Async suspend: a deferred checkpoint, which leaves the snapshot on this node. resp, err := s.Checkpoint(ctx, &ateletpb.CheckpointRequest{ Atespace: atespace, ActorName: actorName, @@ -269,8 +290,8 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) { if _, err := os.Stat(snapshotFile); err != nil { t.Fatalf("pause did not write the local snapshot: %v", err) } - // A paused actor stays resumable on this node, so it keeps its staged - // assets, and with them its binaries in the cache. + // An actor with a local snapshot stays resumable on this node, so it keeps + // its staged assets, and with them its binaries in the cache. if _, err := os.Stat(stagedRunsc(actorUID)); err != nil { t.Errorf("paused actor lost its staged runsc: %v", err) } @@ -315,7 +336,7 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) { t.Fatalf("expected RestoreStateDir to remain empty on local pause restore, got entries=%v err=%v", entries, err) } - // Second pause: must replace, not accumulate alongside, the first snapshot. + // Second deferred checkpoint: must replace, not accumulate alongside, the first snapshot. if _, err := s.Checkpoint(ctx, &ateletpb.CheckpointRequest{ Atespace: atespace, ActorName: actorName, @@ -351,7 +372,7 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) { } // Every RPC hands worker the same directory set, except that a local - // restore points restore_dir at the pause snapshot; the fake already + // restore points restore_dir at the local snapshot; the fake already // relied on checkpoint_dir and restore_dir above to place and find it. for _, rpc := range []string{"RunWorkload", "CheckpointWorkload", "RestoreWorkload", "TerminateWorkload"} { want := ateletpath.ActorDirs(actorUID) @@ -374,16 +395,8 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) { // Terminate is the only chance to reclaim the actor's directory: nothing // else on the node deletes it. - actorDir := ateletpath.ActorPath(actorUID) - if entries, err := os.ReadDir(actorDir); err == nil { - left := make([]string, 0, len(entries)) - for _, e := range entries { - left = append(left, e.Name()) - } - t.Errorf("actor dir %s survived terminate with %d entries: %v", actorDir, len(left), left) - } else if !os.IsNotExist(err) { - t.Errorf("reading actor dir %s: %v", actorDir, err) - } + assertDirGone(t, "actor dir", ateletpath.ActorPath(actorUID)) + assertDirGone(t, "actor snapshot dir", ateletpath.ActorSnapshotPath(actorUID)) } // TestRestoreUsesRequestSandboxRuntime checks that Restore runs the actor with @@ -407,26 +420,27 @@ func TestRestoreUsesRequestSandboxRuntime(t *testing.T) { host := imageVolumeTestRegistry(t) image := host + "/actor:v1" pinnedImage := pushTestImage(t, image, singleFileLayer(t, "bin/app", "app")) - checkpointPause := pushTestImage(t, host+"/pause:v1", singleFileLayer(t, "pause", "pause-v1")) - restorePause := pushTestImage(t, host+"/pause:v2", singleFileLayer(t, "pause", "pause-v2")) - release := newGVisorTestRelease(t) + // The two releases differ in content, so they have distinct bundle hashes. + release := func(runsc string) gvisorTestRelease { + tarball := gvisorReleaseTarballWith(t, runsc) + return gvisorTestRelease{tarball: tarball, entry: assetEntry{URL: "gs://test-bucket/gvisor.tar", SHA256: fmt.Sprintf("%x", sha256.Sum256(tarball))}} + } + checkpointRelease, restoreRelease := release("runsc v1"), release("runsc v2") s := &Atelet{ workerDialer: newWorkerDialer(1), imageCache: newImageVolumeStore(t), sandboxAssetCache: newTestAssetCache(t), - anonGCSClient: fakeObjectStorage{data: release.tarball}, + anonGCSClient: fakeObjectStorage{data: checkpointRelease.tarball}, systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } - recWithPause := func(pause string) *sandboxAssetsRecord { - return testSandboxRecord(release, pause) - } + checkpointRec, restoreRec := testSandboxRecord(checkpointRelease), testSandboxRecord(restoreRelease) spec := &ateletpb.WorkloadSpec{ Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, } - advertiseRuntime(t, s, recWithPause(checkpointPause)) + advertiseRuntime(t, s, checkpointRec) if _, err := s.Run(ctx, &ateletpb.RunRequest{ Atespace: atespace, ActorName: actorName, @@ -434,7 +448,7 @@ func TestRestoreUsesRequestSandboxRuntime(t *testing.T) { ActorTemplateAtespace: "default", ActorTemplateName: "counter", WorkerPodUid: workerPodUID, - SandboxRuntime: runtimeFor(t, recWithPause(checkpointPause)), + SandboxRuntime: runtimeFor(t, checkpointRec), Spec: spec, }); err != nil { t.Fatalf("Run: %v", err) @@ -462,11 +476,12 @@ func TestRestoreUsesRequestSandboxRuntime(t *testing.T) { if err != nil { t.Fatalf("unmarshalling snapshot manifest: %v", err) } - if manifestRec.PauseImage != checkpointPause { - t.Fatalf("manifest pause image = %q, want %q", manifestRec.PauseImage, checkpointPause) + if got, want := manifestRec.Assets[gvisorAssetName].SHA256, checkpointRelease.entry.SHA256; got != want { + t.Fatalf("manifest gvisor sha256 = %q, want %q", got, want) } - advertiseRuntime(t, s, recWithPause(restorePause)) + s.anonGCSClient = fakeObjectStorage{data: restoreRelease.tarball} + advertiseRuntime(t, s, restoreRec) if _, err := s.Restore(ctx, &ateletpb.RestoreRequest{ Atespace: atespace, ActorName: actorName, @@ -474,7 +489,7 @@ func TestRestoreUsesRequestSandboxRuntime(t *testing.T) { ActorTemplateAtespace: "default", ActorTemplateName: "counter", WorkerPodUid: workerPodUID, - SandboxRuntime: runtimeFor(t, recWithPause(restorePause)), + SandboxRuntime: runtimeFor(t, restoreRec), Spec: spec, Snapshot: lifecycleSnapshot(actorUID, snapshotName, 1, ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY), }); err != nil { @@ -485,8 +500,8 @@ func TestRestoreUsesRequestSandboxRuntime(t *testing.T) { if err != nil { t.Fatalf("reading on-node sandbox record: %v", err) } - if got.PauseImage != restorePause { - t.Errorf("restored actor pause image = %q, want the request's %q", got.PauseImage, restorePause) + if sha, want := got.Assets[gvisorAssetName].SHA256, restoreRelease.entry.SHA256; sha != want { + t.Errorf("restored actor gvisor sha256 = %q, want the request's %q", sha, want) } } @@ -500,13 +515,19 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { failure string wantErr string wantPresent bool + // publishedVolume leaves an external volume mount point with + // content in it, as a failed earlier attempt that published the + // volume would. + publishedVolume bool }{ {name: "Run asset failure preserves projection", failure: "asset", wantErr: "asset unavailable", wantPresent: true}, - {name: "Run partial reset removes registration", failure: "reset", wantErr: "while removing volume dir"}, + {name: "Run partial reset removes registration", failure: "reset", wantErr: "while creating volumes dir"}, {name: "Run record failure removes registration", failure: "record", wantErr: "while recording sandbox assets"}, - {name: "Restore partial reset removes registration", restore: true, failure: "reset", wantErr: "while removing volume dir"}, + {name: "Restore partial reset removes registration", restore: true, failure: "reset", wantErr: "while creating volumes dir"}, {name: "Restore manifest failure removes registration", restore: true, failure: "manifest", wantErr: "no uploaded copy"}, {name: "Restore asset failure removes registration", restore: true, failure: "asset", wantErr: "asset unavailable"}, + {name: "Run retry keeps a published volume", failure: "record", publishedVolume: true, wantErr: "while recording sandbox assets"}, + {name: "Restore retry keeps a published volume", restore: true, failure: "manifest", publishedVolume: true, wantErr: "no uploaded copy"}, } { t.Run(tc.name, func(t *testing.T) { useTempNodeDirs(t) @@ -531,19 +552,15 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { release := newGVisorTestRelease(t) if tc.restore && tc.failure != "manifest" { writeLocalSnapshot(t, ateletpath.LocalSnapshotDir(actorUID, snapshotName), sandboxAssetsRecord{ - SandboxClass: "gvisor", PauseImage: testPauseImage, + SandboxClass: "gvisor", Assets: map[string]assetEntry{gvisorAssetName: release.entry}, SnapshotFiles: []string{"checkpoint.img"}, }, map[string]string{"checkpoint.img": "guest-memory"}) } if tc.failure == "reset" { - // Reset refuses to remove a populated external-volume directory, - // but has already removed the system-info roots by then. - dir := ateletpath.ExternalVolumeHostPath(actorUID, "mounted") - if err := os.MkdirAll(dir, 0o750); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "keep"), []byte("volume data"), 0o600); err != nil { + // A file where the volumes dir goes fails the reset after it + // has removed the system-info roots. + if err := os.WriteFile(ateletpath.ExternalVolumeMountsDir(actorUID), nil, 0o600); err != nil { t.Fatal(err) } } @@ -552,6 +569,18 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { t.Fatal(err) } } + // A non-empty dir fails os.Remove like a live mount (EBUSY) + // would, so the activation only gets past the reset to fail + // later with wantErr if the reset leaves volumes/ alone. + volumeFile := filepath.Join(ateletpath.ExternalVolumeHostPath(actorUID, "data"), "data.txt") + if tc.publishedVolume { + if err := os.MkdirAll(filepath.Dir(volumeFile), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(volumeFile, []byte("volume data"), 0o600); err != nil { + t.Fatal(err) + } + } // The live activation's staged binary: a Run that fails before // it swaps its own set in must leave it, and a Run that fails // after must still leave nothing half-staged behind. @@ -571,7 +600,7 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { s := newPluginHerder(t, &recordingObjectStorage{}) s.anonGCSClient = storage s.systemInfoVolumes = refresher - rec := testSandboxRecord(release, testPauseImage) + rec := testSandboxRecord(release) advertiseRuntime(t, s, rec) if tc.restore { _, err = s.Restore(ctx, &ateletpb.RestoreRequest{ @@ -591,6 +620,11 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { if _, err := os.Lstat(ateletpath.SandboxAssetsStagingDir(actorUID)); !os.IsNotExist(err) { t.Errorf("failed activation left the staging dir behind (lstat err = %v)", err) } + if tc.publishedVolume { + if got, err := os.ReadFile(volumeFile); err != nil || string(got) != "volume data" { + t.Errorf("activation touched the published volume: %q, %v", got, err) + } + } // Inspect disk and registry before changing the bundle hash: a later // rotation can hide a stale registration by recreating deleted files. @@ -659,12 +693,13 @@ func TestRunFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } spec := &ateletpb.WorkloadSpec{ + Containers: []*ateletpb.Container{{Name: "app", Image: missingImage(t)}}, Volumes: []*ateletpb.Volume{{ Name: "trust", SystemInfo: trustVolumeSpec("ca.pem"), }}, } - rec := testSandboxRecord(release, missingPauseImage(t)) + rec := testSandboxRecord(release) advertiseRuntime(t, s, rec) _, err := s.Run(t.Context(), &ateletpb.RunRequest{ Atespace: "team-a", @@ -674,7 +709,7 @@ func TestRunFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { SandboxRuntime: runtimeFor(t, rec), Spec: spec, }) - if err == nil || !strings.Contains(err.Error(), "while creating pause OCI bundle") { + if err == nil || !strings.Contains(err.Error(), `while creating "app" OCI bundle`) { t.Fatalf("Run error = %v, want the post-registration OCI preparation failure", err) } if _, err := os.ReadFile(filepath.Join(ateletpath.SystemInfoVolumeRoot(actorUID, "trust"), "ca.pem")); err != nil { @@ -685,12 +720,12 @@ func TestRunFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { } } -// missingPauseImage returns a digest-pinned reference that passes request +// missingImage returns a digest-pinned reference that passes request // validation but names a manifest the local test registry does not serve, so // the pull fails only once OCI bundle preparation starts. -func missingPauseImage(t *testing.T) string { +func missingImage(t *testing.T) string { t.Helper() - return imageVolumeTestRegistry(t) + "/pause@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + return imageVolumeTestRegistry(t) + "/app@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" } func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { @@ -699,22 +734,24 @@ func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { actorUID = "0f6c2b0e-1b7e-4b3f-9a6a-1d2c3e4f5a63" snapshotName = "pause-snap-after" ) - pauseImage := missingPauseImage(t) + image := missingImage(t) store := newCTBStore(t) store.set(t, string(testCertPEM(t))) refresher := newSystemInfoVolumeRefresher(trustbundle.NewSource(store.lister.Get, nil), nil) release := newGVisorTestRelease(t) writeLocalSnapshot(t, ateletpath.LocalSnapshotDir(actorUID, snapshotName), sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: pauseImage, Assets: map[string]assetEntry{gvisorAssetName: release.entry}, SnapshotFiles: []string{"checkpoint.img"}, }, map[string]string{"checkpoint.img": "guest-memory"}) - spec := &ateletpb.WorkloadSpec{Volumes: []*ateletpb.Volume{{ - Name: "trust", - SystemInfo: trustVolumeSpec("ca.pem"), - }}} + spec := &ateletpb.WorkloadSpec{ + Containers: []*ateletpb.Container{{Name: "app", Image: image}}, + Volumes: []*ateletpb.Volume{{ + Name: "trust", + SystemInfo: trustVolumeSpec("ca.pem"), + }}, + } s := &Atelet{ sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{data: release.tarball}, @@ -722,7 +759,7 @@ func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { systemInfoVolumes: refresher, snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } - rec := testSandboxRecord(release, pauseImage) + rec := testSandboxRecord(release) advertiseRuntime(t, s, rec) _, err := s.Restore(t.Context(), &ateletpb.RestoreRequest{ Atespace: "team-a", @@ -733,7 +770,7 @@ func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { Spec: spec, Snapshot: lifecycleSnapshot(actorUID, snapshotName, 1, ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY), }) - if err == nil || !strings.Contains(err.Error(), "while creating pause OCI bundle") { + if err == nil || !strings.Contains(err.Error(), `while creating "app" OCI bundle`) { t.Fatalf("Restore error = %v, want the post-registration OCI preparation failure", err) } if _, err := os.ReadFile(filepath.Join(ateletpath.SystemInfoVolumeRoot(actorUID, "trust"), "ca.pem")); err != nil { @@ -765,13 +802,12 @@ func TestRestorePrefersLocalAndFallsBackToObjectStorage(t *testing.T) { pinnedImage := pushTestImage(t, host+"/actor:v1", singleFileLayer(t, "bin/app", "app")) release := newGVisorTestRelease(t) - sandboxRec := testSandboxRecord(release, pinnedImage) + sandboxRec := testSandboxRecord(release) spec := &ateletpb.WorkloadSpec{ Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, } manifestRec := sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: pinnedImage, Assets: map[string]assetEntry{gvisorAssetName: release.entry}, SnapshotFiles: []string{"checkpoint.img"}, } @@ -791,7 +827,7 @@ func TestRestorePrefersLocalAndFallsBackToObjectStorage(t *testing.T) { } remoteSrcDir := t.TempDir() writeLocalSnapshot(t, remoteSrcDir, manifestRec, map[string]string{"checkpoint.img": "object-storage-memory"}) - if err := s.uploadSnapshot(t.Context(), uri, remoteSrcDir, &manifestRec); err != nil { + if err := s.uploadSnapshot(t.Context(), testActor, uri, remoteSrcDir, &manifestRec); err != nil { t.Fatalf("uploadSnapshot: %v", err) } @@ -889,6 +925,85 @@ func TestRestorePrefersLocalAndFallsBackToObjectStorage(t *testing.T) { } } +// TestRestoreFailureRemovesStagedSnapshotFiles pins what a shared restore +// that fails after its snapshot files were staged leaves behind: nothing in +// the restore-state directory, so the links into the snapshot cache do not +// hold its entries until a later lifecycle call resets the directory. +func TestRestoreFailureRemovesStagedSnapshotFiles(t *testing.T) { + useTempNodeDirs(t) + const ( + actorUID = "0f6c2b0e-1b7e-4b3f-9a6a-1d2c3e4f5a66" + snapshotName = "pause-snap-1" + ) + worker := &fakeWorker{snapshotFiles: map[string]string{"checkpoint.img": ""}, restoreErr: status.Error(codes.Internal, "sandbox would not start")} + serveFakeWorker(t, worker) + host := imageVolumeTestRegistry(t) + pinnedImage := pushTestImage(t, host+"/actor:v1", singleFileLayer(t, "bin/app", "app")) + release := newGVisorTestRelease(t) + sandboxRec := testSandboxRecord(release) + manifestRec := sandboxAssetsRecord{ + SandboxClass: "gvisor", + Assets: map[string]assetEntry{gvisorAssetName: release.entry}, + SnapshotFiles: []string{"checkpoint.img"}, + } + s := newPluginHerder(t, &recordingObjectStorage{}) + s.workerDialer = newWorkerDialer(1) + s.imageCache = newImageVolumeStore(t) + s.anonGCSClient = fakeObjectStorage{data: release.tarball} + s.systemInfoVolumes = newSystemInfoVolumeRefresher(nil, nil) + // No min age, so the only thing standing between a staged entry and a + // pass is the restore's link. + cache, err := filecache.New(filepath.Join(t.TempDir(), "cache"), filecache.WithMinAge(0)) + if err != nil { + t.Fatal(err) + } + s.snapshotCache = cache + uri, err := resources.ParseSnapshotURI(pausedSnapshotURI) + if err != nil { + t.Fatal(err) + } + srcDir := t.TempDir() + writeLocalSnapshot(t, srcDir, manifestRec, map[string]string{"checkpoint.img": "shared-memory"}) + if err := s.uploadSnapshot(t.Context(), testActor, uri, srcDir, &manifestRec); err != nil { + t.Fatalf("uploadSnapshot: %v", err) + } + advertiseRuntime(t, s, sandboxRec) + + _, err = s.Restore(t.Context(), &ateletpb.RestoreRequest{ + Atespace: "ate-demo", + ActorName: "counter", + ActorUid: actorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + WorkerPodUid: "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f", + SandboxRuntime: runtimeFor(t, sandboxRec), + Spec: &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}}, + Snapshot: &ateletpb.Snapshot{ + SnapshotUuid: snapshotName, + Generation: 1, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateletpb.ObjectStorage{SnapshotUri: pausedSnapshotURI, Sharing: ateletpb.SnapshotSharing_SNAPSHOT_SHARING_SHARED}, + }, + }) + if status.Code(err) != codes.Internal { + t.Fatalf("Restore = %v, want the worker's Internal", err) + } + if worker.actorDirs["RestoreWorkload"] == nil { + t.Fatal("the worker was never asked: the failure must come after the snapshot files were staged") + } + if entries, err := os.ReadDir(ateletpath.RestoreStateDir(actorUID)); err == nil && len(entries) != 0 { + t.Errorf("restore-state holds %v after a failed restore, want nothing", entries) + } + stats, err := cache.EvictUnused(t.Context(), 1<<40, true) + if err != nil { + t.Fatal(err) + } + // Two entries, the manifest and checkpoint.img, both unlinked. + if stats.Retired != 2 || stats.SkippedLinked != 0 { + t.Errorf("after the failed restore a pass would retire %d entries and skip %d linked, want 2 and 0: the staged link must be gone", stats.Retired, stats.SkippedLinked) + } +} + // TestRestoreWithoutUsableLocalCopy checks that a Restore with no usable local // copy, naming a snapshot.object that was never uploaded, fails as NotFound // and wraps the local copy's problem. @@ -898,7 +1013,7 @@ func TestRestoreWithoutUsableLocalCopy(t *testing.T) { snapshotName = "pause-snap-1" ) release := newGVisorTestRelease(t) - rec := testSandboxRecord(release, testPauseImage) + rec := testSandboxRecord(release) for _, tc := range []struct { name string // unusableLocal writes a manifest that lists a file that is not there. @@ -916,7 +1031,7 @@ func TestRestoreWithoutUsableLocalCopy(t *testing.T) { useTempNodeDirs(t) if tc.unusableLocal { writeLocalSnapshot(t, ateletpath.LocalSnapshotDir(actorUID, snapshotName), sandboxAssetsRecord{ - SandboxClass: "gvisor", PauseImage: testPauseImage, SnapshotFiles: []string{"checkpoint.img"}, + SandboxClass: "gvisor", SnapshotFiles: []string{"checkpoint.img"}, }, nil) } // The object store is empty: nothing was ever uploaded. @@ -944,7 +1059,7 @@ func TestRestoreWithoutUsableLocalCopy(t *testing.T) { } // TestTerminateWithoutWorkerPodUID verifies that Terminate with an empty -// WorkerPodUid (used when cleaning up the assigned node for a paused or +// WorkerPodUid (used when cleaning up the assigned node for a SUSPENDED or // crashed actor that no longer has a worker pod) skips dialing worker while // still pruning local checkpoints and removing actor directories on the node. func TestTerminateWithoutWorkerPodUID(t *testing.T) { @@ -1001,6 +1116,9 @@ func TestTerminateWithoutWorkerPodUID(t *testing.T) { if _, err := os.Stat(ateletpath.LocalCheckpointsDir(actorUID)); !os.IsNotExist(err) { t.Errorf("local checkpoint dir survived terminate: %v", err) } + if _, err := os.Stat(ateletpath.ActorSnapshotPath(actorUID)); !os.IsNotExist(err) { + t.Errorf("actor snapshot dir survived terminate: %v", err) + } if _, err := os.Stat(ateletpath.ActorPath(actorUID)); !os.IsNotExist(err) { t.Errorf("actor dir survived terminate: %v", err) } @@ -1043,7 +1161,7 @@ func runPluginActor(t *testing.T) *pluginActor { a.s.sandboxAssetCache = newTestAssetCache(t) a.s.anonGCSClient = fakeObjectStorage{data: release.tarball} a.s.systemInfoVolumes = newSystemInfoVolumeRefresher(nil, nil) - a.rec = testSandboxRecord(release, pinnedImage) + a.rec = testSandboxRecord(release) a.spec = &ateletpb.WorkloadSpec{ Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, } @@ -1159,9 +1277,13 @@ func TestImmediateCheckpointKeepsLocalCopy(t *testing.T) { // TestImmediateCheckpointWithoutLocalCopy suspends a running actor whose // snapshot cannot be stored locally: the snapshot is durable, so the -// checkpoint still succeeds, and reports no local snapshot. +// checkpoint still succeeds, and reports no local snapshot. A retry of that +// checkpoint answers from the durable snapshot without checkpointing or +// uploading again, and finishes the first call's cleanup. func TestImmediateCheckpointWithoutLocalCopy(t *testing.T) { a := runPluginActor(t) + var workerCheckpoints atomic.Int32 + a.worker.onCheckpoint = func() { workerCheckpoints.Add(1) } snap := a.snapshot("suspend-snap-1", 1) // A file where the local checkpoint directory belongs fails the move. if err := os.WriteFile(ateletpath.LocalCheckpointsDir(pluginActorUID), nil, 0o600); err != nil { @@ -1185,6 +1307,181 @@ func TestImmediateCheckpointWithoutLocalCopy(t *testing.T) { if got := a.s.snapshots.State(pluginActorUID); !proto.Equal(got, st) { t.Errorf("tracked state = %v, want %v, as Checkpoint returned", got, st) } + + a.store.mu.Lock() + a.store.putErr = errors.New("retried Checkpoint uploaded a snapshot that was already durable") + a.store.mu.Unlock() + // What the first call's cleanup leaves behind when it fails. + leftover := filepath.Join(ateletpath.OCIBundleDir(pluginActorUID), "leftover") + if err := os.WriteFile(leftover, nil, 0o600); err != nil { + t.Fatal(err) + } + + retryResp, err := a.checkpoint(t, snap, ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE) + if err != nil { + t.Fatalf("retried Checkpoint: %v", err) + } + if n := workerCheckpoints.Load(); n != 1 { + t.Errorf("worker checkpointed the actor %d times, want once", n) + } + if !proto.Equal(retryResp.GetState(), st) { + t.Errorf("retried Checkpoint state = %v, want %v", retryResp.GetState(), st) + } + if _, err := os.Stat(leftover); !os.IsNotExist(err) { + t.Errorf("leftover in the bundle dir after the retry: stat = %v, want it cleaned up", err) + } +} + +// TestStaleCheckpointRetryLeavesRestoredActorAlone retries a Checkpoint +// after the actor was restored from the snapshot it committed, from its local +// copy or, once atelet detached it, from its upload: atelet answers from that +// snapshot without checkpointing or cleaning up, since the actor's +// directories and volumes belong to the restored activation now. +func TestStaleCheckpointRetryLeavesRestoredActorAlone(t *testing.T) { + for _, tc := range []struct { + name string + detach bool + }{ + {name: "restored from the local copy"}, + {name: "restored from the upload", detach: true}, + } { + t.Run(tc.name, func(t *testing.T) { + a := runPluginActor(t) + a.s.snapshots = snapshottracker.New(t.Context(), &trackerStore{s: a.s}, clock.RealClock{}) + var workerCheckpoints atomic.Int32 + a.worker.onCheckpoint = func() { workerCheckpoints.Add(1) } + snap := a.snapshot("pause-snap-1", 1) + if _, err := a.checkpoint(t, snap, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); err != nil { + t.Fatalf("Checkpoint: %v", err) + } + if tc.detach { + if _, err := a.s.snapshots.Detach(t.Context(), pluginActorUID); err != nil { + t.Fatalf("Detach: %v", err) + } + } + // Restore reads the local copy if there is one, and otherwise the + // upload. + if _, err := a.s.Restore(t.Context(), &ateletpb.RestoreRequest{ + Atespace: "ate-demo", + ActorName: "counter", + ActorUid: pluginActorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + WorkerPodUid: pluginWorkerPodUID, + SandboxRuntime: runtimeFor(t, a.rec), + Spec: a.spec, + Snapshot: snap, + }); err != nil { + t.Fatalf("Restore: %v", err) + } + inUse := filepath.Join(ateletpath.OCIBundleDir(pluginActorUID), "in-use") + if err := os.WriteFile(inUse, nil, 0o600); err != nil { + t.Fatal(err) + } + + resp, err := a.checkpoint(t, snap, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + if err != nil { + t.Fatalf("stale retry of the Checkpoint: %v", err) + } + if n := workerCheckpoints.Load(); n != 1 { + t.Errorf("worker checkpointed the actor %d times, want once", n) + } + st := resp.GetState() + committed := st.GetLatestLocal().GetSnapshot() + if tc.detach { + committed = st.GetLatestDurable().GetSnapshot() + } + if !proto.Equal(committed, snap) { + t.Errorf("state = %v, want %v as committed", st, snap) + } + if _, err := os.Stat(inUse); err != nil { + t.Errorf("restored activation's bundle dir after the stale retry: stat = %v, want it untouched", err) + } + }) + } +} + +// TestRetriedCheckpointAnswersFromCommittedSnapshot retries a Checkpoint +// that committed its snapshot, as the control plane does when the first +// call's response is lost: atelet answers without checkpointing the worker +// again, which has no workload left to checkpoint, and finishes the first +// call's cleanup. A retry under UPLOAD_MODE_IMMEDIATE uploads the snapshot +// first if the first Checkpoint deferred its upload, and a retry under +// UPLOAD_MODE_DEFERRED after atelet detached the snapshot returns no local +// snapshot. A retry records no checkpoint duration. +func TestRetriedCheckpointAnswersFromCommittedSnapshot(t *testing.T) { + const ( + deferred = ateletpb.UploadMode_UPLOAD_MODE_DEFERRED + immediate = ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE + ) + for _, tc := range []struct { + name string + first, retry ateletpb.UploadMode + detachFirst bool + wantLocal bool + wantDurable bool + }{ + {name: "deferred", first: deferred, retry: deferred, wantLocal: true}, + {name: "immediate", first: immediate, retry: immediate, wantLocal: true, wantDurable: true}, + {name: "deferred then immediate", first: deferred, retry: immediate, wantLocal: true, wantDurable: true}, + {name: "immediate then deferred", first: immediate, retry: deferred, wantLocal: true, wantDurable: true}, + {name: "deferred after detach", first: deferred, retry: deferred, detachFirst: true, wantDurable: true}, + {name: "immediate after detach", first: deferred, retry: immediate, detachFirst: true, wantDurable: true}, + } { + t.Run(tc.name, func(t *testing.T) { + a := runPluginActor(t) + a.s.snapshots = snapshottracker.New(t.Context(), &trackerStore{s: a.s}, clock.RealClock{}) + var workerCheckpoints atomic.Int32 + a.worker.onCheckpoint = func() { workerCheckpoints.Add(1) } + snap := a.snapshot("snap-1", 1) + if _, err := a.checkpoint(t, snap, tc.first); err != nil { + t.Fatalf("Checkpoint: %v", err) + } + if tc.detachFirst { + if _, err := a.s.snapshots.Detach(t.Context(), pluginActorUID); err != nil { + t.Fatalf("Detach: %v", err) + } + } + // What the first call's cleanup leaves behind when it fails. + leftover := filepath.Join(ateletpath.OCIBundleDir(pluginActorUID), "leftover") + if err := os.WriteFile(leftover, nil, 0o600); err != nil { + t.Fatal(err) + } + inst, reader := newTestInstruments(t) + a.s.instruments = inst + + resp, err := a.checkpoint(t, snap, tc.retry) + if err != nil { + t.Fatalf("retried Checkpoint: %v", err) + } + if n := workerCheckpoints.Load(); n != 1 { + t.Errorf("worker checkpointed the actor %d times, want once", n) + } + if _, err := os.Stat(leftover); !os.IsNotExist(err) { + t.Errorf("leftover in the bundle dir after the retry: stat = %v, want it cleaned up", err) + } + st := resp.GetState() + if got := st.GetLatestLocal().GetSnapshot(); proto.Equal(got, snap) != tc.wantLocal { + t.Errorf("latest_local = %v, want local = %v", got, tc.wantLocal) + } + if got := st.GetLatestDurable().GetSnapshot(); proto.Equal(got, snap) != tc.wantDurable { + t.Errorf("latest_durable = %v, want durable = %v", got, tc.wantDurable) + } + if got := slices.Contains(a.store.keys(), manifestKey("snap-1")); got != tc.wantDurable { + t.Errorf("snapshot uploaded = %v, want %v", got, tc.wantDurable) + } + if got := a.s.snapshots.State(pluginActorUID); !proto.Equal(got, st) { + t.Errorf("tracked state = %v, want %v, as the retry returned", got, st) + } + var rm metricdata.ResourceMetrics + if err := reader.Collect(t.Context(), &rm); err != nil { + t.Fatalf("collect: %v", err) + } + if len(rm.ScopeMetrics) != 0 { + t.Error("the retry recorded a checkpoint duration, want none") + } + }) + } } // TestTerminateWaitsForInFlightRPC checks that Terminate waits on the @@ -1296,15 +1593,16 @@ func TestTerminateWaitsForInFlightRPC(t *testing.T) { } // TestFailedCheckpointKeepsLocalSnapshot checkpoints a resumed actor again, and -// fails before the new snapshot commits: the pause snapshot stays on disk and +// fails before the new snapshot commits: the local snapshot stays on disk and // recorded as the actor's latest local snapshot, so the actor can still be // restored from it. func TestFailedCheckpointKeepsLocalSnapshot(t *testing.T) { for _, tc := range []struct { - name string - snap string - mode ateletpb.UploadMode - fail func(t *testing.T, a *pluginActor) + name string + snap string + mode ateletpb.UploadMode + fail func(t *testing.T, a *pluginActor) + wantNoNewSnap bool }{ { name: "suspend upload fails", @@ -1325,6 +1623,22 @@ func TestFailedCheckpointKeepsLocalSnapshot(t *testing.T) { } }, }, + { + name: "missing snapshot file", + snap: "pause-snap-2", + mode: ateletpb.UploadMode_UPLOAD_MODE_DEFERRED, + fail: func(t *testing.T, a *pluginActor) { + var once sync.Once + a.worker.onCheckpoint = func() { + once.Do(func() { + if err := os.Remove(filepath.Join(ateletpath.CheckpointStateDir(pluginActorUID), "checkpoint.img")); err != nil { + t.Error(err) + } + }) + } + }, + wantNoNewSnap: true, + }, } { t.Run(tc.name, func(t *testing.T) { a := runPluginActor(t) @@ -1337,6 +1651,11 @@ func TestFailedCheckpointKeepsLocalSnapshot(t *testing.T) { if _, err := a.checkpoint(t, a.snapshot(tc.snap, 2), tc.mode); err == nil { t.Fatal("Checkpoint succeeded, want it to fail") } + if tc.wantNoNewSnap { + if _, err := os.Stat(ateletpath.LocalSnapshotDir(pluginActorUID, tc.snap)); !os.IsNotExist(err) { + t.Errorf("failed move created local snapshot %q: stat = %v, want not exist", tc.snap, err) + } + } if _, err := os.Stat(ateletpath.LocalSnapshotDir(pluginActorUID, "pause-snap-1")); err != nil { t.Errorf("pause snapshot after the failed checkpoint: stat = %v, want it kept", err) } @@ -1344,14 +1663,17 @@ func TestFailedCheckpointKeepsLocalSnapshot(t *testing.T) { t.Errorf("latest_local = %q, want pause-snap-1", got) } a.restoreLocal(t, "pause-snap-1", 1) + if entries, err := os.ReadDir(ateletpath.CheckpointStateDir(pluginActorUID)); err != nil || len(entries) != 0 { + t.Errorf("checkpoint-state after restore: entries = %v, err = %v, want empty", entries, err) + } }) } } -// TestPromoteSnapshotRecordsUpload suspends a paused actor: the promote -// records the uploaded snapshot as the latest durable one, with the pause's -// commit time, and keeps the local copy on the node. A retry reports the same -// state. +// TestPromoteSnapshotRecordsUpload promotes a suspended actor's local snapshot: +// the promote records the uploaded snapshot as the latest durable one, with the +// checkpoint's commit time, and keeps the local copy on the node. A retry +// reports the same state. func TestPromoteSnapshotRecordsUpload(t *testing.T) { a := runPluginActor(t) snap := a.snapshot("pause-snap-1", 1) @@ -1417,7 +1739,7 @@ func TestConcurrentRunsShareOneAssetDownload(t *testing.T) { systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } - sandboxRec := testSandboxRecord(release, pinnedImage) + sandboxRec := testSandboxRecord(release) spec := &ateletpb.WorkloadSpec{ Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, } @@ -1530,7 +1852,7 @@ func TestTrackerStoreUploadLocalSnapshot(t *testing.T) { } st := &trackerStore{s: a.s} - // An upload asking for a fidelity other than the pause captured is + // An upload asking for a fidelity other than the checkpoint captured is // rejected before uploading. noFidelity := proto.CloneOf(snap) noFidelity.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED @@ -1555,7 +1877,7 @@ func TestTrackerStoreUploadLocalSnapshot(t *testing.T) { } } -// TestRestoredActorIsNotDetached restores a paused actor from its local copy: +// TestRestoredActorIsNotDetached restores an actor from its local copy: // once the actor runs on the node, atelet refuses to detach it. func TestRestoredActorIsNotDetached(t *testing.T) { a := runPluginActor(t) @@ -1569,9 +1891,10 @@ func TestRestoredActorIsNotDetached(t *testing.T) { } } -// TestFailedRestoreUnschedulesDetach fails a restore of a paused actor after -// it began: as after a restore that succeeds, atelet keeps the actor's local -// snapshot, and does not detach the actor until its next checkpoint. +// TestFailedRestoreUnschedulesDetach fails a restore of an actor from its local +// snapshot after it began: as after a restore that succeeds, atelet keeps the +// actor's local snapshot, and does not detach the actor until its next +// checkpoint. func TestFailedRestoreUnschedulesDetach(t *testing.T) { a := runPluginActor(t) a.s.snapshots = snapshottracker.New(t.Context(), &trackerStore{s: a.s}, clock.RealClock{}) @@ -1602,7 +1925,7 @@ func TestFailedRestoreUnschedulesDetach(t *testing.T) { } } -// TestRestoreAfterDetachReadsUpload resumes a paused actor that atelet +// TestRestoreAfterDetachReadsUpload resumes a suspended actor that atelet // detached before the control plane recorded the upload, so the control plane // still sends the Restore here: atelet restores the actor from the upload the // request names. @@ -1647,9 +1970,9 @@ func detachActorRequest() *ateletpb.DetachActorRequest { return &ateletpb.DetachActorRequest{Atespace: "ate-demo", ActorName: "counter", ActorUid: pluginActorUID} } -// TestDetachActorDetachesPausedActor detaches a paused actor through the RPC: -// atelet uploads its local snapshot, deletes the local copy, and returns the -// new state. +// TestDetachActorDetachesPausedActor detaches a suspended actor through the +// RPC: atelet uploads its local snapshot, deletes the local copy, and returns +// the new state. func TestDetachActorDetachesPausedActor(t *testing.T) { a := runPluginActor(t) a.s.snapshots = snapshottracker.New(t.Context(), &trackerStore{s: a.s}, clock.RealClock{}) @@ -1802,8 +2125,8 @@ func (s *blockingTrackerStore) UploadLocalSnapshot(ctx context.Context, _ string return false, ctx.Err() } -// startBlockedDetach pauses the actor as pause-snap-1, and starts a Detach of -// it whose upload blocks until the test ends. +// startBlockedDetach checkpoints the actor locally as pause-snap-1, and starts +// a Detach of it whose upload blocks until the test ends. func startBlockedDetach(t *testing.T, a *pluginActor) { t.Helper() store := &blockingTrackerStore{started: make(chan struct{}, 1)} @@ -1820,6 +2143,12 @@ func startBlockedDetach(t *testing.T, a *pluginActor) { t.Cleanup(func() { cancel() <-detached + // The detach outlives the call, until the tracker's context ends. + waitCtx, cancelWait := context.WithTimeout(context.Background(), 10*time.Second) + defer cancelWait() + if err := a.s.snapshots.Wait(waitCtx); err != nil { + t.Errorf("detach still running after the test: %v", err) + } }) select { case <-store.started: @@ -1885,10 +2214,10 @@ func TestPromoteSnapshotRecordsDetachWait(t *testing.T) { } } -// TestPromoteSnapshotFoundAlreadyUploaded promotes a paused actor's snapshot -// whose upload committed before the promote, unrecorded, and whose local copy -// is gone since: the promote records the snapshot as durable, without an -// upload time, since it wrote nothing. +// TestPromoteSnapshotFoundAlreadyUploaded promotes an asynchronously suspended +// actor's snapshot whose upload committed before the promote, unrecorded, and +// whose local copy is gone since: the promote records the snapshot as durable, +// without an upload time, since it wrote nothing. func TestPromoteSnapshotFoundAlreadyUploaded(t *testing.T) { a := runPluginActor(t) snap := a.snapshot("pause-snap-1", 1) @@ -1909,10 +2238,9 @@ func TestPromoteSnapshotFoundAlreadyUploaded(t *testing.T) { // testSandboxRecord is the record of a gVisor runtime whose only asset is // release, as the syncer plans it from a SandboxConfig version. -func testSandboxRecord(release gvisorTestRelease, pauseImage string) *sandboxAssetsRecord { +func testSandboxRecord(release gvisorTestRelease) *sandboxAssetsRecord { return &sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: pauseImage, Assets: map[string]assetEntry{gvisorAssetName: release.entry}, } } @@ -1926,10 +2254,7 @@ func advertiseRuntime(t *testing.T, s *Atelet, rec *sandboxAssetsRecord) { if s.runtimes == nil { s.runtimes = newRuntimeIndex() } - hash, err := versionBundleHash(rec) - if err != nil { - t.Fatalf("versionBundleHash: %v", err) - } + hash := bundleHash(rec) s.runtimes.register("test-worker-pod", advertisedClass{ class: v1alpha1.SandboxClass(rec.SandboxClass), config: "test-config", @@ -1955,14 +2280,14 @@ func TestRunAndRestoreRejectUnadvertisedRuntime(t *testing.T) { anonGCSClient: fakeObjectStorage{data: release.tarball, gets: &gets}, systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), } - rec := testSandboxRecord(release, testPauseImage) + rec := testSandboxRecord(release) for _, tc := range []struct { name string - call func(t *testing.T, rt *ateletpb.SandboxRuntime) error + call func(t *testing.T, rt *ateletpb.ActorSandboxRuntime) error }{{ name: "Run", - call: func(t *testing.T, rt *ateletpb.SandboxRuntime) error { + call: func(t *testing.T, rt *ateletpb.ActorSandboxRuntime) error { _, err := s.Run(t.Context(), &ateletpb.RunRequest{ Atespace: "team-a", ActorName: "actor-1", ActorUid: actorUID, WorkerPodUid: workerPodUID, SandboxRuntime: rt, Spec: &ateletpb.WorkloadSpec{}, @@ -1971,7 +2296,7 @@ func TestRunAndRestoreRejectUnadvertisedRuntime(t *testing.T) { }, }, { name: "Restore", - call: func(t *testing.T, rt *ateletpb.SandboxRuntime) error { + call: func(t *testing.T, rt *ateletpb.ActorSandboxRuntime) error { _, err := s.Restore(t.Context(), &ateletpb.RestoreRequest{ Atespace: "team-a", ActorName: "actor-1", ActorUid: actorUID, WorkerPodUid: workerPodUID, SandboxRuntime: rt, Spec: &ateletpb.WorkloadSpec{}, @@ -2006,16 +2331,13 @@ func TestRunAndRestoreRejectUnadvertisedRuntime(t *testing.T) { } } -// runtimeFor returns the SandboxRuntime ate-api-server sends for an actor +// runtimeFor returns the ActorSandboxRuntime ate-api-server sends for an actor // placed on a worker that advertises the runtime rec describes: the worker's // registration, whose sandbox_bundle_hash is the hash of rec. -func runtimeFor(t *testing.T, rec *sandboxAssetsRecord) *ateletpb.SandboxRuntime { +func runtimeFor(t *testing.T, rec *sandboxAssetsRecord) *ateletpb.ActorSandboxRuntime { t.Helper() - hash, err := versionBundleHash(rec) - if err != nil { - t.Fatalf("versionBundleHash: %v", err) - } - return &ateletpb.SandboxRuntime{ + hash := bundleHash(rec) + return &ateletpb.ActorSandboxRuntime{ SandboxClass: rec.SandboxClass, Version: &ateletpb.VersionedSandboxCompat{ SchemaVersion: hardware.SchemaVersionV1, @@ -2028,10 +2350,15 @@ func runtimeFor(t *testing.T, rec *sandboxAssetsRecord) *ateletpb.SandboxRuntime } type gatingTrackerStore struct { - inner snapshottracker.Store + inner snapshottracker.Store + // objects, if set, fails every upload after the first gated one. objects *recordingObjectStorage started chan struct{} release chan struct{} + // canceled, if set, receives once an upload's context ends, and that + // upload still waits for release before it returns, as an object-store + // call that is finishing would. + canceled chan struct{} } func (s *gatingTrackerStore) UploadLocalSnapshot(ctx context.Context, actorUID string, ref snapshottracker.SnapshotRef) (bool, error) { @@ -2039,9 +2366,16 @@ func (s *gatingTrackerStore) UploadLocalSnapshot(ctx context.Context, actorUID s select { case <-s.release: case <-ctx.Done(): + if s.canceled != nil { + s.canceled <- struct{}{} + <-s.release + } return false, ctx.Err() } uploaded, err := s.inner.UploadLocalSnapshot(ctx, actorUID, ref) + if s.objects == nil { + return uploaded, err + } // Fail any second upload attempt so PromoteSnapshot proves it skips // re-uploading once the background upload made the snapshot durable. s.objects.mu.Lock() @@ -2132,8 +2466,8 @@ func TestPromoteSnapshotWaitsForBackgroundUpload(t *testing.T) { } } -// TestCheckpointKeepsSnapshotBeingUploaded pauses a resumed actor while the -// background upload of its previous pause snapshot still runs: the +// TestCheckpointKeepsSnapshotBeingUploaded checkpoints a resumed actor locally +// while the background upload of its previous local snapshot still runs: the // checkpoint's prune keeps that snapshot for the upload, which deletes it once // it ends. func TestCheckpointKeepsSnapshotBeingUploaded(t *testing.T) { @@ -2182,6 +2516,228 @@ func TestCheckpointKeepsSnapshotBeingUploaded(t *testing.T) { } } +// TestImmediateCheckpointCancelsBackgroundUpload suspends a resumed actor with +// an immediate upload while the background upload of its previous local +// snapshot still runs: the checkpoint checkpoints the worker without waiting +// for that upload, then cancels it, and returns only once it has stopped, so +// the control plane can delete the older snapshot's objects. +func TestImmediateCheckpointCancelsBackgroundUpload(t *testing.T) { + a := runPluginActor(t) + clk := clocktesting.NewFakeClock(time.Now()) + store := &gatingTrackerStore{ + inner: &trackerStore{s: a.s}, + started: make(chan struct{}, 1), + release: make(chan struct{}), + canceled: make(chan struct{}, 1), + } + a.s.snapshots = snapshottracker.New(t.Context(), store, clk, + snapshottracker.WithUploadDelay(time.Second), + snapshottracker.WithDetachDelay(time.Hour)) + a.s.snapshots.Start() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if err := a.s.snapshots.Wait(ctx); err != nil { + t.Errorf("Wait: %v", err) + } + }) + // A canceled upload holds on until released; release it before Wait + // even when the test stops early. + release := sync.OnceFunc(func() { close(store.release) }) + t.Cleanup(release) + + cpResp, err := a.checkpoint(t, a.snapshot("pause-snap-1", 1), ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + if err != nil { + t.Fatalf("Checkpoint (pause-snap-1): %v", err) + } + stepToUpload(t, clk, cpResp.GetState().GetUploadDueTime().AsTime(), store.started) + a.restoreLocal(t, "pause-snap-1", 1) + a.worker.mu.Lock() + delete(a.worker.actorDirs, "CheckpointWorkload") + a.worker.mu.Unlock() + + type checkpointResult struct { + state *ateletpb.ActorSnapshotState + err error + } + resCh := make(chan checkpointResult, 1) + go func() { + resp, err := a.checkpoint(t, a.snapshot("suspend-snap-2", 2), ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE) + resCh <- checkpointResult{resp.GetState(), err} + }() + select { + case <-store.canceled: + case res := <-resCh: + t.Fatalf("immediate Checkpoint returned (%v, %v) without canceling the background upload", res.state, res.err) + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for the immediate Checkpoint to cancel the background upload") + } + a.worker.mu.Lock() + _, checkpointed := a.worker.actorDirs["CheckpointWorkload"] + a.worker.mu.Unlock() + if !checkpointed { + t.Error("background upload canceled before the worker was checkpointed, want the checkpoint not to wait for it") + } + select { + case res := <-resCh: + t.Fatalf("immediate Checkpoint returned (%v, %v) before the canceled upload stopped", res.state, res.err) + case <-time.After(50 * time.Millisecond): + } + + release() + var res checkpointResult + select { + case res = <-resCh: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for the immediate Checkpoint once the canceled upload stopped") + } + if res.err != nil { + t.Fatalf("Checkpoint (suspend-snap-2): %v", res.err) + } + if got := res.state.GetLatestDurable().GetSnapshot().GetSnapshotUuid(); got != "suspend-snap-2" { + t.Errorf("latest_durable = %q, want suspend-snap-2", got) + } + if res.state.GetUploading() != nil { + t.Errorf("uploading = %v, want no upload in progress once the checkpoint returns", res.state.GetUploading()) + } +} + +// resumedWithPendingUpload runs a plugin actor, checkpoints it into pause-snap-1 +// with a deferred upload, and resumes it from that snapshot's local copy +// before the upload is due. It returns the actor, its gated tracker store, and +// a func that steps the clock past the upload's due time and reports whether +// the upload starts within a moment. +func resumedWithPendingUpload(t *testing.T) (*pluginActor, *gatingTrackerStore, func() bool) { + t.Helper() + a := runPluginActor(t) + clk := clocktesting.NewFakeClock(time.Now()) + store := &gatingTrackerStore{ + inner: &trackerStore{s: a.s}, + started: make(chan struct{}, 1), + release: make(chan struct{}), + } + a.s.snapshots = snapshottracker.New(t.Context(), store, clk, + snapshottracker.WithUploadDelay(time.Second), + snapshottracker.WithDetachDelay(time.Hour)) + a.s.snapshots.Start() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if err := a.s.snapshots.Wait(ctx); err != nil { + t.Errorf("Wait: %v", err) + } + }) + // An upload that starts holds on until released; release it before Wait. + t.Cleanup(func() { close(store.release) }) + + cpResp, err := a.checkpoint(t, a.snapshot("pause-snap-1", 1), ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + if err != nil { + t.Fatalf("Checkpoint (pause-snap-1): %v", err) + } + due := cpResp.GetState().GetUploadDueTime().AsTime() + a.restoreLocal(t, "pause-snap-1", 1) + // Step on in one-second steps past the due time, in case the queue's timer + // was armed late (see stepToUpload). + stepPastDue := func() bool { + clk.Step(due.Sub(clk.Now())) + for range 5 { + select { + case <-store.started: + return true + case <-time.After(20 * time.Millisecond): + clk.Step(time.Second) + } + } + return false + } + return a, store, stepPastDue +} + +// TestImmediateCheckpointHoldsOffBackgroundUpload checkpoints an actor resumed +// from its local snapshot while that snapshot's background upload comes due: +// under UPLOAD_MODE_IMMEDIATE the upload does not start, since the checkpoint +// would only cancel it, while under UPLOAD_MODE_DEFERRED it starts. +func TestImmediateCheckpointHoldsOffBackgroundUpload(t *testing.T) { + for _, tc := range []struct { + mode ateletpb.UploadMode + snapshot string + wantStarted bool + }{ + {mode: ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE, snapshot: "suspend-snap-2"}, + {mode: ateletpb.UploadMode_UPLOAD_MODE_DEFERRED, snapshot: "pause-snap-2", wantStarted: true}, + } { + t.Run(tc.mode.String(), func(t *testing.T) { + a, _, stepPastDue := resumedWithPendingUpload(t) + var started atomic.Bool + a.worker.onCheckpoint = func() { started.Store(stepPastDue()) } + + if _, err := a.checkpoint(t, a.snapshot(tc.snapshot, 2), tc.mode); err != nil { + t.Fatalf("Checkpoint (%s): %v", tc.snapshot, err) + } + if got := started.Load(); got != tc.wantStarted { + t.Errorf("background upload of pause-snap-1 started during the checkpoint = %v, want %v", got, tc.wantStarted) + } + }) + } +} + +// TestTerminateHoldsOffBackgroundUpload terminates an actor resumed from its +// local snapshot, whose background upload comes due while the worker tears the +// actor down: the upload does not start, since Terminate would only cancel it. +// When the worker fails to tear the actor down, Terminate fails before it +// forgets the actor, and the upload, still due, starts once Terminate returns. +func TestTerminateHoldsOffBackgroundUpload(t *testing.T) { + for _, tc := range []struct { + name string + // teardownErr is what the worker's TerminateWorkload fails with. + teardownErr error + }{ + {name: "Terminate succeeds"}, + {name: "Terminate fails", teardownErr: status.Error(codes.Internal, "teardown failed")}, + } { + t.Run(tc.name, func(t *testing.T) { + a, store, stepPastDue := resumedWithPendingUpload(t) + var started atomic.Bool + a.worker.onTerminate = func() error { + started.Store(stepPastDue()) + return tc.teardownErr + } + + _, err := a.s.Terminate(t.Context(), &ateletpb.TerminateRequest{ + Atespace: "ate-demo", + ActorName: "counter", + ActorUid: pluginActorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + WorkerPodUid: pluginWorkerPodUID, + Spec: a.spec, + }) + switch { + case tc.teardownErr == nil && err != nil: + t.Fatalf("Terminate: %v", err) + case tc.teardownErr != nil && err == nil: + t.Fatal("Terminate succeeded, want it to fail with the worker's teardown") + } + if started.Load() { + t.Error("background upload of pause-snap-1 started during Terminate, want none") + } + if tc.teardownErr == nil { + select { + case <-store.started: + t.Error("background upload of pause-snap-1 started during Terminate, want none") + default: + } + return + } + select { + case <-store.started: + case <-time.After(10 * time.Second): + t.Fatal("background upload of pause-snap-1 did not start once Terminate failed") + } + }) + } +} + // TestCheckpointRejectsStaleGeneration checks that Checkpoint fails with // Aborted when given a generation below the actor's newest snapshot // or equal to it under another snapshot_uuid, before calling worker, pruning @@ -2240,7 +2796,6 @@ func TestPromoteSnapshotRejectsStaleGeneration(t *testing.T) { } { writeLocalSnapshot(t, ateletpath.LocalSnapshotDir(pluginActorUID, stale.GetSnapshotUuid()), sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: a.rec.PauseImage, SnapshotFiles: []string{"checkpoint.img"}, Fidelity: "MEMORY", }, map[string]string{"checkpoint.img": "stale-memory"}) @@ -2374,7 +2929,6 @@ func TestCheckpointSkipsStaleCommitAndKeepsNewerLocal(t *testing.T) { snap3Dir := ateletpath.LocalSnapshotDir(pluginActorUID, snap3.GetSnapshotUuid()) writeLocalSnapshot(t, snap3Dir, sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: a.rec.PauseImage, SnapshotFiles: []string{"checkpoint.img"}, Fidelity: "MEMORY", }, map[string]string{"checkpoint.img": "newer-memory"}) diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 59fc49d785..612b5a9d09 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -53,7 +53,6 @@ import ( "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/internal/installdefaults" "go.ate.dev/substrate/internal/nodepath" - "go.ate.dev/substrate/internal/ocispec" "go.ate.dev/substrate/internal/otlprelay" "go.ate.dev/substrate/internal/pluginsocket" "go.ate.dev/substrate/internal/preview" @@ -76,7 +75,6 @@ import ( "go.opentelemetry.io/otel/metric" semconv "go.opentelemetry.io/otel/semconv/v1.40.0" "golang.org/x/sync/errgroup" - "golang.org/x/sys/unix" "google.golang.org/api/option" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -131,8 +129,8 @@ var ( objectStorePluginSocket = pflag.String("objectstore-plugin-socket", "/run/objectstore-plugin/node.sock", "Unix socket of the node object-store plugin that external snapshots are fetched and uploaded through, and every sandbox asset is fetched through. A gs:// asset the plugin cannot serve is then read anonymously.") snapshotUploadDelay = pflag.Duration("snapshot-upload-delay", snapshottracker.DefaultUploadDelay, "How long after an actor's oldest snapshot not yet in object storage was taken atelet uploads the actor's latest local snapshot in the background; later while an upload is in progress or failing.") snapshotDetachDelay = pflag.Duration("snapshot-detach-delay", snapshottracker.DefaultDetachDelay, "How long an actor stays suspended on this node, its latest snapshot kept locally, before atelet detaches it in the background.") - snapshotUploadTimeout = pflag.Duration("snapshot-upload-timeout", snapshottracker.DefaultUploadTimeout, "How long one background snapshot upload, including the one a background detach runs first, may run before atelet cancels and retries it, on top of the time the snapshot takes at --snapshot-upload-est-throughput-mib. Each consecutive failure doubles the next attempt's timeout, to at most 8 times the first's. 0 turns the timeout off.") - snapshotUploadEstMiBps = pflag.Int("snapshot-upload-est-throughput-mib", snapshottracker.DefaultUploadEstThroughput>>20, "Estimated background snapshot upload throughput, in MiB per second of the snapshot's allocated size: each upload attempt may run --snapshot-upload-timeout plus the time the snapshot takes at this rate. 0 adds no time for the snapshot's size.") + snapshotUploadTimeout = pflag.Duration("snapshot-upload-timeout", snapshottracker.DefaultUploadTimeout, "How long one snapshot upload, background or part of a detach, may run before atelet cancels and retries it, on top of the time the snapshot takes at --snapshot-upload-est-throughput-mib. Each consecutive failure doubles the next attempt's timeout, to at most 8 times the first's. 0 turns the timeout off.") + snapshotUploadEstMiBps = pflag.Int("snapshot-upload-est-throughput-mib", snapshottracker.DefaultUploadEstThroughput>>20, "Estimated snapshot upload throughput, in MiB per second of the snapshot's allocated size: each upload attempt may run --snapshot-upload-timeout plus the time the snapshot takes at this rate. 0 adds no time for the snapshot's size.") snapshotUploadWorkers = pflag.Int("snapshot-upload-workers", snapshottracker.DefaultUploadWorkers, "How many background snapshot uploads atelet runs at once.") snapshotDetachWorkers = pflag.Int("snapshot-detach-workers", snapshottracker.DefaultDetachWorkers, "How many background detaches atelet runs at once.") snapshotReportWorkers = pflag.Int("snapshot-report-workers", snapshottracker.DefaultReportWorkers, "How many background snapshot state reports atelet runs at once. 0 disables background reporting.") @@ -283,6 +281,18 @@ func main() { if err := filecachegc.Start(ctx, ateattr.FileCacheSandboxAsset, sandboxAssetCache, sandboxAssetCacheGCFlags, fileCacheInstruments); err != nil { serverboot.Fatal(ctx, "Failed to start the sandbox asset cache GC", err) } + if err := validateSnapshotCacheFlags(); err != nil { + serverboot.Fatal(ctx, "Invalid snapshot cache flags", err) + } + snapshotCache, err := openSnapshotCache(ctx, *snapshotCacheDir, *snapshotCacheMinAge, *snapshotCacheFetchTimeout) + if err != nil { + serverboot.Fatal(ctx, "Failed to open snapshot cache", err) + } + if snapshotCache != nil { + if err := filecachegc.Start(ctx, ateattr.FileCacheSnapshot, snapshotCache, snapshotCacheGCFlags, fileCacheInstruments); err != nil { + serverboot.Fatal(ctx, "Failed to start the snapshot cache GC", err) + } + } // Sandbox assets go through the object-store plugin; a gs:// asset it // cannot serve, such as one in the public gs://gvisor bucket, is read @@ -352,6 +362,7 @@ func main() { } var ateapiConn *grpc.ClientConn + var workerService ateapipb.WorkerServiceClient var reporter snapshottracker.Reporter if !*grpcInsecure { dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ @@ -371,7 +382,8 @@ func main() { serverboot.Fatal(ctx, "Failed to create ateapi client", err) } defer ateapiConn.Close() - reporter = &trackerReporter{workers: ateapipb.NewWorkerServiceClient(ateapiConn)} + workerService = ateapipb.NewWorkerServiceClient(ateapiConn) + reporter = &trackerReporter{workers: workerService} } // The snapshot tracker's workers get a context of their own, cancelled only @@ -407,12 +419,14 @@ func main() { *objectStorePluginSocket, imageCache, sandboxAssetCache, + snapshotCache, runtimes, instruments, volPlugins, csiDriverConfigGetter, systemInfoVolumes, snapshots, + workerService, ) store.s = wmService snapshots.Start() @@ -466,7 +480,7 @@ func main() { workerFacingSrv := grpc.NewServer(grpc.Creds(credentials.NewTLS(workerFacingTLS))) ateletpb.RegisterWorkerSupportServer(workerFacingSrv, &workerSupportServer{ - workers: ateapipb.NewWorkerServiceClient(ateapiConn), + workers: workerService, control: ateapipb.NewControlClient(ateapiConn), }) go func() { @@ -502,7 +516,7 @@ func main() { // the worker, so the first Run/Restore on this node hits the cache. // Best-effort: on failure the on-demand fetch in ensureSandboxAssets // still covers correctness. - prewarmer := newRuntimePrewarmer(runtimes, wmService, imageCache) + prewarmer := newRuntimePrewarmer(runtimes, wmService) go prewarmer.run(ctx) // Keep a sandbox runtimes file in each local worker's directory: the // runtimes of the enabled versions of its pool's SandboxConfig, @@ -688,6 +702,14 @@ type Atelet struct { // sandboxAssetCache holds every sandbox asset on the node once; actors get hard // links into their own directories (see ensureSandboxAssets). sandboxAssetCache *filecache.Store + // snapshotCache dedupes and retains shared snapshot files across + // restores. nil means caching is disabled (--snapshot-cache-dir=""): + // every restore downloads its snapshot files directly. + snapshotCache *filecache.Store + // snapshotCacheCrossMount warns once per process when a cache hit + // cannot be linked into a restore dir; every later hit says so at + // debug. + snapshotCacheCrossMount sync.Once // runtimes resolves the runtime a Run or Restore names to the assets to // stage, among those this node's workers advertise. runtimes *runtimeIndex @@ -713,6 +735,8 @@ type Atelet struct { snapshots *snapshottracker.Tracker // actorLocks serializes lifecycle RPCs per actor. actorLocks actorlock.Locks + // workerActorJWTs mints the JWT every snapshot transfer carries. + workerActorJWTs *workerActorJWTs } var _ ateletpb.AteletServer = (*Atelet)(nil) @@ -736,17 +760,20 @@ func NewService( pluginSocket string, imageCache *imagecache.Store, sandboxAssetCache *filecache.Store, + snapshotCache *filecache.Store, runtimes *runtimeIndex, instruments *Instruments, volumePlugins map[string]volume.VolumePluginWorkerPlane, csiDriverConfigGetter csi.CSIDriverConfigGetter, systemInfoVolumes *systemInfoVolumeRefresher, snapshots *snapshottracker.Tracker, + workers ateapipb.WorkerServiceClient, ) *Atelet { wms := &Atelet{ workerDialer: workerDialer, imageCache: imageCache, sandboxAssetCache: sandboxAssetCache, + snapshotCache: snapshotCache, runtimes: runtimes, anonGCSClient: anonGCSClient, objectStorePlugin: objectStorePlugin, @@ -758,6 +785,7 @@ func NewService( csiDriverConfigGetter: csiDriverConfigGetter, systemInfoVolumes: systemInfoVolumes, snapshots: snapshots, + workerActorJWTs: newWorkerActorJWTs(workers), } return wms } @@ -770,10 +798,9 @@ func (s *Atelet) Run(ctx context.Context, req *ateletpb.RunRequest) (resp *atele actorUID := req.GetActorUid() actorRef := resources.ActorRef{Atespace: req.GetAtespace(), Name: req.GetActorName()} - // The sandbox (binaries + pause image) is the runtime the control plane - // assigned the actor to, resolved through the index. Resolved before any - // on-node work, BeginStart included, so an invalid request changes - // nothing. + // The sandbox binaries are the runtime the control plane assigned the + // actor to, resolved through the index. Resolved before any on-node work, + // BeginStart included, so an invalid request changes nothing. sandboxRec, err := s.advertisedRuntime(ctx, req.GetSandboxRuntime()) if err != nil { return nil, err @@ -815,7 +842,7 @@ func (s *Atelet) Run(ctx context.Context, req *ateletpb.RunRequest) (resp *atele }() s.systemInfoVolumes.Deregister(actorUID) - if err := resetActorDirs(actorUID); err != nil { + if err := resetActorDirsKeepingVolumes(actorUID); err != nil { return nil, fmt.Errorf("while resetting actor dirs: %w", err) } @@ -843,9 +870,7 @@ func (s *Atelet) Run(ctx context.Context, req *ateletpb.RunRequest) (resp *atele if registration, err = s.systemInfoVolumes.Register(actorUID, actorRef, systemInfoVolumesFor(actorUID, req.GetSpec())); err != nil { return nil, err } - containerSpecs, err := s.prepareOCIBundles(ctx, actorUID, actorRef, - req.GetSpec(), sandboxRec.PauseImage, - ) + containerSpecs, err := s.prepareOCIBundles(ctx, actorUID, actorRef, req.GetSpec()) if err != nil { return nil, err } @@ -923,6 +948,9 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest // reports the phases it completed. Phases left at zero never ran. tStart := time.Now() var dAssets, dWorker, dPersist, dCleanup time.Duration + // committed is set when BeginCheckpoint finds this Checkpoint is a retry + // of one that committed its snapshot. + var committed bool op := snapshotOp{ templateNamespace: req.GetActorTemplateAtespace(), templateName: req.GetActorTemplateName(), @@ -946,7 +974,11 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest {ateattr.SnapshotPhaseCleanup, dCleanup}, {ateattr.SnapshotPhaseTotal, time.Since(tStart)}, } - s.instruments.recordCheckpoint(ctx, op, phases...) + // A retry answered from the committed snapshot takes no checkpoint, + // so it is only logged. + if !committed { + s.instruments.recordCheckpoint(ctx, op, phases...) + } slog.LogAttrs(ctx, slog.LevelInfo, "Checkpoint timing breakdown", snapshotLogAttrs(attribution, op, checkpointDurationMetric, err, phases)...) }() @@ -959,11 +991,35 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest // A detach in progress finishes first, and no detach starts until the // checkpoint returns, so it cannot delete the snapshot being written. - endCheckpoint, err := s.snapshots.BeginCheckpoint(ctx, actorUID, req.GetSnapshot()) + // Under UPLOAD_MODE_IMMEDIATE, no background upload starts either, so + // none competes with this checkpoint's upload. + committed, endCheckpoint, err := s.snapshots.BeginCheckpoint(ctx, actorUID, req.GetSnapshot(), req.GetUploadConfig().GetMode()) if err != nil { return nil, err } defer endCheckpoint() + if committed { + // A retry of a Checkpoint that committed the snapshot. That one + // deleted the workload, so checkpointing again would fail. + state, err := s.committedCheckpointState(ctx, req, &op, &dPersist) + if err != nil { + return nil, err + } + // Finish the cleanup that one may have left undone, unless a Run or + // Restore of the actor has begun since it committed the snapshot: + // the actor's directories and volumes may belong to a later + // activation now. None begins until this Checkpoint ends, so the + // check holds through the cleanup. + if s.snapshots.NotStartedSince(actorUID, req.GetSnapshot()) { + tCleanup := time.Now() + err = s.cleanUpAfterCheckpoint(ctx, req) + dCleanup = time.Since(tCleanup) + if err != nil { + return nil, err + } + } + return &ateletpb.CheckpointResponse{State: state}, nil + } // Checkpoint requests no longer carry the sandbox config; recover the // version this actor was started with from the on-node record and re-fetch @@ -1055,8 +1111,8 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest durable.UploadTime = time.Now() state = s.snapshots.CommitDurable(actorUID, durable) // The snapshot is durable now, so the local copy only saves a - // download: failing to keep it must not fail the checkpoint. The prune - // below deletes a partial copy. + // download: failing to keep it must not fail the checkpoint. + // resetActorDirs cleans up checkpointDir if the move fails. if err := s.moveLocalCheckpoint(ctx, req, sandboxRec); err != nil { slog.WarnContext(ctx, "Failed to keep a local copy of the uploaded snapshot", slog.Any("actor", actorRef), slog.Any("err", err)) } else { @@ -1067,6 +1123,48 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest } dPersist = time.Since(tPersist) + tCleanup := time.Now() + err = s.cleanUpAfterCheckpoint(ctx, req) + dCleanup = time.Since(tCleanup) + if err != nil { + return nil, err + } + return &ateletpb.CheckpointResponse{State: state}, nil +} + +// committedCheckpointState returns the actor's state for a retry of a +// Checkpoint that committed req's snapshot, canceling any pending background +// report. Under UPLOAD_MODE_IMMEDIATE it first makes the snapshot durable, as +// PromoteSnapshot does, since the Checkpoint that committed it may have +// deferred its upload. It records in op and dPersist what promoteLocalSnapshot +// records. +func (s *Atelet) committedCheckpointState(ctx context.Context, req *ateletpb.CheckpointRequest, op *snapshotOp, dPersist *time.Duration) (*ateletpb.ActorSnapshotState, error) { + if req.GetUploadConfig().GetMode() == ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE { + uri, err := resources.ParseSnapshotURI(req.GetSnapshot().GetObject().GetSnapshotUri()) + if err != nil { + return nil, apierror.InvalidArgument("%v", err) + } + if _, err := s.promoteLocalSnapshot(ctx, &ateletpb.PromoteSnapshotRequest{ + Atespace: req.GetAtespace(), + ActorName: req.GetActorName(), + ActorUid: req.GetActorUid(), + ActorTemplateAtespace: req.GetActorTemplateAtespace(), + ActorTemplateName: req.GetActorTemplateName(), + Snapshot: req.GetSnapshot(), + }, uri, op, dPersist); err != nil { + return nil, err + } + } + return s.snapshots.CommittedCheckpointState(req.GetActorUid()), nil +} + +// cleanUpAfterCheckpoint finishes a Checkpoint that has committed its +// snapshot: it prunes the actor's local snapshots, unmounts its external +// volumes, and empties its directories. Each step is idempotent, so a retried +// Checkpoint runs them again. +func (s *Atelet) cleanUpAfterCheckpoint(ctx context.Context, req *ateletpb.CheckpointRequest) error { + actorUID := req.GetActorUid() + // Delete the actor's earlier local snapshots only now that this checkpoint // has committed its own. A checkpoint that fails before then leaves the // previous local snapshot in place and recorded as the actor's latest, so @@ -1079,29 +1177,26 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest // Do not move this above CheckpointWorkload to keep MergeDeltaIntoBase on its // in-place path: that leaves the whole checkpoint window with no local snapshot // while LocalSnapshotInfo still names the pruned one, and a crash there strands - // the actor for good (a paused actor has no uploaded copy for resume to fall - // back to, RequiredNodes pins it to this node, nothing clears the field). + // the actor for good (before the background upload completes, the actor has no + // uploaded copy for resume to fall back to, RequiredNodes pins it to this node, + // and nothing clears the field). // // Best-effort: if this fails, the actor's next checkpoint or terminate // prunes again. - tCleanup := time.Now() if err := snapshottracker.PruneLocalCheckpoints(ctx, actorUID, s.snapshots.LocalSnapshotsToKeep(actorUID)...); err != nil { - slog.WarnContext(ctx, "failed to prune superseded local checkpoints", slog.Any("actor", actorRef), slog.Any("err", err)) + slog.WarnContext(ctx, "failed to prune superseded local checkpoints", + slog.Any("actor", resources.ActorRef{Atespace: req.GetAtespace(), Name: req.GetActorName()}), slog.Any("err", err)) } if err := s.unmountExternalVolumes(ctx, actorUID, req.GetSpec().GetVolumes()); err != nil { - dCleanup += time.Since(tCleanup) - return nil, fmt.Errorf("while unmounting external volumes: %w", err) + return fmt.Errorf("while unmounting external volumes: %w", err) } // Note: we do not crash the actor if resetting the directory fails. if err := resetActorDirs(actorUID); err != nil { - dCleanup += time.Since(tCleanup) - return nil, fmt.Errorf("while resetting actor dirs: %w", err) + return fmt.Errorf("while resetting actor dirs: %w", err) } - dCleanup += time.Since(tCleanup) - - return &ateletpb.CheckpointResponse{State: state}, nil + return nil } func checkpointSnapshotFiles(resp *ateworkerpb.CheckpointWorkloadResponse, required bool) (files, dataFiles []string, err error) { @@ -1136,30 +1231,38 @@ func toWorkerFidelity(fidelity ateletpb.SnapshotFidelity) ateworkerpb.SnapshotFi } func (s *Atelet) moveLocalCheckpoint(ctx context.Context, req *ateletpb.CheckpointRequest, rec *sandboxAssetsRecord) error { - actorDir := ateletpath.ActorPath(req.GetActorUid()) - root, err := os.OpenRoot(actorDir) + snapshotRoot := ateletpath.ActorSnapshotPath(req.GetActorUid()) + root, err := os.OpenRoot(snapshotRoot) if err != nil { - return fmt.Errorf("while opening actor directory: %w", err) + return fmt.Errorf("while opening actor snapshot directory: %w", err) } defer root.Close() - checkpointDir, err := filepath.Rel(actorDir, ateletpath.CheckpointStateDir(req.GetActorUid())) + checkpointDir, err := filepath.Rel(snapshotRoot, ateletpath.CheckpointStateDir(req.GetActorUid())) if err != nil { return err } snapshotName := req.GetSnapshot().GetSnapshotUuid() - localDir, err := filepath.Rel(actorDir, ateletpath.LocalSnapshotDir(req.GetActorUid(), snapshotName)) + localDir, err := filepath.Rel(snapshotRoot, ateletpath.LocalSnapshotDir(req.GetActorUid(), snapshotName)) if err != nil { return err } - if err := root.MkdirAll(localDir, 0o700); err != nil { - return fmt.Errorf("while creating local checkpoint directory: %w", err) + + // Stage the files worker reported and the snapshot manifest inside + // checkpointDir before renaming the directory into local-checkpoint/, so a + // failure leaves nothing under local-checkpoint/ and resetActorDirs cleans + // checkpointDir up. + stageDir := filepath.Join(checkpointDir, ".staged") + if err := root.RemoveAll(stageDir); err != nil { + return fmt.Errorf("while clearing local checkpoint staging directory: %w", err) + } + if err := root.MkdirAll(stageDir, 0o700); err != nil { + return fmt.Errorf("while creating local checkpoint staging directory: %w", err) } - // Move exactly the files worker reported. for _, fileName := range rec.SnapshotFiles { src := filepath.Join(checkpointDir, fileName) - dst := filepath.Join(localDir, fileName) + dst := filepath.Join(stageDir, fileName) info, err := root.Lstat(src) if err != nil { return fmt.Errorf("while inspecting checkpoint file %s: %w", fileName, err) @@ -1182,10 +1285,17 @@ func (s *Atelet) moveLocalCheckpoint(ctx context.Context, req *ateletpb.Checkpoi if err != nil { return fmt.Errorf("while marshaling snapshot manifest: %w", err) } - if err := root.WriteFile(filepath.Join(localDir, sandboxManifestName), manifest, 0o600); err != nil { + if err := root.WriteFile(filepath.Join(stageDir, sandboxManifestName), manifest, 0o600); err != nil { return fmt.Errorf("while writing snapshot manifest: %w", err) } + if err := root.MkdirAll(filepath.Dir(localDir), 0o700); err != nil { + return fmt.Errorf("while creating local checkpoint directory: %w", err) + } + if err := root.Rename(stageDir, localDir); err != nil { + return fmt.Errorf("failed to move %s to %s: %w", stageDir, localDir, err) + } + return nil } @@ -1212,7 +1322,8 @@ func (s *Atelet) uploadExternalCheckpoint(ctx context.Context, req *ateletpb.Che if err != nil { return err } - return s.uploadSnapshot(ctx, uri, checkpointDir, rec, grpc.WaitForReady(true)) + actor := actorRef{atespace: req.GetAtespace(), name: req.GetActorName(), uid: req.GetActorUid()} + return s.uploadSnapshot(ctx, actor, uri, checkpointDir, rec, grpc.WaitForReady(true)) } // uploadSnapshot uploads rec's snapshot files from srcDir to uri through the @@ -1222,8 +1333,8 @@ func (s *Atelet) uploadExternalCheckpoint(ctx context.Context, req *ateletpb.Che // files, never a manifest pointing at files that never landed; retries // overwrite the deterministic object names. opts are passed to every plugin // call, and plugin errors are returned as uploadSnapshotFiles returns them. -func (s *Atelet) uploadSnapshot(ctx context.Context, uri resources.SnapshotURI, srcDir string, rec *sandboxAssetsRecord, opts ...grpc.CallOption) error { - if err := s.uploadSnapshotFiles(ctx, uri.String(), srcDir, rec.SnapshotFiles, opts...); err != nil { +func (s *Atelet) uploadSnapshot(ctx context.Context, actor actorRef, uri resources.SnapshotURI, srcDir string, rec *sandboxAssetsRecord, opts ...grpc.CallOption) error { + if err := s.uploadSnapshotFiles(ctx, actor, uri.String(), srcDir, rec.SnapshotFiles, opts...); err != nil { return fmt.Errorf("while uploading snapshot files: %w", err) } @@ -1231,17 +1342,17 @@ func (s *Atelet) uploadSnapshot(ctx context.Context, uri resources.SnapshotURI, if err != nil { return fmt.Errorf("while marshaling snapshot manifest: %w", err) } - if err := s.uploadManifest(ctx, uri.String(), manifest, opts...); err != nil { + if err := s.uploadManifest(ctx, actor, uri.String(), manifest, opts...); err != nil { return fmt.Errorf("while uploading snapshot manifest: %w", err) } return nil } -// PromoteSnapshot uploads a paused actor's local checkpoint to object -// storage so the actor can be suspended and is no longer tied to this node. It -// drives no worker — the actor's sandbox is gone; the checkpoint files and their -// self-describing manifest already sit under the actor's local-checkpoints -// directory, written by an earlier local Checkpoint (pause). +// PromoteSnapshot uploads a suspended actor's local checkpoint to object +// storage while keeping the local copy on this node (e.g. for CreateTag). It +// drives no worker — the actor's sandbox is gone; the checkpoint files and +// their self-describing manifest already sit under the actor's +// local-checkpoints directory, written by an earlier Checkpoint. func (s *Atelet) PromoteSnapshot(ctx context.Context, req *ateletpb.PromoteSnapshotRequest) (_ *ateletpb.PromoteSnapshotResponse, err error) { if errs := apivalidation.ValidatePromoteSnapshotRequest(ctx, req); len(errs) > 0 { return nil, resources.ToAPIError(errs) @@ -1252,7 +1363,7 @@ func (s *Atelet) PromoteSnapshot(ctx context.Context, req *ateletpb.PromoteSnaps op := snapshotOp{ templateNamespace: req.GetActorTemplateAtespace(), templateName: req.GetActorTemplateName(), - // Uploading a paused checkpoint creates a durable latest snapshot. + // Uploading a local checkpoint creates a durable latest snapshot. kind: ateattr.SnapshotKindLatest, fidelity: ateattr.SnapshotFidelityValue(req.GetSnapshot().GetFidelity()), } @@ -1266,7 +1377,6 @@ func (s *Atelet) PromoteSnapshot(ctx context.Context, req *ateletpb.PromoteSnaps if err != nil { return nil, apierror.InvalidArgument("%v", err) } - localDir := ateletpath.LocalSnapshotDir(req.GetActorUid(), req.GetSnapshot().GetSnapshotUuid()) unlockActor, err := s.lockActor(ctx, req.GetActorUid()) if err != nil { @@ -1274,6 +1384,21 @@ func (s *Atelet) PromoteSnapshot(ctx context.Context, req *ateletpb.PromoteSnaps } defer unlockActor() + state, err := s.promoteLocalSnapshot(ctx, req, uri, &op, &dPersist) + if err != nil { + return nil, err + } + return &ateletpb.PromoteSnapshotResponse{State: state}, nil +} + +// promoteLocalSnapshot makes the local snapshot req names durable at uri, +// unless it is already, and records it as the actor's latest durable +// snapshot. It returns the actor's state. When it uploads, it records the +// sandbox class the snapshot's manifest names in op, and how long the upload +// took in dPersist. +func (s *Atelet) promoteLocalSnapshot(ctx context.Context, req *ateletpb.PromoteSnapshotRequest, uri resources.SnapshotURI, op *snapshotOp, dPersist *time.Duration) (state *ateletpb.ActorSnapshotState, err error) { + localDir := ateletpath.LocalSnapshotDir(req.GetActorUid(), req.GetSnapshot().GetSnapshotUuid()) + // A detach or a background upload of this snapshot in progress finishes // first, and no detach starts until the promote returns. snap := snapshottracker.SnapshotRef{Snapshot: req.GetSnapshot(), Actor: resources.ActorRef{Atespace: req.GetAtespace(), Name: req.GetActorName()}} @@ -1286,7 +1411,7 @@ func (s *Atelet) PromoteSnapshot(ctx context.Context, req *ateletpb.PromoteSnaps if !alreadyDurable { tPersist := time.Now() sandboxClass, uploaded, err := s.uploadLocalCheckpointDir(ctx, req, localDir, uri) - dPersist = time.Since(tPersist) + *dPersist = time.Since(tPersist) op.sandboxClass = sandboxClass if err != nil { return nil, err @@ -1297,9 +1422,7 @@ func (s *Atelet) PromoteSnapshot(ctx context.Context, req *ateletpb.PromoteSnaps snap.UploadTime = time.Now() } } - state := s.snapshots.CommitPromoted(req.GetActorUid(), snap) - - return &ateletpb.PromoteSnapshotResponse{State: state}, nil + return s.snapshots.CommitPromoted(req.GetActorUid(), snap), nil } // uploadLocalCheckpointDir uploads the local checkpoint in localDir to uri. It @@ -1307,6 +1430,7 @@ func (s *Atelet) PromoteSnapshot(ctx context.Context, req *ateletpb.PromoteSnaps // manifest was not read), and whether it uploaded: it does not when the local // copy is gone but already uploaded. Parameterized by localDir for tests. func (s *Atelet) uploadLocalCheckpointDir(ctx context.Context, req *ateletpb.PromoteSnapshotRequest, localDir string, uri resources.SnapshotURI) (sandboxClass string, uploaded bool, err error) { + actor := actorRef{atespace: req.GetAtespace(), name: req.GetActorName(), uid: req.GetActorUid()} localSnapshotName := req.GetSnapshot().GetSnapshotUuid() manifest, err := readSnapshotManifest(localDir) if errors.Is(err, os.ErrNotExist) { @@ -1314,7 +1438,7 @@ func (s *Atelet) uploadLocalCheckpointDir(ctx context.Context, req *ateletpb.Pro // it: the remote manifest is uploaded last, so its presence means the // whole snapshot is committed. Absent on both sides, the local // snapshot is unrecoverable. - _, fetchErr := s.fetchManifest(ctx, uri.String()) + _, fetchErr := s.fetchManifest(ctx, actor, uri.String()) if fetchErr == nil { slog.InfoContext(ctx, "Local snapshot already uploaded and pruned; nothing to do", slog.String("snapshot_uri", uri.String())) return "", false, nil @@ -1336,19 +1460,18 @@ func (s *Atelet) uploadLocalCheckpointDir(ctx context.Context, req *ateletpb.Pro capturedFidelity := rec.Fidelity if capturedFidelity == "" { - return rec.SandboxClass, false, apierror.FailedPrecondition("local snapshot %q has no fidelity recorded in its manifest; resume and pause the actor again before suspending it", localSnapshotName) + return rec.SandboxClass, false, apierror.FailedPrecondition("local snapshot %q has no fidelity recorded in its manifest; resume and suspend the actor again", localSnapshotName) } - // Pause and suspend capture the template's one preferred fidelity, and a - // paused actor's template cannot change, so the upload is always of what - // the pause captured. A mismatch means the store or the control plane - // drifted; refuse rather than guess. + // Every suspend records the fidelity it captured on the snapshot, so the + // upload is always of what the checkpoint captured. A mismatch means the + // store or the control plane drifted; refuse rather than guess. if desiredFidelity := ateattr.SnapshotFidelityValue(req.GetSnapshot().GetFidelity()); capturedFidelity != desiredFidelity { - return rec.SandboxClass, false, apierror.FailedPrecondition("pause snapshot captured %s; upload asks for %s", capturedFidelity, desiredFidelity) + return rec.SandboxClass, false, apierror.FailedPrecondition("local snapshot captured %s; upload asks for %s", capturedFidelity, desiredFidelity) } // The local snapshot stays until the upload succeeds, so a retry can // upload it again. - if err := pluginsocket.CallError(s.uploadSnapshot(ctx, uri, localDir, rec)); err != nil { + if err := pluginsocket.CallError(s.uploadSnapshot(ctx, actor, uri, localDir, rec)); err != nil { return rec.SandboxClass, false, err } return rec.SandboxClass, true, nil @@ -1412,13 +1535,14 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res } actorUID := req.GetActorUid() + actor := actorRef{atespace: req.GetAtespace(), name: req.GetActorName(), uid: actorUID} actorRef := resources.ActorRef{Atespace: req.GetAtespace(), Name: req.GetActorName()} - // The sandbox (binaries + pause image) that runs the restored workload is - // the runtime the control plane assigned the actor to, resolved through - // the index. The snapshot manifests only supply the files to restore and - // the actor identity. Resolved before any on-node work so an invalid - // request changes nothing. + // The sandbox binaries that run the restored workload are the runtime the + // control plane assigned the actor to, resolved through the index. The + // snapshot manifests only supply the files to restore and the actor + // identity. Resolved before any on-node work so an invalid request changes + // nothing. runtimeRec, err := s.advertisedRuntime(ctx, req.GetSandboxRuntime()) if err != nil { return nil, err @@ -1476,7 +1600,7 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res // Not crashing the actor, because terminal errors here indicate problems with atelet, // node or the disk itself. s.systemInfoVolumes.Deregister(actorUID) - if err := resetActorDirs(actorUID); err != nil { + if err := resetActorDirsKeepingVolumes(actorUID); err != nil { return nil, fmt.Errorf("while resetting actor dirs: %w", err) } // A restore may bring another runtime than the one staged here before, @@ -1488,9 +1612,13 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res // The record below is written last, so until then it describes the // previous activation; a failed restore must not leave this one's // asset links behind for a later Terminate to take for that record's. + // Nor its snapshot files: staged ones are links into the snapshot cache + // or copies, and either would hold the cache's entries, or disk, until + // the next lifecycle call resets the directory. defer func() { if err != nil { _ = os.RemoveAll(ateletpath.SandboxAssetsDir(actorUID)) + _ = os.RemoveAll(ateletpath.RestoreStateDir(actorUID)) } }() @@ -1530,7 +1658,7 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res slog.WarnContext(ctx, "Local snapshot is unusable; restoring from object storage", slog.String("local_dir", localDir), slog.Any("err", localErr)) } - manifest, err := s.fetchManifest(ctx, durable.GetSnapshotUri()) + manifest, err := s.fetchManifestCached(ctx, actor, durable.GetSnapshotUri(), snapshotCacheMode(durable, cacheModeLink)) if status.Code(err) == codes.NotFound { return nil, apierror.NotFound("snapshot %q has no usable local copy and no uploaded copy: %w", req.GetSnapshot().GetSnapshotUuid(), localErr) @@ -1582,7 +1710,12 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res // readLocalSnapshot checked; no staging. return nil } - return s.downloadExternalCheckpoint(gctx, durable.GetSnapshotUri(), checkpointDir, sandboxRec.SnapshotFiles) + // Cache-eligible only when the control plane declared the snapshot + // shared; how a hit is served depends on the sandbox class that + // consumes it, the one the request resolved, which is what runs the + // restored workload. + mode := snapshotCacheMode(durable, cacheModeFor(runtimeRec.SandboxClass)) + return s.downloadExternalCheckpoint(gctx, actor, durable.GetSnapshotUri(), checkpointDir, sandboxRec.SnapshotFiles, mode) }) g.Go(func() (err error) { defer func() { prepErr = err }() @@ -1598,7 +1731,7 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res return err } t := time.Now() - containerSpecs, err = s.prepareOCIBundles(gctx, actorUID, actorRef, req.GetSpec(), runtimeRec.PauseImage) + containerSpecs, err = s.prepareOCIBundles(gctx, actorUID, actorRef, req.GetSpec()) dBundles = time.Since(t) if err != nil { prepFailedPhase = ateattr.SnapshotPhaseOCIUnpack @@ -1621,8 +1754,8 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res return nil, err } - // Tell worker to do runsc create + runsc restore for pause container and - // all application containers. + // Tell worker to do runsc create + runsc restore for the sandbox and all + // application containers. spec, err := buildWorkerWorkloadSpec(req.GetSpec(), containerSpecs) if err != nil { return nil, apierror.InvalidArgument("invalid workload spec: %v", err) @@ -1700,6 +1833,10 @@ func (s *Atelet) Terminate(ctx context.Context, req *ateletpb.TerminateRequest) return nil, err } defer unlockActor() + // No background work on the actor starts while it is being terminated. + // Forget below cancels an upload or detach already in progress. + endTerminate := s.snapshots.BeginTerminate(actorUID) + defer endTerminate() if req.GetWorkerPodUid() != "" { var assetPaths map[string]string @@ -1748,10 +1885,9 @@ func (s *Atelet) Terminate(ctx context.Context, req *ateletpb.TerminateRequest) return nil, fmt.Errorf("failed to unmount external volumes during terminate (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) } - // The actor is gone, so no pause snapshot of it can ever be restored again. - // Forget cancels the actor's background upload or detach in progress - // without waiting for it. The cancelled operation leaves the actor's files - // alone, unless it was already deleting them, as this does too. + // The actor is gone, so no local snapshot of it can ever be restored again. + // Forget cancels the actor's background upload or detach in progress and + // waits for it to end before this prunes the actor's files. // TODO(#664): this only removes local snapshots in one node. We should clean // up the copies on any other NodeVmsWithLocalSnapshots. This is fine *as of // the day this was written* because today NodeVmsWithLocalSnapshots has at @@ -1792,96 +1928,19 @@ func checkLocalSnapshotFiles(dir string, files []string) error { return nil } -// copyLocalCheckpoint stages files from the local checkpoint snapshotName under -// srcDir into dstDir. Both must be inside actorDir, which confines every access. -func (s *Atelet) copyLocalCheckpoint(ctx context.Context, actorDir, snapshotName, srcDir, dstDir string, files []string) error { - root, err := os.OpenRoot(actorDir) - if err != nil { - return fmt.Errorf("while opening actor directory: %w", err) - } - defer root.Close() - srcDir, err = filepath.Rel(actorDir, filepath.Join(srcDir, snapshotName)) - if err != nil { - return err - } - dstDir, err = filepath.Rel(actorDir, dstDir) - if err != nil { - return err - } - - for _, fileName := range files { - if ctx.Err() != nil { - return fmt.Errorf("context cancelled: %w", ctx.Err()) - } - src := filepath.Join(srcDir, fileName) - dst := filepath.Join(dstDir, fileName) - // A link to a symlink would be followed later, outside the root. - info, err := root.Lstat(src) - if err != nil { - return fmt.Errorf("while inspecting %s: %w", src, err) - } - if !info.Mode().IsRegular() { - return fmt.Errorf("%s is not a regular file", src) - } - if err := root.MkdirAll(filepath.Dir(dst), 0o700); err != nil { - return fmt.Errorf("while creating parent directory for %s: %w", dst, err) - } - // Link rather than copy. The local checkpoint lives under the same actor dir - // as the restore staging area, so this stages the memory image in constant - // time instead of re-writing its whole working set. Nothing rewrites the - // shared inode: CH demand-pages from the staged image read-only, - // rewriteSnapshotSocketPaths renames its rewritten config.json into place - // rather than truncating, and MergeDeltaIntoBase refuses its in-place overlay - // once the image carries a second link. - // - // EXDEV alone falls back to copying, so an unexpected link failure surfaces - // instead of silently reverting to the full copy this exists to remove. It - // also keeps copyRootFile off a dst that is already a link to src, where its - // O_TRUNC would empty both and report a successful copy of the old size. - switch err := linkFile(root, src, dst); { - case err == nil: - continue - case !errors.Is(err, unix.EXDEV): - return fmt.Errorf("failed to link %s to %s: %w", src, dst, err) - } - slog.WarnContext(ctx, "local checkpoint and restore dir are on different filesystems; copying instead of linking", - slog.String("src", src), slog.String("dst", dst)) - if _, err := copyRootFile(root, src, dst); err != nil { - return fmt.Errorf("failed to copy %s to %s: %w", src, dst, err) - } - } - - return nil -} - -// linkFile is os.Root.Link, indirected so a test can force the cross-filesystem -// fallback in copyLocalCheckpoint without mounting a second filesystem. -var linkFile = (*os.Root).Link - -func copyRootFile(root *os.Root, src, dst string) (int64, error) { - source, err := root.Open(src) - if err != nil { - return 0, err - } - defer source.Close() - info, err := source.Stat() - if err != nil { - return 0, err - } - if !info.Mode().IsRegular() { - return 0, fmt.Errorf("%s is not a regular file", src) +// downloadExternalCheckpoint stages files of one snapshot into dstDir. With +// a mode other than cacheModeOff and the snapshot cache enabled, each file +// is served from the cache (see downloadSnapshotFilesCached): the caller +// asserts the snapshot is immutable at its URI. Otherwise the files are +// downloaded into the restore directory directly. +func (s *Atelet) downloadExternalCheckpoint(ctx context.Context, actor actorRef, snapshotURI string, dstDir string, files []string, mode cacheMode) error { + var err error + if mode != cacheModeOff && s.snapshotCache != nil { + err = s.downloadSnapshotFilesCached(ctx, actor, snapshotURI, dstDir, files, mode) + } else { + err = s.fetchSnapshotFiles(ctx, actor, snapshotURI, dstDir, files) } - - destination, err := root.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o666) if err != nil { - return 0, err - } - nBytes, err := sparsefile.Copy(source, destination) - return nBytes, errors.Join(err, destination.Close()) -} - -func (s *Atelet) downloadExternalCheckpoint(ctx context.Context, snapshotURI string, dstDir string, files []string) error { - if err := s.fetchSnapshotFiles(ctx, snapshotURI, dstDir, files); err != nil { return fmt.Errorf("while downloading snapshot files: %w", err) } return nil @@ -1901,19 +1960,14 @@ func prepareDurableDirVolume(path string) error { return nil } -// prepareOCIBundles pulls images and assembles OCI bundles for the pause -// container and every application container in spec, in parallel. pauseImage -// comes from the sandbox record, not the workload spec: it is sandbox -// configuration, and on a restore it must be the image the snapshot was taken -// with. It is empty for sandboxes without a pause container, which get no -// pause bundle. It returns each application container's spec, indexed like -// spec.GetContainers(). +// prepareOCIBundles pulls images and assembles OCI bundles for every +// application container in spec, in parallel. It returns each application +// container's spec, indexed like spec.GetContainers(). func (s *Atelet) prepareOCIBundles( ctx context.Context, actorUID string, actorRef resources.ActorRef, spec *ateletpb.WorkloadSpec, - pauseImage string, ) ([]*ateworkerpb.ContainerSpec, error) { // Prepare host folders for volume types that need them. for _, vol := range spec.GetVolumes() { @@ -1926,16 +1980,6 @@ func (s *Atelet) prepareOCIBundles( g, gCtx := errgroup.WithContext(ctx) - if pauseImage != "" { - g.Go(func() error { - // pause is sandbox infra; it mounts no volumes. - if _, err := prepareOCIDirectory(gCtx, s.imageCache, actorUID, ocispec.PauseContainer, pauseImage, nil, nil); err != nil { - return wrapFileSystemErr("while creating pause OCI bundle", err) - } - return nil - }) - } - // Application containers. containerSpecs := make([]*ateworkerpb.ContainerSpec, len(spec.GetContainers())) for i, ctr := range spec.GetContainers() { @@ -2135,6 +2179,29 @@ func (d *WorkerDialer) DialWorkerPod(ctx context.Context, podUID string) (*grpc. // resetActorDirs empties the actor's directories and leaves them in place for // its next activation. Use removeActorDirs when the actor will not come back. func resetActorDirs(actorUID string) error { + if err := resetActorDirsKeepingVolumes(actorUID); err != nil { + return err + } + // Do not call RemoveAll on volume directories in case the unmount failed. + // We do not want to delete mount content. + volumesDir := ateletpath.ExternalVolumeMountsDir(actorUID) + entries, err := os.ReadDir(volumesDir) + if err != nil { + return wrapFileSystemErr("while reading volumes dir", err) + } + for _, entry := range entries { + volPath := filepath.Join(volumesDir, entry.Name()) + if err := os.Remove(volPath); err != nil { + return wrapFileSystemErr("while removing volume dir", err) + } + } + return nil +} + +// resetActorDirsKeepingVolumes is resetActorDirs for Run and Restore, which +// leave the external volume mount points alone: a failed earlier attempt can +// leave a volume published there, and publishing it again is a no-op. +func resetActorDirsKeepingVolumes(actorUID string) error { // Explicitly leave runsc logs dir untouched. // RemoveAllWritable, not os.RemoveAll: the bundle's upper dir can hold @@ -2160,24 +2227,9 @@ func resetActorDirs(actorUID string) error { if err := resetDir("system-info volume roots dir", ateletpath.SystemInfoVolumeRootsDir(actorUID), os.RemoveAll, 0o755); err != nil { return err } - - // Do not call RemoveAll on volume directories in case the unmount failed. - // We do not want to delete mount content. - volumesDir := ateletpath.ExternalVolumeMountsDir(actorUID) - entries, err := os.ReadDir(volumesDir) - if err != nil && !os.IsNotExist(err) { - return wrapFileSystemErr("while reading volumes dir", err) - } - for _, entry := range entries { - volPath := filepath.Join(volumesDir, entry.Name()) - if err := os.Remove(volPath); err != nil { - return wrapFileSystemErr("while removing volume dir", err) - } - } - if err := os.MkdirAll(volumesDir, 0o755); err != nil { + if err := os.MkdirAll(ateletpath.ExternalVolumeMountsDir(actorUID), 0o755); err != nil { return wrapFileSystemErr("while creating volumes dir", err) } - return nil } @@ -2232,8 +2284,9 @@ func resetDir(what, dir string, remove func(string) error, mode os.FileMode) err return nil } -// removeActorDirs reclaims the actor's whole directory tree, root included: -// nothing else on the node deletes it, and no later activation will look here. +// removeActorDirs reclaims both of the actor's directory trees, roots +// included: nothing else on the node deletes them, and no later activation +// will look there. // // resetActorDirs runs first for the care a blanket RemoveAll lacks. It refuses // to proceed while a volume directory is still populated, so a failed unmount @@ -2243,6 +2296,11 @@ func removeActorDirs(actorUID string) error { if err := resetActorDirs(actorUID); err != nil { return err } + // The actor dir holds sandbox-assets.json, which a retried Terminate needs, + // so remove it last. + if err := os.RemoveAll(ateletpath.ActorSnapshotPath(actorUID)); err != nil { + return wrapFileSystemErr("while deleting actor snapshot dir", err) + } if err := os.RemoveAll(ateletpath.ActorPath(actorUID)); err != nil { return wrapFileSystemErr("while deleting actor dir", err) } diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index 9fb7a4e906..dcadeb47d5 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -30,7 +30,6 @@ import ( "strings" "sync" "sync/atomic" - "syscall" "testing" "time" @@ -42,14 +41,11 @@ import ( "go.ate.dev/substrate/internal/ateattr" "go.ate.dev/substrate/internal/ateinterceptors" "go.ate.dev/substrate/internal/atelet" - "go.ate.dev/substrate/internal/imagecache" - "go.ate.dev/substrate/internal/ocispec" "go.ate.dev/substrate/internal/proto/ateletpb" "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/internal/serverboot" "go.ate.dev/substrate/pkg/objectstorage" - "golang.org/x/sys/unix" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" @@ -58,8 +54,6 @@ import ( "google.golang.org/protobuf/types/known/emptypb" ) -const testPauseImage = "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" - const ( snapshotOwnerUID = "3d7f1b62-8a04-4c19-b5e6-2f9c7a1d0e38" @@ -197,94 +191,6 @@ func TestValidateSnapshotWorkerFlags(t *testing.T) { } } -// TestCopyLocalCheckpointLinks covers staging a local checkpoint into the restore -// dir: the files must land as extra links to the cached snapshot rather than -// copies, so a resume does not rewrite the image's working set. Sharing the inode -// is what MergeDeltaIntoBase's Nlink check keys off to refuse its in-place overlay. -func TestCopyLocalCheckpointLinks(t *testing.T) { - const snapshot = "snap-1" - want := []byte("checkpoint pages") - - newDirs := func(t *testing.T) (srcDir, dstDir string) { - t.Helper() - root := t.TempDir() - srcDir, dstDir = filepath.Join(root, "local-checkpoint"), filepath.Join(root, "restore-state") - if err := os.MkdirAll(filepath.Join(srcDir, snapshot), 0o700); err != nil { - t.Fatal(err) - } - if err := os.MkdirAll(dstDir, 0o700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(srcDir, snapshot, "memory-ranges"), want, 0o600); err != nil { - t.Fatal(err) - } - return srcDir, dstDir - } - inode := func(t *testing.T, path string) uint64 { - t.Helper() - fi, err := os.Stat(path) - if err != nil { - t.Fatal(err) - } - return fi.Sys().(*syscall.Stat_t).Ino - } - - t.Run("links when it can", func(t *testing.T) { - srcDir, dstDir := newDirs(t) - s := &Atelet{} - if err := s.copyLocalCheckpoint(context.Background(), filepath.Dir(srcDir), snapshot, srcDir, dstDir, []string{"memory-ranges"}); err != nil { - t.Fatalf("copyLocalCheckpoint: %v", err) - } - src := filepath.Join(srcDir, snapshot, "memory-ranges") - dst := filepath.Join(dstDir, "memory-ranges") - if got, err := os.ReadFile(dst); err != nil || !bytes.Equal(got, want) { - t.Fatalf("dst content = %q (err %v), want %q", got, err, want) - } - if inode(t, src) != inode(t, dst) { - t.Error("staged file is a copy; expected a link to the cached snapshot") - } - }) - - t.Run("falls back to copying across filesystems", func(t *testing.T) { - srcDir, dstDir := newDirs(t) - // EXDEV stands in for the mount boundary a unit test cannot produce. - orig := linkFile - linkFile = func(*os.Root, string, string) error { return unix.EXDEV } - t.Cleanup(func() { linkFile = orig }) - - s := &Atelet{} - if err := s.copyLocalCheckpoint(context.Background(), filepath.Dir(srcDir), snapshot, srcDir, dstDir, []string{"memory-ranges"}); err != nil { - t.Fatalf("copyLocalCheckpoint: %v", err) - } - dst := filepath.Join(dstDir, "memory-ranges") - if got, err := os.ReadFile(dst); err != nil || !bytes.Equal(got, want) { - t.Fatalf("dst content = %q (err %v), want %q", got, err, want) - } - if inode(t, filepath.Join(srcDir, snapshot, "memory-ranges")) == inode(t, dst) { - t.Error("expected a copy on the fallback path, got a link") - } - }) - - // Only EXDEV may fall back. Copying on any other link failure would silently - // undo this optimization, and would hand sparsefile.CopyFile a dst that may already be a - // link to src, where its O_TRUNC empties both before the copy reads a byte. - t.Run("other link failures are fatal", func(t *testing.T) { - srcDir, dstDir := newDirs(t) - dst := filepath.Join(dstDir, "memory-ranges") - if err := os.Link(filepath.Join(srcDir, snapshot, "memory-ranges"), dst); err != nil { - t.Fatal(err) - } - s := &Atelet{} - // dst already exists, so os.Link fails with EEXIST. - if err := s.copyLocalCheckpoint(context.Background(), filepath.Dir(srcDir), snapshot, srcDir, dstDir, []string{"memory-ranges"}); err == nil { - t.Fatal("copyLocalCheckpoint accepted a non-EXDEV link failure, want an error") - } - if got, err := os.ReadFile(dst); err != nil || !bytes.Equal(got, want) { - t.Fatalf("dst content = %q (err %v), want the untouched %q", got, err, want) - } - }) -} - func TestSnapshotManifestActorMetadata(t *testing.T) { rec := sandboxAssetsRecord{ Atespace: "team-a", @@ -309,7 +215,7 @@ func TestSnapshotManifestActorMetadata(t *testing.T) { // Run/Restore, which has no fidelity, parses back with an empty fidelity and // does not serialize a fidelity key at all. func TestSnapshotManifestFidelityAbsent(t *testing.T) { - legacy := []byte(`{"sandboxClass":"gvisor","pauseImage":"` + testPauseImage + `","snapshotFiles":["checkpoint.img"]}`) + legacy := []byte(`{"sandboxClass":"gvisor","snapshotFiles":["checkpoint.img"]}`) rec, err := unmarshalSandboxRecord(legacy) if err != nil { t.Fatalf("unmarshalSandboxRecord(legacy manifest): %v", err) @@ -327,70 +233,6 @@ func TestSnapshotManifestFidelityAbsent(t *testing.T) { } } -// TestSnapshotManifestPauseImage pins that only a gVisor manifest needs a -// pause image: micro-VM sandboxes run no pause container. -func TestSnapshotManifestPauseImage(t *testing.T) { - for _, tc := range []struct { - name string - manifest string - wantErr bool - }{ - {"gvisor without pause image", `{"sandboxClass":"gvisor","snapshotFiles":["checkpoint.img"]}`, true}, - {"gvisor with pause image", `{"sandboxClass":"gvisor","pauseImage":"` + testPauseImage + `","snapshotFiles":["checkpoint.img"]}`, false}, - {"microvm without pause image", `{"sandboxClass":"microvm","snapshotFiles":["checkpoint.img"]}`, false}, - } { - t.Run(tc.name, func(t *testing.T) { - _, err := unmarshalSandboxRecord([]byte(tc.manifest)) - if tc.wantErr { - if err == nil { - t.Fatal("unmarshalSandboxRecord succeeded, want an error") - } - if !strings.Contains(err.Error(), "pauseImage") { - t.Errorf("error = %v, want it to name pauseImage", err) - } - } else if err != nil { - t.Fatalf("unmarshalSandboxRecord: %v", err) - } - }) - } -} - -// TestPrepareOCIBundlesPause pins that the pause bundle is built only when the -// sandbox has a pause image. -func TestPrepareOCIBundlesPause(t *testing.T) { - host := imageVolumeTestRegistry(t) - image := host + "/actor:v1" - pushTestImage(t, image, singleFileLayer(t, "bin/app", "app")) - spec := &ateletpb.WorkloadSpec{ - Containers: []*ateletpb.Container{{Name: "app", Image: image, Command: []string{"/bin/app"}}}, - } - - for _, tc := range []struct { - name string - pauseImage string - wantPause bool - }{ - {"with pause image", image, true}, - {"without pause image", "", false}, - } { - t.Run(tc.name, func(t *testing.T) { - useTempNodeDirs(t) - const actorUID = "actor-uid-1" - s := &Atelet{imageCache: newImageVolumeStore(t)} - if _, err := s.prepareOCIBundles(t.Context(), actorUID, resources.ActorRef{}, spec, tc.pauseImage); err != nil { - t.Fatalf("prepareOCIBundles: %v", err) - } - if _, err := os.Stat(filepath.Join(ateletpath.OCIBundlePath(actorUID, "app"), imagecache.OverlaySpecFileName)); err != nil { - t.Errorf("app bundle: %v", err) - } - _, err := os.Stat(ateletpath.OCIBundlePath(actorUID, ocispec.PauseContainer)) - if gotPause := err == nil; gotPause != tc.wantPause { - t.Errorf("pause bundle exists = %v, want %v (stat: %v)", gotPause, tc.wantPause, err) - } - }) - } -} - // TestPrepareOCIBundlesFillsContainerSpec pins that each container's spec // carries its resolved args, env, capabilities and resource limits. func TestPrepareOCIBundlesFillsContainerSpec(t *testing.T) { @@ -413,7 +255,7 @@ func TestPrepareOCIBundlesFillsContainerSpec(t *testing.T) { } s := &Atelet{imageCache: newImageVolumeStore(t)} - got, err := s.prepareOCIBundles(t.Context(), "actor-uid-1", resources.ActorRef{}, spec, "") + got, err := s.prepareOCIBundles(t.Context(), "actor-uid-1", resources.ActorRef{}, spec) if err != nil { t.Fatalf("prepareOCIBundles: %v", err) } @@ -431,7 +273,6 @@ func TestPrepareOCIBundlesFillsContainerSpec(t *testing.T) { func TestSnapshotManifestRejectsNonLocalFile(t *testing.T) { manifest, err := json.Marshal(sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: testPauseImage, SnapshotFiles: []string{"../outside"}, }) if err != nil { @@ -445,7 +286,6 @@ func TestSnapshotManifestRejectsNonLocalFile(t *testing.T) { func TestSnapshotManifestRejectsDataFileNotInSnapshotFiles(t *testing.T) { manifest, err := json.Marshal(sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: testPauseImage, SnapshotFiles: []string{"checkpoint.img"}, DataSnapshotFiles: []string{"data.tar"}, }) @@ -508,7 +348,7 @@ func TestUploadSnapshotRejectsSymlinkOutsideRoot(t *testing.T) { t.Fatal(err) } store := &recordingObjectStorage{} - err = newPluginHerder(t, store).uploadSnapshot(context.Background(), uri, checkpointDir, + err = newPluginHerder(t, store).uploadSnapshot(context.Background(), testActor, uri, checkpointDir, &sandboxAssetsRecord{SnapshotFiles: []string{"checkpoint.img"}}) if err == nil { t.Fatal("uploadSnapshot() followed a symlink outside the checkpoint directory") @@ -541,7 +381,7 @@ func TestDownloadExternalCheckpointRejectsSymlinkOutsideRoot(t *testing.T) { t.Fatal(err) } err := newPluginHerder(t, store).downloadExternalCheckpoint( - context.Background(), testSnapshotURI, restoreDir, []string{"checkpoint.img"}) + context.Background(), testActor, testSnapshotURI, restoreDir, []string{"checkpoint.img"}, cacheModeOff) if err == nil { t.Fatal("downloadExternalCheckpoint() followed a symlink outside the restore directory") } @@ -550,35 +390,6 @@ func TestDownloadExternalCheckpointRejectsSymlinkOutsideRoot(t *testing.T) { } } -func TestCopyLocalCheckpointRejectsSymlinkOutsideRoot(t *testing.T) { - parent := t.TempDir() - snapshotName := "pause-1" - snapshotDir := filepath.Join(parent, "local-checkpoint", snapshotName) - if err := os.MkdirAll(snapshotDir, 0o700); err != nil { - t.Fatal(err) - } - outside := filepath.Join(parent, "outside") - if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil { - t.Fatal(err) - } - if err := os.Symlink(outside, filepath.Join(snapshotDir, "checkpoint.img")); err != nil { - t.Fatal(err) - } - restoreDir := filepath.Join(parent, "restore-state") - if err := os.Mkdir(restoreDir, 0o700); err != nil { - t.Fatal(err) - } - - err := (&Atelet{}).copyLocalCheckpoint(context.Background(), parent, snapshotName, - filepath.Join(parent, "local-checkpoint"), restoreDir, []string{"checkpoint.img"}) - if err == nil { - t.Fatal("copyLocalCheckpoint() followed a symlink outside the local checkpoint directory") - } - if _, err := os.Stat(filepath.Join(restoreDir, "checkpoint.img")); !errors.Is(err, os.ErrNotExist) { - t.Fatalf("restore file exists after rejected copy: %v", err) - } -} - func TestCheckLocalSnapshotFilesRejectsSymlink(t *testing.T) { parent := t.TempDir() snapshotDir := filepath.Join(parent, "local-checkpoint", "pause-1") @@ -1073,15 +884,91 @@ func TestRemoveActorDirsReclaimsTheRoot(t *testing.T) { t.Fatalf("removeActorDirs: %v", err) } - actorDir := ateletpath.ActorPath(actorUID) - if entries, err := os.ReadDir(actorDir); err == nil { - left := make([]string, 0, len(entries)) - for _, e := range entries { - left = append(left, e.Name()) + assertDirGone(t, "actor dir", ateletpath.ActorPath(actorUID)) + + assertDirGone(t, "actor snapshot dir", ateletpath.ActorSnapshotPath(actorUID)) +} + +// assertDirGone fails t if dir still exists, naming what is left in it. +func assertDirGone(t *testing.T, what, dir string) { + t.Helper() + entries, err := os.ReadDir(dir) + if os.IsNotExist(err) { + return + } + if err != nil { + t.Errorf("reading %s %s: %v", what, dir, err) + return + } + left := make([]string, 0, len(entries)) + for _, e := range entries { + left = append(left, e.Name()) + } + t.Errorf("%s %s survived with %d entries: %v", what, dir, len(left), left) +} + +func TestResetActorDirsCreatesBothTrees(t *testing.T) { + useTempNodeDirs(t) + const actorUID = "actor-uid-isolation" + + if err := resetActorDirs(actorUID); err != nil { + t.Fatalf("resetActorDirs(%q) failed: %v", actorUID, err) + } + for root, dirs := range map[string]map[string]string{ + ateletpath.ActorSnapshotPath(actorUID): { + "CheckpointStateDir": ateletpath.CheckpointStateDir(actorUID), + "RestoreStateDir": ateletpath.RestoreStateDir(actorUID), + }, + ateletpath.ActorPath(actorUID): { + "OCIBundleDir": ateletpath.OCIBundleDir(actorUID), + "DurableDirVolumeMountsDir": ateletpath.DurableDirVolumeMountsDir(actorUID), + "SystemInfoVolumeRootsDir": ateletpath.SystemInfoVolumeRootsDir(actorUID), + "ExternalVolumeMountsDir": ateletpath.ExternalVolumeMountsDir(actorUID), + }, + } { + for name, dir := range dirs { + if rel, err := filepath.Rel(root, dir); err != nil || !filepath.IsLocal(rel) { + t.Errorf("%s = %q, want it under %q", name, dir, root) + } + if info, err := os.Stat(dir); err != nil || !info.IsDir() { + t.Errorf("resetActorDirs did not create %s (%q): err=%v", name, dir, err) + } } - t.Errorf("actor dir %s survived removeActorDirs with %d entries: %v", actorDir, len(left), left) - } else if !os.IsNotExist(err) { - t.Errorf("reading actor dir %s: %v", actorDir, err) + } +} + +func TestRemoveActorDirsKeepsActorDirWhenSnapshotTreeRemains(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root ignores directory permissions, so no snapshot can be made undeletable") + } + useTempNodeDirs(t) + const actorUID = "actor-uid-1" + + if err := resetActorDirs(actorUID); err != nil { + t.Fatalf("resetActorDirs(%q) failed: %v", actorUID, err) + } + record := ateletpath.ActorSandboxAssetsFile(actorUID) + if err := os.WriteFile(record, []byte("{}"), 0o600); err != nil { + t.Fatal(err) + } + // A local snapshot dir with no write bit: its file cannot be unlinked. + stuck := ateletpath.LocalSnapshotDir(actorUID, "pause-stuck") + if err := os.MkdirAll(stuck, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(stuck, "checkpoint.img"), nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chmod(stuck, 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(stuck, 0o700) }) + + if err := removeActorDirs(actorUID); err == nil { + t.Fatal("removeActorDirs() = nil, want an error for the undeletable snapshot") + } + if _, err := os.Stat(record); err != nil { + t.Errorf("sandbox record %s is gone (err=%v), want it kept for a retried Terminate", record, err) } } @@ -1385,7 +1272,7 @@ func (r *recordingObjectStorage) keys() []string { return keys } -// writeLocalSnapshot lays a local pause snapshot out in dir: the given files +// writeLocalSnapshot lays a node-local snapshot out in dir: the given files // plus the marshaled manifest beside them. func writeLocalSnapshot(t *testing.T, dir string, rec sandboxAssetsRecord, contents map[string]string) { t.Helper() @@ -1462,7 +1349,6 @@ func TestUploadLocalCheckpointDir(t *testing.T) { fullRec := func(class string) sandboxAssetsRecord { return sandboxAssetsRecord{ SandboxClass: class, - PauseImage: testPauseImage, SnapshotFiles: []string{"config.json", "memory-ranges", "data.tar"}, DataSnapshotFiles: []string{"data.tar"}, Fidelity: ateattr.SnapshotFidelityMemory, @@ -1513,7 +1399,6 @@ func TestUploadLocalCheckpointDir(t *testing.T) { dir := filepath.Join(t.TempDir(), "pause-snap-1") writeLocalSnapshot(t, dir, sandboxAssetsRecord{ SandboxClass: "microvm", - PauseImage: testPauseImage, SnapshotFiles: []string{"rootfs-upper-app.tar", "data.tar"}, DataSnapshotFiles: []string{"data.tar"}, Fidelity: ateattr.SnapshotFidelityRootfs, @@ -1577,7 +1462,6 @@ func TestUploadLocalCheckpointDir(t *testing.T) { dir := filepath.Join(t.TempDir(), "pause-snap-1") writeLocalSnapshot(t, dir, sandboxAssetsRecord{ SandboxClass: "microvm", - PauseImage: testPauseImage, SnapshotFiles: []string{"data.tar"}, }, map[string]string{"data.tar": "data"}) diff --git a/cmd/atelet/metrics.go b/cmd/atelet/metrics.go index ed31bad2f5..0fb01a77e2 100644 --- a/cmd/atelet/metrics.go +++ b/cmd/atelet/metrics.go @@ -166,8 +166,9 @@ func restoreSnapshotKind(directLocal bool, rec *sandboxAssetsRecord) string { return ateattr.SnapshotKindLatest } -// checkpointSnapshotKind classifies which snapshot a checkpoint writes: a pause -// writes the node-local one, and a suspend writes a durable latest snapshot. +// checkpointSnapshotKind classifies which snapshot a checkpoint writes: a +// deferred-upload checkpoint (async suspend) writes the node-local one, and an +// immediate-upload checkpoint (sync suspend) writes a durable latest snapshot. func checkpointSnapshotKind(req *ateletpb.CheckpointRequest) string { if req.GetUploadConfig().GetMode() == ateletpb.UploadMode_UPLOAD_MODE_DEFERRED { return ateattr.SnapshotKindLocal diff --git a/cmd/atelet/runtime_prewarm.go b/cmd/atelet/runtime_prewarm.go index 6801ed0706..ec648b138f 100644 --- a/cmd/atelet/runtime_prewarm.go +++ b/cmd/atelet/runtime_prewarm.go @@ -21,19 +21,14 @@ import ( "io/fs" "log/slog" "os" - "sync" "time" "k8s.io/client-go/util/workqueue" - - "go.ate.dev/substrate/internal/imagecache" ) // defaultPrewarmAttemptTimeout bounds one prewarm attempt. The queue is // drained by one worker, so an attempt that never returned would block -// every other runtime's prewarm. The asset downloads are bounded by the -// asset cache's own fetch timeout; this covers the pause image pull beside -// them. +// every other runtime's prewarm. const defaultPrewarmAttemptTimeout = 10 * time.Minute // prewarmMaxRetries bounds how often one failed prewarm is retried before @@ -60,7 +55,7 @@ type prewarmItem struct { } // runtimePrewarmer fetches the default runtime of each worker on this node -// into the asset and image caches before any actor asks for it, so the +// into the asset cache before any actor asks for it, so the // fetch inside the first Run/Restore on the node is a cache hit instead of // a download+extract on the critical path. The worker sandbox syncer hands // it each default runtime a worker advertises, once the worker's file says @@ -76,14 +71,12 @@ type prewarmItem struct { // without fetching, retrying, or recreating anything. // // Prewarming is purely a latency optimization: every failure is logged and -// left to the on-demand fetch in ensureSandboxAssets and the pull inside -// prepareOCIBundles, which remain the correctness path. Only default -// runtimes are prewarmed; the first restore of another version a worker -// advertises pays its download. +// left to the on-demand fetch in ensureSandboxAssets, which remains the +// correctness path. Only default runtimes are prewarmed; the first restore of +// another version a worker advertises pays its download. type runtimePrewarmer struct { index *runtimeIndex assets sandboxAssetFetcher - images *imagecache.Store // queue carries items. It dedupes while an item waits, and it // rate-limits retries after failures. A single worker drains it, which // also serializes downloads so concurrent prewarms never compete for @@ -100,11 +93,10 @@ type runtimePrewarmer struct { onFinished func(item prewarmItem, err error) } -func newRuntimePrewarmer(index *runtimeIndex, assets sandboxAssetFetcher, images *imagecache.Store) *runtimePrewarmer { +func newRuntimePrewarmer(index *runtimeIndex, assets sandboxAssetFetcher) *runtimePrewarmer { return &runtimePrewarmer{ index: index, assets: assets, - images: images, attemptTimeout: defaultPrewarmAttemptTimeout, // Downloads fail on the scale of network timeouts, not API conflicts, // so back off in seconds and cap in minutes rather than the @@ -183,11 +175,10 @@ func (p *runtimePrewarmer) process(ctx context.Context, item prewarmItem) { } // prewarm fetches the runtime's assets into the asset cache, linked into -// the item's directory, and its pause image into the image cache, the two -// side by side. Racing an on-demand ensureSandboxAssets for the same assets -// is safe: the asset cache runs one download per asset and serves every -// caller from it, and the image cache collapses concurrent pulls of one -// digest. A directory the syncer already filled costs a stat per asset. +// the item's directory. Racing an on-demand ensureSandboxAssets for the same +// assets is safe: the asset cache runs one download per asset and serves +// every caller from it. A directory the syncer already filled costs a stat +// per asset. // // Nothing here creates a directory: the staging goes into the item's // directory as it is, so a directory removed before or during the work, by @@ -204,20 +195,7 @@ func (p *runtimePrewarmer) prewarm(ctx context.Context, item prewarmItem, rt *in return err } - var assetErr, imageErr error - var wg sync.WaitGroup - wg.Go(func() { - _, assetErr = p.assets.stageSandboxAssets(ctx, rt.rec, item.dir) - }) - if rt.rec.PauseImage != "" { - wg.Go(func() { - if _, err := p.images.EnsureImage(ctx, rt.rec.PauseImage); err != nil { - imageErr = fmt.Errorf("while prewarming pause image %q: %w", rt.rec.PauseImage, err) - } - }) - } - wg.Wait() - if err := errors.Join(assetErr, imageErr); err != nil { + if _, err := p.assets.stageSandboxAssets(ctx, rt.rec, item.dir); err != nil { if gone, gerr := dirGone(item.dir); gerr == nil && gone { slog.DebugContext(ctx, "Sandbox runtime prewarm superseded while staging", slog.String("runtime_hash", item.hash), slog.String("dir", item.dir)) return nil @@ -229,7 +207,6 @@ func (p *runtimePrewarmer) prewarm(ctx context.Context, item prewarmItem, rt *in slog.String("config", rt.config), slog.String("version", rt.version), slog.Int("assets", len(rt.rec.Assets)), - slog.String("pause_image", rt.rec.PauseImage), slog.String("dir", item.dir), slog.Duration("duration", time.Since(t))) return nil diff --git a/cmd/atelet/runtime_prewarm_test.go b/cmd/atelet/runtime_prewarm_test.go index 4dc1d72482..19105e3cf9 100644 --- a/cmd/atelet/runtime_prewarm_test.go +++ b/cmd/atelet/runtime_prewarm_test.go @@ -30,11 +30,6 @@ import ( "testing" "time" - "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/empty" - "github.com/google/go-containerregistry/pkg/v1/mutate" - "github.com/google/go-containerregistry/pkg/v1/remote" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sruntime "k8s.io/apimachinery/pkg/runtime" kubeinformers "k8s.io/client-go/informers" @@ -42,34 +37,11 @@ import ( "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" "go.ate.dev/substrate/cmd/atelet/internal/filecache" - "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/pkg/api/v1alpha1" atefake "go.ate.dev/substrate/pkg/client/clientset/versioned/fake" "go.ate.dev/substrate/pkg/client/informers/externalversions" ) -// pushPauseImage pushes a tiny image to ref and returns its manifest digest, -// so tests can later assert a cache hit by digest with the registry gone. -func pushPauseImage(t *testing.T, ref string) v1.Hash { - t.Helper() - img, err := mutate.AppendLayers(empty.Image, singleFileLayer(t, "pause", "pause bytes")) - if err != nil { - t.Fatalf("mutate.AppendLayers: %v", err) - } - digest, err := img.Digest() - if err != nil { - t.Fatalf("img.Digest: %v", err) - } - tag, err := name.ParseReference(ref, name.Insecure) - if err != nil { - t.Fatalf("name.ParseReference(%q): %v", ref, err) - } - if err := remote.Write(tag, img); err != nil { - t.Fatalf("remote.Write(%q): %v", ref, err) - } - return digest -} - func gvisorConfig(name, url, sha string) *v1alpha1.SandboxConfig { return &v1alpha1.SandboxConfig{ ObjectMeta: metav1.ObjectMeta{Name: name}, @@ -77,8 +49,7 @@ func gvisorConfig(name, url, sha string) *v1alpha1.SandboxConfig { SandboxClass: v1alpha1.SandboxClassGvisor, DefaultVersion: "v1", Versions: []v1alpha1.SandboxVersionConfig{{ - Name: "v1", - PauseImage: "registry.k8s.io/pause@sha256:abc", + Name: "v1", Assets: map[string]map[string]v1alpha1.AssetFile{ runtime.GOARCH: { gvisorAssetName: {URL: url, SHA256: sha}, @@ -100,35 +71,21 @@ func (hangingObjectStorage) GetObject(ctx context.Context, _, _ string) (io.Read func (hangingObjectStorage) PutObject(_ context.Context, _, _ string, _ io.Reader) error { return nil } -// pauseImageInTestRegistry pushes a pause image to a test registry and -// returns its digest-pinned reference, the only form a runtime hash accepts. -func pauseImageInTestRegistry(t *testing.T) (ref string, host string) { - t.Helper() - host = imageVolumeTestRegistry(t) - digest := pushPauseImage(t, host+"/pause:3.10") - return host + "/pause@" + digest.String(), host -} - // indexedTestRuntime registers a runtime with one gvisor release asset, the -// tarball content, and the given pause image as a worker's default, and -// returns its hash. -func indexedTestRuntime(t *testing.T, index *runtimeIndex, content []byte, pauseImage string) string { +// tarball content, as a worker's default, and returns its hash. +func indexedTestRuntime(t *testing.T, index *runtimeIndex, content []byte) string { t.Helper() - return indexedTestRuntimeAt(t, index, "gs://bucket/gvisor.tar", content, pauseImage) + return indexedTestRuntimeAt(t, index, "gs://bucket/gvisor.tar", content) } // indexedTestRuntimeAt is indexedTestRuntime with the asset at url. -func indexedTestRuntimeAt(t *testing.T, index *runtimeIndex, url string, content []byte, pauseImage string) string { +func indexedTestRuntimeAt(t *testing.T, index *runtimeIndex, url string, content []byte) string { t.Helper() rec := &sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: pauseImage, Assets: map[string]assetEntry{gvisorAssetName: {URL: url, SHA256: fmt.Sprintf("%x", sha256.Sum256(content))}}, } - hash, err := versionBundleHash(rec) - if err != nil { - t.Fatal(err) - } + hash := bundleHash(rec) index.register("pod-a", advertisedClass{class: v1alpha1.SandboxClassGvisor, config: "gvisor-release", hashed: []hashedVersion{{ versionPlan: versionPlan{name: "v1", isDefault: true, rec: rec}, hash: hash, }}}) @@ -147,25 +104,16 @@ func defaultItem(t *testing.T, root, hash string) prewarmItem { return prewarmItem{hash: hash, dir: dir} } -func newTestImageStore(t *testing.T) *imagecache.Store { - t.Helper() - store, err := imagecache.New(t.TempDir()) - if err != nil { - t.Fatalf("imagecache.New: %v", err) - } - return store -} - // TestRuntimePrewarmProcess covers the worker's failure handling: a failed // prewarm is requeued with backoff until prewarmMaxRetries and then dropped, // and an item naming a hash that is not in the index is dropped at once. func TestRuntimePrewarmProcess(t *testing.T) { ctx := context.Background() index := newRuntimeIndex() - hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t), "") + hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t)) root := t.TempDir() herder := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}} - p := newRuntimePrewarmer(index, herder, newTestImageStore(t)) + p := newRuntimePrewarmer(index, herder) item := defaultItem(t, root, hash) for i := 1; i <= prewarmMaxRetries; i++ { @@ -194,7 +142,7 @@ func TestRuntimePrewarmSuperseded(t *testing.T) { ctx := context.Background() index := newRuntimeIndex() content := gvisorReleaseTarball(t) - hash := indexedTestRuntime(t, index, content, "") + hash := indexedTestRuntime(t, index, content) root := t.TempDir() herder := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: content}} @@ -203,7 +151,7 @@ func TestRuntimePrewarmSuperseded(t *testing.T) { if err := os.RemoveAll(item.dir); err != nil { t.Fatal(err) } - p := newRuntimePrewarmer(index, herder, newTestImageStore(t)) + p := newRuntimePrewarmer(index, herder) p.process(ctx, item) if got := p.queue.NumRequeues(item); got != 0 { t.Errorf("NumRequeues after a superseded prewarm = %d, want 0 (not retried)", got) @@ -219,7 +167,7 @@ func TestRuntimePrewarmSuperseded(t *testing.T) { t.Run("worker gone while staging", func(t *testing.T) { item := defaultItem(t, root, hash) removing := &removingFetcher{inner: herder, remove: filepath.Join(root, "workers", "pod-a")} - p := newRuntimePrewarmer(index, removing, newTestImageStore(t)) + p := newRuntimePrewarmer(index, removing) p.process(ctx, item) if got := p.queue.NumRequeues(item); got != 0 { t.Errorf("NumRequeues after the worker went mid-staging = %d, want 0 (not retried)", got) @@ -248,25 +196,23 @@ func (r *removingFetcher) stageSandboxAssets(ctx context.Context, rec *sandboxAs return r.inner.stageSandboxAssets(ctx, rec, dstDir) } -// TestRuntimePrewarmFillsBothCaches covers a successful prewarm: the asset +// TestRuntimePrewarmFillsTheAssetCache covers a successful prewarm: the asset // lands in the asset cache, linked into the runtime's directory so an -// eviction pass keeps it, and the pause image lands in the image cache. A -// second prewarm of the same directory fetches nothing. -func TestRuntimePrewarmFillsBothCaches(t *testing.T) { +// eviction pass keeps it. A second prewarm of the same directory fetches +// nothing. +func TestRuntimePrewarmFillsTheAssetCache(t *testing.T) { ctx := context.Background() - pauseRef, _ := pauseImageInTestRegistry(t) content := gvisorReleaseTarball(t) index := newRuntimeIndex() - hash := indexedTestRuntime(t, index, content, pauseRef) + hash := indexedTestRuntime(t, index, content) rt, _ := index.lookup(hash) root := t.TempDir() - images := newTestImageStore(t) var gets atomic.Int32 // No min age, so the only thing standing between the entry and a pass is // the directory's link. s := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root, filecache.WithMinAge(0)), anonGCSClient: fakeObjectStorage{data: content, gets: &gets}} - p := newRuntimePrewarmer(index, s, images) + p := newRuntimePrewarmer(index, s) item := defaultItem(t, root, hash) if err := p.prewarm(ctx, item, rt); err != nil { t.Fatalf("prewarm: %v", err) @@ -277,9 +223,6 @@ func TestRuntimePrewarmFillsBothCaches(t *testing.T) { if _, err := os.Stat(filepath.Join(item.dir, gvisorAssetName, "runsc")); err != nil { t.Errorf("prewarm did not link the runtime's asset into its directory: %v", err) } - if _, err := images.EnsureImage(ctx, pauseRef); err != nil { - t.Errorf("pause image not prewarmed into the image cache: %v", err) - } stats, err := s.sandboxAssetCache.EvictUnused(ctx, 1<<40, false) if err != nil { t.Fatalf("EvictUnused: %v", err) @@ -307,24 +250,19 @@ func TestRuntimePrewarmFillsBothCaches(t *testing.T) { } } -// TestRuntimePrewarmAssetFailureStillPullsImage pins that the two halves -// are independent: a failing asset fetch does not stop the pause image pull. -func TestRuntimePrewarmAssetFailureStillPullsImage(t *testing.T) { +// TestRuntimePrewarmAssetFailure pins that a failing asset fetch fails the +// prewarm. +func TestRuntimePrewarmAssetFailure(t *testing.T) { ctx := context.Background() - pauseRef, _ := pauseImageInTestRegistry(t) index := newRuntimeIndex() - hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t), pauseRef) + hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t)) rt, _ := index.lookup(hash) - images := newTestImageStore(t) root := t.TempDir() - p := newRuntimePrewarmer(index, &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}}, images) + p := newRuntimePrewarmer(index, &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}}) if err := p.prewarm(ctx, defaultItem(t, root, hash), rt); err == nil { t.Error("prewarm returned nil despite the asset fetch failing") } - if _, err := images.EnsureImage(ctx, pauseRef); err != nil { - t.Errorf("pause image not pulled when the asset fetch failed: %v", err) - } } // TestRuntimePrewarmAttemptTimeout verifies a single attempt is bounded by @@ -332,10 +270,10 @@ func TestRuntimePrewarmAssetFailureStillPullsImage(t *testing.T) { // that never returned would block every other runtime's prewarm. func TestRuntimePrewarmAttemptTimeout(t *testing.T) { index := newRuntimeIndex() - hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t), "") + hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t)) rt, _ := index.lookup(hash) root := t.TempDir() - p := newRuntimePrewarmer(index, &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: hangingObjectStorage{}}, newTestImageStore(t)) + p := newRuntimePrewarmer(index, &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: hangingObjectStorage{}}) p.attemptTimeout = 50 * time.Millisecond item := defaultItem(t, root, hash) @@ -479,9 +417,9 @@ func (r *recordingPrewarm) count() int { // request. func TestDefaultRuntimePrewarmThroughSyncer(t *testing.T) { useTempAssetScratch(t) // the syncer's runsc probe stages into it - // The worker's own prewarm call returns only when the asset has landed - // and the pause-image pull has finished. Waiting for the call keeps the - // pull's last writes from racing the TempDir cleanup. + // The worker's own prewarm call returns only when the asset has landed. + // Waiting for the call keeps its last writes from racing the TempDir + // cleanup. prewarmed := make(chan error, 1) onFinished := func(_ prewarmItem, err error) { // Non-blocking: a failed attempt is retried, and the worker must not @@ -492,10 +430,8 @@ func TestDefaultRuntimePrewarmThroughSyncer(t *testing.T) { } } - pauseRef, _ := pauseImageInTestRegistry(t) content := gvisorReleaseTarball(t) cfg := gvisorConfig("gvisor-release", "gs://bucket/gvisor.tar", fmt.Sprintf("%x", sha256.Sum256(content))) - cfg.Spec.Versions[0].PauseImage = pauseRef kube := kubefake.NewClientset([]k8sruntime.Object{workerPod("pod-a", "pool")}...) ate := atefake.NewSimpleClientset(resolvedPool("pool", "gvisor-release"), cfg) @@ -507,7 +443,7 @@ func TestDefaultRuntimePrewarmThroughSyncer(t *testing.T) { workersDir := filepath.Join(root, "workers") // on the cache's filesystem index := newRuntimeIndex() herder := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: content}} - p := newRuntimePrewarmer(index, herder, newTestImageStore(t)) + p := newRuntimePrewarmer(index, herder) p.onFinished = onFinished go p.run(ctx) // Join the worker before the TempDir cleanups remove what it writes into. diff --git a/cmd/atelet/sandbox_assets.go b/cmd/atelet/sandbox_assets.go index c5043cbecc..bea59b8a62 100644 --- a/cmd/atelet/sandbox_assets.go +++ b/cmd/atelet/sandbox_assets.go @@ -42,7 +42,6 @@ import ( "go.ate.dev/substrate/cmd/atelet/internal/filecache" "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/resources" - "go.ate.dev/substrate/pkg/api/v1alpha1" "go.ate.dev/substrate/pkg/objectstorage" objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc/codes" @@ -77,19 +76,14 @@ func (e assetEntry) displayURL() string { } // sandboxAssetsRecord is the sandbox runtime an actor is running, projected onto -// the local node's architecture: the sandbox class and pause image plus the -// asset set keyed by asset name (gVisor uses a single "gvisor" release-tarball -// asset). +// the local node's architecture: the sandbox class plus the asset set keyed by +// asset name (gVisor uses a single "gvisor" release-tarball asset). // It is both the per-actor on-node record (written at Run/Restore, read at // Checkpoint) and the snapshot manifest (written at Checkpoint, read at // Restore). type sandboxAssetsRecord struct { SandboxClass string `json:"sandboxClass"` Assets map[string]assetEntry `json:"assets"` - // PauseImage is the root sandbox container's image. It is recorded here - // rather than taken from the request at Restore so a snapshot is rebuilt - // with the same sandbox it was captured from. Only gVisor has one. - PauseImage string `json:"pauseImage,omitempty"` // Actor identity makes a flat snapshot self-identifying if control-plane // persistence is unavailable. Atespace string `json:"atespace,omitempty"` @@ -114,15 +108,6 @@ type sandboxAssetsRecord struct { Fidelity string `json:"fidelity,omitempty"` } -// checkPauseImage requires a pause image for gVisor, the only sandbox class -// that runs a pause container. -func checkPauseImage(sandboxClass, pauseImage string) error { - if sandboxClass == string(v1alpha1.SandboxClassGvisor) && pauseImage == "" { - return fmt.Errorf("gvisor sandbox has no pauseImage") - } - return nil -} - // ensureSandboxAssets stages every asset in the record under dstDir, named // by asset name, and returns a map of asset name to the local path the worker // runs it from. gVisor has a single "gvisor" release-tarball asset (staged @@ -676,9 +661,6 @@ func unmarshalSandboxRecord(data []byte) (*sandboxAssetsRecord, error) { if err := json.Unmarshal(data, rec); err != nil { return nil, fmt.Errorf("while parsing sandbox record/manifest: %w", err) } - if err := checkPauseImage(rec.SandboxClass, rec.PauseImage); err != nil { - return nil, fmt.Errorf("invalid sandbox record/manifest: %w", err) - } if err := validateSnapshotFiles(rec.SnapshotFiles); err != nil { return nil, fmt.Errorf("sandbox record/manifest has invalid snapshotFiles: %w", err) } diff --git a/cmd/atelet/sandbox_runtime_index.go b/cmd/atelet/sandbox_runtime_index.go index 0dc8050808..4c5124df48 100644 --- a/cmd/atelet/sandbox_runtime_index.go +++ b/cmd/atelet/sandbox_runtime_index.go @@ -158,11 +158,7 @@ func checkIndexedRuntime(hash string, e indexedRuntimeFile) error { if e.Record == nil { return errors.New("no record") } - got, err := versionBundleHash(e.Record) - if err != nil { - return fmt.Errorf("while hashing the record: %w", err) - } - if got != hash { + if got := bundleHash(e.Record); got != hash { return fmt.Errorf("record hashes to %s", got) } return nil @@ -237,9 +233,9 @@ func (x *runtimeIndex) registerRuntimes(config string, hashed []hashedVersion) e // addRuntime records v under its hash, unless the hash is already known, // persisting the index first when it has a path. The hash covers every -// asset's name and digest and the pause image, so two versions with one -// hash differ at most in where identical bytes are fetched from; keeping -// the first keeps the version the file was built from. Called with mu held. +// asset's name and digest, so two versions with one hash differ at most in +// where identical bytes are fetched from; keeping the first keeps the version +// the file was built from. Called with mu held. func (x *runtimeIndex) addRuntime(config string, v hashedVersion) error { if _, ok := x.runtimes[v.hash]; ok { return nil @@ -288,7 +284,7 @@ func (x *runtimeIndex) worker(podUID string) (*workerRuntime, bool) { // A miss means the request reached a node whose workers never advertised the // runtime, which must surface. A runtime registered before an atelet restart // is no miss: the index is loaded from disk before atelet serves. -func (s *Atelet) advertisedRuntime(ctx context.Context, rt *ateletpb.SandboxRuntime) (*sandboxAssetsRecord, error) { +func (s *Atelet) advertisedRuntime(ctx context.Context, rt *ateletpb.ActorSandboxRuntime) (*sandboxAssetsRecord, error) { hash, err := bundleHashOf(rt) if err != nil { return nil, apierror.InvalidArgument("sandbox_runtime: %v", err) @@ -308,7 +304,7 @@ func (s *Atelet) advertisedRuntime(ctx context.Context, rt *ateletpb.SandboxRunt // bundleHashOf returns the sandbox_bundle_hash attribute of rt, which names // the SandboxConfig version a worker runs an actor under. -func bundleHashOf(rt *ateletpb.SandboxRuntime) (string, error) { +func bundleHashOf(rt *ateletpb.ActorSandboxRuntime) (string, error) { if rt == nil { return "", errors.New("missing") } diff --git a/cmd/atelet/sandbox_runtime_index_test.go b/cmd/atelet/sandbox_runtime_index_test.go index 535939924c..57d4f9a60a 100644 --- a/cmd/atelet/sandbox_runtime_index_test.go +++ b/cmd/atelet/sandbox_runtime_index_test.go @@ -35,14 +35,9 @@ func hashedTestVersion(t *testing.T, name string, isDefault bool, sha string) ha t.Helper() rec := &sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: testPauseImage, Assets: map[string]assetEntry{gvisorAssetName: {URL: "gs://bucket/" + name, SHA256: sha}}, } - h, err := versionBundleHash(rec) - if err != nil { - t.Fatal(err) - } - return hashedVersion{versionPlan: versionPlan{name: name, isDefault: isDefault, rec: rec}, hash: h} + return hashedVersion{versionPlan: versionPlan{name: name, isDefault: isDefault, rec: rec}, hash: bundleHash(rec)} } func TestRuntimeIndex(t *testing.T) { @@ -148,8 +143,8 @@ func TestResolveRuntime(t *testing.T) { const sha = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" s := &Atelet{runtimes: newRuntimeIndex()} advertised := hashedTestVersion(t, "v1", true, sha) - runtimeWith := func(class string, attrs ...*ateletpb.AttributeEntry) *ateletpb.SandboxRuntime { - return &ateletpb.SandboxRuntime{SandboxClass: class, Version: &ateletpb.VersionedSandboxCompat{SchemaVersion: hardware.SchemaVersionV1, Attributes: attrs}} + runtimeWith := func(class string, attrs ...*ateletpb.AttributeEntry) *ateletpb.ActorSandboxRuntime { + return &ateletpb.ActorSandboxRuntime{SandboxClass: class, Version: &ateletpb.VersionedSandboxCompat{SchemaVersion: hardware.SchemaVersionV1, Attributes: attrs}} } placed := runtimeWith("gvisor", &ateletpb.AttributeEntry{Key: hardware.AttrArchitecture, Value: "amd64"}, @@ -167,7 +162,7 @@ func TestResolveRuntime(t *testing.T) { t.Errorf("advertisedRuntime returned %+v, want the advertised record", rec) } - for name, rt := range map[string]*ateletpb.SandboxRuntime{ + for name, rt := range map[string]*ateletpb.ActorSandboxRuntime{ "no runtime": nil, "no attributes": runtimeWith("gvisor"), "no bundle hash": runtimeWith("gvisor", diff --git a/cmd/atelet/snapshot_transfer.go b/cmd/atelet/snapshot_transfer.go index 83fdfa5b35..fedac43674 100644 --- a/cmd/atelet/snapshot_transfer.go +++ b/cmd/atelet/snapshot_transfer.go @@ -48,45 +48,57 @@ func pluginCallError(op, snapshotURI string, err error) error { return status.Errorf(code, "object-store plugin %s of %s failed (%s)", op, redactURLString(snapshotURI), code) } -// fetchSnapshotFiles downloads the named snapshot files into dstDir. -func (s *Atelet) fetchSnapshotFiles(ctx context.Context, snapshotURI, dstDir string, files []string) error { +// fetchSnapshotFiles downloads the named snapshot files into dstDir, with the +// JWT the plugin authorizes the transfer for actor with. +func (s *Atelet) fetchSnapshotFiles(ctx context.Context, actor actorRef, snapshotURI, dstDir string, files []string) error { if len(files) == 0 { return nil } - _, err := s.objectStorePlugin.FetchSnapshot(ctx, &objectstorev1.FetchSnapshotRequest{ + // A mint failure is shaped like a plugin failure and mapped the same way. + jwt, err := s.workerActorJWT(ctx, actor) + if err != nil { + return pluginsocket.CallError(err) + } + _, err = s.objectStorePlugin.FetchSnapshot(ctx, &objectstorev1.FetchSnapshotRequest{ SnapshotUri: snapshotURI, WritePath: dstDir, Files: files, + ActorJwt: jwt, }) return pluginsocket.CallError(pluginCallError("fetch", snapshotURI, err)) } // uploadSnapshotFiles uploads the named files in srcDir to the snapshot, // passing opts to the plugin call. It returns the plugin's code without its -// message, and leaves the Unavailable mapping to the caller: an upload that can -// be retried reports it through pluginsocket.CallError, one that cannot -// does not. -func (s *Atelet) uploadSnapshotFiles(ctx context.Context, snapshotURI, srcDir string, files []string, opts ...grpc.CallOption) error { +// message, or the mint's, and leaves the Unavailable mapping to the caller: +// an upload that can be retried reports it through pluginsocket.CallError, +// one that cannot does not. +func (s *Atelet) uploadSnapshotFiles(ctx context.Context, actor actorRef, snapshotURI, srcDir string, files []string, opts ...grpc.CallOption) error { if len(files) == 0 { return nil } - _, err := s.objectStorePlugin.UploadSnapshot(ctx, &objectstorev1.UploadSnapshotRequest{ + jwt, err := s.workerActorJWT(ctx, actor) + if err != nil { + return err + } + _, err = s.objectStorePlugin.UploadSnapshot(ctx, &objectstorev1.UploadSnapshotRequest{ SnapshotUri: snapshotURI, LocalPath: srcDir, Files: files, + ActorJwt: jwt, }, opts...) return pluginCallError("upload", snapshotURI, err) } // fetchManifest returns a snapshot's manifest. The returned error is the // plugin's gRPC status, so a missing manifest reports codes.NotFound. -func (s *Atelet) fetchManifest(ctx context.Context, snapshotURI string) ([]byte, error) { +func (s *Atelet) fetchManifest(ctx context.Context, actor actorRef, snapshotURI string) ([]byte, error) { dir, err := s.snapshotScratch() if err != nil { return nil, err } defer func() { _ = os.RemoveAll(dir) }() - if err := s.fetchSnapshotFiles(ctx, snapshotURI, dir, []string{sandboxManifestName}); err != nil { + if err := s.fetchSnapshotFiles(ctx, actor, snapshotURI, dir, []string{sandboxManifestName}); err != nil { return nil, err } return os.ReadFile(filepath.Join(dir, sandboxManifestName)) @@ -94,7 +106,7 @@ func (s *Atelet) fetchManifest(ctx context.Context, snapshotURI string) ([]byte, // uploadManifest uploads manifest as the snapshot's manifest, as // uploadSnapshotFiles does. -func (s *Atelet) uploadManifest(ctx context.Context, snapshotURI string, manifest []byte, opts ...grpc.CallOption) error { +func (s *Atelet) uploadManifest(ctx context.Context, actor actorRef, snapshotURI string, manifest []byte, opts ...grpc.CallOption) error { dir, err := s.snapshotScratch() if err != nil { return err @@ -103,7 +115,7 @@ func (s *Atelet) uploadManifest(ctx context.Context, snapshotURI string, manifes if err := os.WriteFile(filepath.Join(dir, sandboxManifestName), manifest, 0o600); err != nil { return fmt.Errorf("while staging snapshot manifest: %w", err) } - return s.uploadSnapshotFiles(ctx, snapshotURI, dir, []string{sandboxManifestName}, opts...) + return s.uploadSnapshotFiles(ctx, actor, snapshotURI, dir, []string{sandboxManifestName}, opts...) } // snapshotScratch creates a fresh directory for one manifest transfer. The diff --git a/cmd/atelet/snapshot_transfer_test.go b/cmd/atelet/snapshot_transfer_test.go index d92692998d..d46518c376 100644 --- a/cmd/atelet/snapshot_transfer_test.go +++ b/cmd/atelet/snapshot_transfer_test.go @@ -57,6 +57,7 @@ func newPluginHerder(t *testing.T, store objectstorage.ObjectStorage) *Atelet { } return &Atelet{ objectStorePlugin: objectstoreplugintest.NodeClient(plugin), + workerActorJWTs: testWorkerActorJWTs(), snapshotScratchDir: t.TempDir(), snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), sandboxAssetCache: newTestAssetCache(t), @@ -104,14 +105,14 @@ func TestUploadSnapshotManifestLast(t *testing.T) { t.Fatal(err) } dir := t.TempDir() - rec := sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: testPauseImage, SnapshotFiles: []string{"a", "b", "c"}} + rec := sandboxAssetsRecord{SandboxClass: "gvisor", SnapshotFiles: []string{"a", "b", "c"}} for _, f := range rec.SnapshotFiles { if err := os.WriteFile(dir+"/"+f, []byte(f), 0o600); err != nil { t.Fatal(err) } } - if err := s.uploadSnapshot(context.Background(), uri, dir, &rec); err != nil { + if err := s.uploadSnapshot(context.Background(), testActor, uri, dir, &rec); err != nil { t.Fatalf("uploadSnapshot: %v", err) } if len(store.puts) != 4 { @@ -125,7 +126,7 @@ func TestUploadSnapshotManifestLast(t *testing.T) { } // The manifest round-trips, and scratch space is cleaned up. - got, err := s.fetchManifest(context.Background(), uri.String()) + got, err := s.fetchManifest(context.Background(), testActor, uri.String()) if err != nil { t.Fatalf("fetchManifest: %v", err) } @@ -144,10 +145,10 @@ func TestUploadSnapshotManifestLast(t *testing.T) { func TestSnapshotTransferSkipsEmptyFileLists(t *testing.T) { store := &orderedObjectStorage{} s := newPluginHerder(t, store) - if err := s.downloadExternalCheckpoint(context.Background(), pausedSnapshotURI, t.TempDir(), nil); err != nil { + if err := s.downloadExternalCheckpoint(context.Background(), testActor, pausedSnapshotURI, t.TempDir(), nil, cacheModeOff); err != nil { t.Errorf("downloadExternalCheckpoint(no files) = %v, want nil", err) } - if err := s.uploadSnapshotFiles(context.Background(), pausedSnapshotURI, t.TempDir(), nil); err != nil { + if err := s.uploadSnapshotFiles(context.Background(), testActor, pausedSnapshotURI, t.TempDir(), nil); err != nil { t.Errorf("uploadSnapshotFiles(no files) = %v, want nil", err) } if len(store.puts) != 0 { @@ -180,7 +181,6 @@ func TestObjectStorePluginErrorCodeThroughAtelet(t *testing.T) { } rec := sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: testPauseImage, SnapshotFiles: []string{"a"}, Fidelity: ateattr.SnapshotFidelityMemory, } @@ -190,7 +190,13 @@ func TestObjectStorePluginErrorCodeThroughAtelet(t *testing.T) { run func(ctx context.Context, s *Atelet) error }{ {"download", true, func(ctx context.Context, s *Atelet) error { - return s.downloadExternalCheckpoint(ctx, pausedSnapshotURI, t.TempDir(), []string{"a"}) + return s.downloadExternalCheckpoint(ctx, testActor, pausedSnapshotURI, t.TempDir(), []string{"a"}, cacheModeOff) + }}, + {"cached download, link", true, func(ctx context.Context, s *Atelet) error { + return s.downloadExternalCheckpoint(ctx, testActor, pausedSnapshotURI, t.TempDir(), []string{"a"}, cacheModeLink) + }}, + {"cached download, copy", true, func(ctx context.Context, s *Atelet) error { + return s.downloadExternalCheckpoint(ctx, testActor, pausedSnapshotURI, t.TempDir(), []string{"a"}, cacheModeCopy) }}, {"paused upload", true, func(ctx context.Context, s *Atelet) error { dir := t.TempDir() @@ -215,7 +221,9 @@ func TestObjectStorePluginErrorCodeThroughAtelet(t *testing.T) { } s := &Atelet{ objectStorePlugin: failingNodePlugin{err: status.Error(pluginCode, "plugin failed")}, + workerActorJWTs: testWorkerActorJWTs(), snapshotScratchDir: t.TempDir(), + snapshotCache: newTestAssetCache(t), } // atelet's server interceptor decides the code ate-api-server sees. _, err := ateinterceptors.InternalServerUnaryInterceptor(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "/test"}, @@ -228,6 +236,59 @@ func TestObjectStorePluginErrorCodeThroughAtelet(t *testing.T) { } } +// A mint failure reaches ate-api-server the way a plugin failure does: +// Unavailable where the transfer can be retried, Internal where it cannot, +// and never NotFound, which callers read as a missing snapshot object. +func TestMintErrorCodeThroughAtelet(t *testing.T) { + uri, err := resources.ParseSnapshotURI(pausedSnapshotURI) + if err != nil { + t.Fatal(err) + } + rec := sandboxAssetsRecord{SandboxClass: "gvisor", SnapshotFiles: []string{"a"}, Fidelity: ateattr.SnapshotFidelityMemory} + transfers := []struct { + name string + retryable bool + run func(ctx context.Context, s *Atelet) error + }{ + {"download", true, func(ctx context.Context, s *Atelet) error { + return s.downloadExternalCheckpoint(ctx, testActor, pausedSnapshotURI, t.TempDir(), []string{"a"}, cacheModeOff) + }}, + {"paused upload", true, func(ctx context.Context, s *Atelet) error { + dir := t.TempDir() + writeLocalSnapshot(t, dir, rec, map[string]string{"a": "a"}) + _, _, err := s.uploadLocalCheckpointDir(ctx, validPromoteSnapshotRequest(), dir, uri) + return err + }}, + {"checkpoint upload", false, func(ctx context.Context, s *Atelet) error { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "a"), []byte("a"), 0o600); err != nil { + return err + } + return s.uploadExternalCheckpoint(ctx, validCheckpointRequest(), dir, &rec) + }}, + } + for _, mint := range []codes.Code{codes.Unavailable, codes.NotFound, codes.Aborted, codes.Internal} { + for _, tr := range transfers { + t.Run(mint.String()+"/"+tr.name, func(t *testing.T) { + want := codes.Internal + if mint == codes.Unavailable && tr.retryable { + want = codes.Unavailable + } + s := &Atelet{ + objectStorePlugin: failingNodePlugin{err: status.Error(codes.Internal, "the plugin must not be reached")}, + workerActorJWTs: newWorkerActorJWTs(&fakeJWTMinter{err: status.Error(mint, "mint failed")}), + snapshotScratchDir: t.TempDir(), + } + _, err := ateinterceptors.InternalServerUnaryInterceptor(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "/test"}, + func(ctx context.Context, _ any) (any, error) { return nil, tr.run(ctx, s) }) + if got := status.Code(err); got != want { + t.Errorf("code = %s (%v), want %s", got, err, want) + } + }) + } + } +} + // An object-store plugin's status text can quote a signed URL or other credential. // Neither it nor the snapshot URI's query reaches atelet's RPC log or the // error ate-api-server receives: only the plugin's code and the snapshot URI @@ -242,20 +303,28 @@ func TestObjectStorePluginErrorKeepsCredentialsOut(t *testing.T) { if err != nil { t.Fatal(err) } - rec := sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: testPauseImage, SnapshotFiles: []string{"a"}, Fidelity: ateattr.SnapshotFidelityMemory} + rec := sandboxAssetsRecord{SandboxClass: "gvisor", SnapshotFiles: []string{"a"}, Fidelity: ateattr.SnapshotFidelityMemory} transfers := []struct { name string uri string // the URI the transfer names in its error, when it takes one run func(ctx context.Context, s *Atelet) error }{ {"download", signedURI, func(ctx context.Context, s *Atelet) error { - return s.downloadExternalCheckpoint(ctx, signedURI, t.TempDir(), []string{"a"}) + return s.downloadExternalCheckpoint(ctx, testActor, signedURI, t.TempDir(), []string{"a"}, cacheModeOff) + }}, + // The cached modes also build a cache key from the URI, which the + // cache quotes in its errors and logs. + {"cached download, link", signedURI, func(ctx context.Context, s *Atelet) error { + return s.downloadExternalCheckpoint(ctx, testActor, signedURI, t.TempDir(), []string{"a"}, cacheModeLink) + }}, + {"cached download, copy", signedURI, func(ctx context.Context, s *Atelet) error { + return s.downloadExternalCheckpoint(ctx, testActor, signedURI, t.TempDir(), []string{"a"}, cacheModeCopy) }}, {"upload", signedURI, func(ctx context.Context, s *Atelet) error { - return s.uploadSnapshotFiles(ctx, signedURI, t.TempDir(), []string{"a"}) + return s.uploadSnapshotFiles(ctx, testActor, signedURI, t.TempDir(), []string{"a"}) }}, {"manifest fetch", signedURI, func(ctx context.Context, s *Atelet) error { - _, err := s.fetchManifest(ctx, signedURI) + _, err := s.fetchManifest(ctx, testActor, signedURI) return err }}, {"paused upload", "", func(ctx context.Context, s *Atelet) error { @@ -271,7 +340,9 @@ func TestObjectStorePluginErrorKeepsCredentialsOut(t *testing.T) { logs := captureLogs(t) s := &Atelet{ objectStorePlugin: failingNodePlugin{err: status.Error(pluginCode, "get "+signedURI+": AccountKey="+secret)}, + workerActorJWTs: testWorkerActorJWTs(), snapshotScratchDir: t.TempDir(), + snapshotCache: newTestAssetCache(t), } _, err := ateinterceptors.InternalServerUnaryInterceptor(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "/test"}, func(ctx context.Context, _ any) (any, error) { return nil, tr.run(ctx, s) }) @@ -366,7 +437,7 @@ func (s *pluginSidecar) stop() { } // The upload after CheckpointWorkload waits out a plugin outage longer than -// the ready wait, since a retried Checkpoint cannot redo it. The paused +// the ready wait, since a retried Checkpoint cannot redo it. The PromoteSnapshot // upload, which a retry can redo, still fails within the ready wait. func TestCheckpointUploadWaitsOutPluginOutage(t *testing.T) { const wait = 200 * time.Millisecond @@ -385,11 +456,11 @@ func TestCheckpointUploadWaitsOutPluginOutage(t *testing.T) { } s := &Atelet{ objectStorePlugin: objectstorev1.NewNodeProviderClient(conn), + workerActorJWTs: testWorkerActorJWTs(), snapshotScratchDir: t.TempDir(), } rec := sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: testPauseImage, SnapshotFiles: []string{"a"}, Fidelity: ateattr.SnapshotFidelityMemory, } @@ -441,14 +512,15 @@ func TestCheckpointUploadWaitsOutPluginOutage(t *testing.T) { } } -// fetchManifest keeps the plugin's NotFound readable, which the paused +// fetchManifest keeps the plugin's NotFound readable, which the local // snapshot upload uses to tell a missing snapshot from a failed probe. func TestFetchManifestKeepsNotFound(t *testing.T) { s := &Atelet{ objectStorePlugin: failingNodePlugin{err: status.Error(codes.NotFound, "no manifest")}, + workerActorJWTs: testWorkerActorJWTs(), snapshotScratchDir: t.TempDir(), } - _, err := s.fetchManifest(context.Background(), pausedSnapshotURI) + _, err := s.fetchManifest(context.Background(), testActor, pausedSnapshotURI) if status.Code(err) != codes.NotFound { t.Errorf("fetchManifest = %v, want %s", err, codes.NotFound) } diff --git a/cmd/atelet/snapshotcache.go b/cmd/atelet/snapshotcache.go new file mode 100644 index 0000000000..d7eeb28769 --- /dev/null +++ b/cmd/atelet/snapshotcache.go @@ -0,0 +1,305 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +// The snapshot file cache. +// +// A shared snapshot's files, the snapshot a tag holds and every actor +// created from it restores, are immutable once published, yet every restore +// that needs them downloads them again into its own per-actor dir. This +// cache (a filecache.Store) makes those files a node-level resource: +// concurrent restores of one snapshot share a single download, and later +// restores are served from disk instead of object storage. +// +// Whether a snapshot is cacheable is the control plane's call, carried on +// the request as ObjectStorage.sharing; atelet only maps that property to a +// serving mode per sandbox class. This file holds the cache's flags and +// validation, its opening with the startup debris sweep, and the restore +// path's cached reads. The eviction loop is filecachegc, started from main. + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/url" + "os" + "path/filepath" + "syscall" + "time" + + "github.com/spf13/pflag" + "golang.org/x/sync/errgroup" + + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/cmd/atelet/internal/filecache" + "go.ate.dev/substrate/cmd/atelet/internal/filecachegc" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" +) + +var ( + snapshotCacheDir = pflag.String("snapshot-cache-dir", ateletpath.SnapshotCacheDir, "Directory for the node-local shared snapshot file cache. Empty disables caching: every restore downloads its snapshot files, and the other --snapshot-cache-* flags are validated but not read. Must be under the node state dir, "+nodepath.BasePath+": the object-store plugin writes only there, and cache hits are served as hard links into the per-actor restore dirs on the same mount.") + snapshotCacheMinAge = pflag.Duration("snapshot-cache-min-age", 10*time.Minute, "Cached snapshot files younger than this are never evicted, protecting files fetched but not yet linked into a restore dir. Read only while the cache is enabled.") + snapshotCacheFetchTimeout = pflag.Duration("snapshot-cache-fetch-timeout", 10*time.Minute, "Bound on one cached snapshot file download. Downloads run detached from the restores waiting on them, so this is the only bound on how long one can run; a restore arriving while a download hangs waits on it until then. Sized for multi-GiB memory images on a busy node. Read only while the cache is enabled.") + // snapshotCacheGCFlags are --snapshot-cache-gc-period, + // --snapshot-cache-max-bytes and --snapshot-cache-gc-dry-run. + snapshotCacheGCFlags = filecachegc.NewFlags(pflag.CommandLine, "snapshot-cache", "snapshot cache") +) + +// validateSnapshotCacheFlags checks every flag of the cache, enabled or +// not: a value outside its domain is a mistake whatever else is set, and +// startup is the cheapest moment to say so. +func validateSnapshotCacheFlags() error { + if *snapshotCacheMinAge < 0 { + // A negative min-age inverts the veto (the cutoff lands in the + // future), making just-fetched files evictable mid-restore. + return fmt.Errorf("--snapshot-cache-min-age %v must be >= 0", *snapshotCacheMinAge) + } + if *snapshotCacheFetchTimeout <= 0 { + return fmt.Errorf("--snapshot-cache-fetch-timeout %v must be > 0", *snapshotCacheFetchTimeout) + } + if err := snapshotCacheGCFlags.Validate(); err != nil { + return err + } + if *snapshotCacheDir != "" && !nodepath.UnderBasePath(*snapshotCacheDir) { + // The object-store plugin confines its writes to the base path, so + // a cache elsewhere could never be filled: every shared restore + // would fail at the fetch. + return fmt.Errorf("--snapshot-cache-dir %q must be under %s, the only tree the object-store plugin writes to", *snapshotCacheDir, nodepath.BasePath) + } + return nil +} + +// openSnapshotCache opens the snapshot cache rooted at dir and clears crash +// debris before the store serves any restore. An empty dir disables +// caching: the returned store is nil and restores download their snapshot +// files directly. +func openSnapshotCache(ctx context.Context, dir string, minAge, fetchTimeout time.Duration) (*filecache.Store, error) { + if dir == "" { + slog.InfoContext(ctx, "Snapshot cache disabled; every restore downloads its snapshot files") + return nil, nil + } + store, err := filecache.New(dir, filecache.WithMinAge(minAge), filecache.WithFetchTimeout(fetchTimeout)) + if err != nil { + return nil, fmt.Errorf("while opening snapshot cache at %s: %w", dir, err) + } + stats, err := store.SweepDebris(ctx) + if err != nil { + // Leftover debris wastes space but affects no lookup, so the store + // is fully usable: log and carry on rather than failing startup. + slog.WarnContext(ctx, "Snapshot cache debris sweep incomplete", slog.Any("err", err)) + } + slog.InfoContext(ctx, "Snapshot cache open", + slog.String("dir", dir), + slog.Duration("min_age", minAge), + slog.Duration("fetch_timeout", fetchTimeout), + slog.Int("tmp_removed", stats.TmpRemoved), + slog.Int("retired_removed", stats.RetiredRemoved)) + return store, nil +} + +// cacheMode is how a restore may materialize snapshot files from the +// cache. The mode exists because the fast path, a hard link, shares the +// cached inode with the consumer, which is only safe when the consumer never +// writes the staged file in place. +type cacheMode int + +const ( + // cacheModeOff downloads fresh, bypassing the cache. + cacheModeOff cacheMode = iota + // cacheModeLink hard-links the cached copy: zero-cost hits, but the + // consumer must treat the staged file as read-only. + cacheModeLink + // cacheModeCopy stages a private, hole-preserving copy: costlier per hit + // than a link, but the inode is the consumer's alone, to write in place, + // to drop from the page cache, or to keep open without pinning the + // cache's entry, and a copy can cross mounts. + cacheModeCopy +) + +// cacheModeFor returns how this sandbox class's restores may use the +// snapshot cache. ateworker-gvisor reads restore state and gets hard +// links. Once the restore's page loading is done it drops the memory and +// filesystem images from the page cache and unlinks them from its restore +// dir, keeping only the small descriptors linked, so the actor's cgroup is +// charged for its guest memory alone, as for any restore, and a link +// protects an image's entry through the restore and the min age, not for +// the actor's life: an entry evicted later is still read to the end by the +// descriptors runsc holds open, and the next restore of the snapshot reads +// the images from local disk. ateworker-microvm gets private copies. Not because a link would +// corrupt the cache: it replaces config.json by rename, and its delta +// merge copies instead of writing in place when the memory file's inode +// is shared. Because a link would cost more than it saves: an on-demand +// restore keeps memory-ranges open for the VM's whole life, so a link +// would pin the cache's entry that long and turn every suspend from the +// in-place delta merge into a full copy, and the worker drops its restore +// files from the page cache, which on a shared inode would drop them for +// every other restore too. A copy is still one download per snapshot per +// node. +func cacheModeFor(sandboxClass string) cacheMode { + switch atev1alpha1.SandboxClass(sandboxClass) { + case atev1alpha1.SandboxClassGvisor: + return cacheModeLink + case atev1alpha1.SandboxClassMicroVM: + return cacheModeCopy + default: + return cacheModeOff + } +} + +// snapshotCacheMode returns the cache mode for a restore's durable +// snapshot: classMode when the control plane declared the snapshot shared, +// cacheModeOff otherwise, which covers private snapshots and a caller that +// did not say. atelet never infers cacheability from the URI. +func snapshotCacheMode(object *ateletpb.ObjectStorage, classMode cacheMode) cacheMode { + if object.GetSharing() == ateletpb.SnapshotSharing_SNAPSHOT_SHARING_SHARED { + return classMode + } + return cacheModeOff +} + +// downloadSnapshotFilesCached stages the named files of the snapshot at +// snapshotURI into dstDir through the snapshot cache, each per mode. The +// caller asserts the snapshot is immutable at its URI. Files sit directly +// in dstDir or in its one checkpoint subdirectory, as the plugin's own +// download allows; a destination that already exists, a symlink included, +// fails rather than being followed. +func (s *Atelet) downloadSnapshotFilesCached(ctx context.Context, actor actorRef, snapshotURI, dstDir string, files []string, mode cacheMode) error { + if err := resources.ValidateSnapshotFileNames(files); err != nil { + return err + } + // Subdirectories first, once each: the files are fetched in parallel, + // and several of them share the checkpoint subdirectory. + made := map[string]bool{} + for _, fileName := range files { + dir := filepath.Dir(fileName) + if dir == "." || made[dir] { + continue + } + if err := mkdirNoFollow(filepath.Join(dstDir, dir)); err != nil { + return fmt.Errorf("while creating %s in restore directory: %w", dir, err) + } + made[dir] = true + } + g, gCtx := errgroup.WithContext(ctx) + for _, fileName := range files { + g.Go(func() error { + if err := s.fetchSnapshotFile(gCtx, actor, snapshotURI, fileName, filepath.Join(dstDir, fileName), mode); err != nil { + return fmt.Errorf("while downloading %s: %w", fileName, err) + } + return nil + }) + } + return g.Wait() +} + +// mkdirNoFollow creates dir unless it already is a directory. Anything else +// in its place, a symlink above all, is an error: the restore directory is +// written by path here, outside the os.Root the plugin's own download uses. +func mkdirNoFollow(dir string) error { + fi, err := os.Lstat(dir) + switch { + case errors.Is(err, os.ErrNotExist): + return os.Mkdir(dir, 0o700) + case err != nil: + return err + case !fi.IsDir(): + return fmt.Errorf("%s exists and is not a directory", dir) + } + return nil +} + +// fetchManifestCached reads the manifest of the snapshot at snapshotURI, +// through the snapshot cache when mode allows it: the manifest of a shared +// snapshot is as immutable as its files, and a restore that finds both on +// the node makes no request to object storage at all. atelet reads the +// manifest itself, so a link serves whatever class the files will take; +// it is read from a scratch dir and the link dropped again. +func (s *Atelet) fetchManifestCached(ctx context.Context, actor actorRef, snapshotURI string, mode cacheMode) ([]byte, error) { + if mode == cacheModeOff || s.snapshotCache == nil { + return s.fetchManifest(ctx, actor, snapshotURI) + } + dir, err := s.snapshotScratch() + if err != nil { + return nil, err + } + defer func() { _ = os.RemoveAll(dir) }() + local := filepath.Join(dir, sandboxManifestName) + if err := s.fetchSnapshotFile(ctx, actor, snapshotURI, sandboxManifestName, local, cacheModeLink); err != nil { + return nil, err + } + return os.ReadFile(local) +} + +// fetchSnapshotFile stages one file of the snapshot at snapshotURI at +// local, through the snapshot cache per mode. A cacheModeLink hit is a hard +// link, so local stays valid regardless of later eviction; a cacheModeCopy +// hit is a private copy. Fetch errors pass through the cache wrapped, so +// the plugin's code classification survives. +func (s *Atelet) fetchSnapshotFile(ctx context.Context, actor actorRef, snapshotURI, fileName, local string, mode cacheMode) error { + // Keyed by the object's address without its query: a signed URL's + // query is a credential, and the key is logged and written to meta.json. + u, err := url.Parse(snapshotURI) + if err != nil { + return fmt.Errorf("while parsing the snapshot URI: %w", err) + } + objectURI, err := url.JoinPath(redactURL(u), fileName) + if err != nil { + return fmt.Errorf("while addressing snapshot file %q: %w", fileName, err) + } + key := filecache.URIKey(objectURI) + fetch := func(ctx context.Context, dst string) error { + return s.fetchSnapshotFileTo(ctx, actor, snapshotURI, fileName, dst) + } + if mode == cacheModeCopy { + return s.snapshotCache.GetFileCopyTo(ctx, key, local, fetch) + } + err = s.snapshotCache.GetFileTo(ctx, key, local, fetch) + if err == nil || !errors.Is(err, syscall.EXDEV) { + return err + } + // The cache sits on a different mount than the restore dir, a bind + // mount under the node state dir, so a link cannot work, but a copy + // can, and the failed link's fetch already published the entry, so + // this is a local read rather than a second download. It is the node's + // configuration, so it is said once, with the detail per file at debug. + s.snapshotCacheCrossMount.Do(func() { + slog.WarnContext(ctx, "Snapshot cache is on a different filesystem than the restore dirs; serving every hit as a copy instead of a hard link", + slog.String("snapshot_cache_dir", *snapshotCacheDir), slog.String("snapshots_dir", nodepath.SnapshotsDir)) + }) + slog.DebugContext(ctx, "Snapshot cache hit served as a copy across mounts", + slog.String("snapshot_uri", redactURL(u)), slog.String("file", fileName)) + return s.snapshotCache.GetFileCopyTo(ctx, key, local, fetch) +} + +// fetchSnapshotFileTo downloads one file of a snapshot to dst through the +// object-store plugin, which writes a directory of files: it fetches into +// a scratch dir beside dst, on the cache's filesystem, and moves the file +// into place. The plugin accepts a write path under the node state dir, +// which is where the cache lives. +func (s *Atelet) fetchSnapshotFileTo(ctx context.Context, actor actorRef, snapshotURI, fileName, dst string) error { + scratch, err := os.MkdirTemp(filepath.Dir(dst), ".fetch-*") + if err != nil { + return fmt.Errorf("while creating a snapshot fetch dir: %w", err) + } + defer func() { _ = os.RemoveAll(scratch) }() + if err := s.fetchSnapshotFiles(ctx, actor, snapshotURI, scratch, []string{fileName}); err != nil { + return err + } + return os.Rename(filepath.Join(scratch, fileName), dst) +} diff --git a/cmd/atelet/snapshotcache_linux_test.go b/cmd/atelet/snapshotcache_linux_test.go new file mode 100644 index 0000000000..75ed7af2ff --- /dev/null +++ b/cmd/atelet/snapshotcache_linux_test.go @@ -0,0 +1,84 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestDownloadExternalCheckpointLinkFallsBackToCopyAcrossMounts needs two +// real filesystems, which /dev/shm gives on Linux: a link-mode restore into +// a directory on another mount than the cache must degrade to a private +// copy, not to a download. An unusable /dev/shm fails the test rather than +// skipping it; this is where the fallback is proven. +func TestDownloadExternalCheckpointLinkFallsBackToCopyAcrossMounts(t *testing.T) { + s, store, base := newSnapshotCacheHerder(t, map[string]string{"memory-ranges": "golden memory"}) + ctx := context.Background() + files := []string{"memory-ranges"} + logs := captureLogs(t) + + otherFS, err := os.MkdirTemp("/dev/shm", "snapshot-cache-test-") + if err != nil { + t.Fatalf("creating a directory on /dev/shm: %v", err) + } + t.Cleanup(func() { os.RemoveAll(otherFS) }) + // Prove the two dirs really are different mounts; same-mount tmpdirs + // would silently test the plain link path. + probe := filepath.Join(base, "probe") + if err := os.WriteFile(probe, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Link(probe, filepath.Join(otherFS, "probe")); err == nil { + t.Fatal("test dirs share a filesystem; cannot exercise EXDEV") + } + + first := filepath.Join(otherFS, "first") + if err := os.Mkdir(first, 0o700); err != nil { + t.Fatal(err) + } + if err := s.downloadExternalCheckpoint(ctx, testActor, testSnapshotURI, first, files, cacheModeLink); err != nil { + t.Fatalf("cross-mount link-mode restore: %v", err) + } + if got, err := os.ReadFile(filepath.Join(first, "memory-ranges")); err != nil || string(got) != "golden memory" { + t.Fatalf("staged %q, %v", got, err) + } + if n := store.gets.Load(); n != 1 { + t.Errorf("cross-mount fallback performed %d downloads, want 1 (the copy must reuse the published entry)", n) + } + + // And the fallback stays cache-served: a second cross-mount restore is a + // local copy, not a download. + second := filepath.Join(otherFS, "second") + if err := os.Mkdir(second, 0o700); err != nil { + t.Fatal(err) + } + if err := s.downloadExternalCheckpoint(ctx, testActor, testSnapshotURI, second, files, cacheModeLink); err != nil { + t.Fatal(err) + } + if n := store.gets.Load(); n != 1 { + t.Errorf("second cross-mount restore downloaded again (gets=%d, want 1)", n) + } + // The misconfiguration is the node's, not the restore's: one warning per + // process, not one per file per restore. + if n := strings.Count(logs.String(), "serving every hit as a copy"); n != 1 { + t.Errorf("cross-mount warning logged %d times over two restores, want once", n) + } +} diff --git a/cmd/atelet/snapshotcache_test.go b/cmd/atelet/snapshotcache_test.go new file mode 100644 index 0000000000..109315c78d --- /dev/null +++ b/cmd/atelet/snapshotcache_test.go @@ -0,0 +1,464 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "go.ate.dev/substrate/cmd/atelet/internal/filecache" + "go.ate.dev/substrate/internal/proto/ateletpb" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/objectstorage" +) + +func TestValidateSnapshotCacheFlags(t *testing.T) { + origDir, origMinAge, origPeriod, origFetch := *snapshotCacheDir, *snapshotCacheMinAge, *snapshotCacheGCFlags.Period, *snapshotCacheFetchTimeout + t.Cleanup(func() { + *snapshotCacheDir, *snapshotCacheMinAge, *snapshotCacheGCFlags.Period, *snapshotCacheFetchTimeout = origDir, origMinAge, origPeriod, origFetch + }) + + cases := []struct { + name string + dir string + minAge time.Duration + wantErr bool + }{ + {"defaults", origDir, 10 * time.Minute, false}, + {"disabled", "", 10 * time.Minute, false}, + {"disabled still rejects a bad min-age", "", -time.Second, true}, + {"zero min-age", origDir, 0, false}, + {"outside base path: the plugin could not fill it", "/var/lib/elsewhere/snapshot-cache", 10 * time.Minute, true}, + {"negative min-age inverts the veto", origDir, -time.Second, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + *snapshotCacheDir, *snapshotCacheMinAge = tc.dir, tc.minAge + err := validateSnapshotCacheFlags() + if (err != nil) != tc.wantErr { + t.Errorf("dir=%q minAge=%v: err=%v, wantErr=%v", tc.dir, tc.minAge, err, tc.wantErr) + } + }) + } + // The eviction flags are validated here too, enabled or not. + for _, dir := range []string{origDir, ""} { + *snapshotCacheDir, *snapshotCacheMinAge, *snapshotCacheGCFlags.Period = dir, 10*time.Minute, -time.Second + if err := validateSnapshotCacheFlags(); err == nil { + t.Errorf("validateSnapshotCacheFlags() = nil with a negative GC period and dir %q, want an error", dir) + } + } + // So is the fetch timeout: a zero or negative bound would end every + // download at once. + *snapshotCacheGCFlags.Period = origPeriod + for _, timeout := range []time.Duration{0, -time.Second} { + *snapshotCacheFetchTimeout = timeout + if err := validateSnapshotCacheFlags(); err == nil { + t.Errorf("validateSnapshotCacheFlags() = nil with fetch timeout %v, want an error", timeout) + } + } +} + +func TestOpenSnapshotCacheDisabled(t *testing.T) { + store, err := openSnapshotCache(context.Background(), "", 10*time.Minute, time.Minute) + if err != nil { + t.Fatalf("openSnapshotCache(\"\") error: %v", err) + } + if store != nil { + t.Errorf("openSnapshotCache(\"\") = %v, want nil (disabled)", store) + } +} + +func TestOpenSnapshotCacheCreatesRoot(t *testing.T) { + dir := filepath.Join(t.TempDir(), "nested", "snapshot-cache") + store, err := openSnapshotCache(context.Background(), dir, 10*time.Minute, time.Minute) + if err != nil { + t.Fatalf("openSnapshotCache(%q) error: %v", dir, err) + } + if store == nil { + t.Fatal("openSnapshotCache returned a nil store for a non-empty dir") + } + if fi, err := os.Stat(dir); err != nil || !fi.IsDir() { + t.Errorf("cache root %s not created: %v", dir, err) + } +} + +// TestOpenSnapshotCacheSweepsDebris pins the startup ordering contract: crash +// debris (unfinished fetches in tmp/, interrupted evictions as .rm-*) is gone +// by the time openSnapshotCache returns, while published entries survive. +func TestOpenSnapshotCacheSweepsDebris(t *testing.T) { + dir := t.TempDir() + // A prior life's layout: one published entry, plus debris of both kinds. + entryDir := filepath.Join(dir, "entries", "aaaa") + for _, d := range []string{ + filepath.Join(dir, "tmp", "aaaa-12345"), + filepath.Join(dir, ".rm-bbbb-xyz"), + entryDir, + } { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(d, "data"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + + store, err := openSnapshotCache(context.Background(), dir, 10*time.Minute, time.Minute) + if err != nil { + t.Fatalf("openSnapshotCache error: %v", err) + } + if store == nil { + t.Fatal("openSnapshotCache returned a nil store for a non-empty dir") + } + + if children, err := os.ReadDir(filepath.Join(dir, "tmp")); err != nil || len(children) != 0 { + t.Errorf("tmp debris not swept: %d children, %v", len(children), err) + } + if _, err := os.Stat(filepath.Join(dir, ".rm-bbbb-xyz")); !os.IsNotExist(err) { + t.Errorf("retired debris not swept: %v", err) + } + if _, err := os.Stat(filepath.Join(entryDir, "data")); err != nil { + t.Errorf("published entry did not survive the sweep: %v", err) + } +} + +// TestSnapshotCacheMode pins the caching gate: only a snapshot the control +// plane declared shared is served from the cache. Private, unstated, and +// absent sources all download fresh; atelet never infers cacheability from +// the URI. +func TestSnapshotCacheMode(t *testing.T) { + withSharing := func(sh ateletpb.SnapshotSharing) *ateletpb.ObjectStorage { + return &ateletpb.ObjectStorage{SnapshotUri: testSnapshotURI, Sharing: sh} + } + cases := []struct { + name string + object *ateletpb.ObjectStorage + want cacheMode + }{ + {"shared uses the class mode", withSharing(ateletpb.SnapshotSharing_SNAPSHOT_SHARING_SHARED), cacheModeLink}, + {"private is never cached", withSharing(ateletpb.SnapshotSharing_SNAPSHOT_SHARING_PRIVATE), cacheModeOff}, + {"unstated is never cached", withSharing(ateletpb.SnapshotSharing_SNAPSHOT_SHARING_UNSPECIFIED), cacheModeOff}, + {"no object", nil, cacheModeOff}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := snapshotCacheMode(tc.object, cacheModeLink); got != tc.want { + t.Errorf("snapshotCacheMode() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestCacheModeFor(t *testing.T) { + if got := cacheModeFor(string(atev1alpha1.SandboxClassGvisor)); got != cacheModeLink { + t.Errorf("gVisor consumes restore state read-only; want cacheModeLink, got %v", got) + } + // ateworker-microvm keeps memory-ranges open for the VM's life, merges + // deltas into it, and drops it from the page cache; through a link each + // of those would reach the cache's entry, so it gets private copies. + if got := cacheModeFor(string(atev1alpha1.SandboxClassMicroVM)); got != cacheModeCopy { + t.Errorf("micro-VM keeps and reworks its restore files; want cacheModeCopy, got %v", got) + } + if got := cacheModeFor(""); got != cacheModeOff { + t.Errorf("unknown sandbox class: want cacheModeOff, got %v", got) + } +} + +// countingObjectStorage counts reads, so a test can pin how many downloads +// a sequence of restores performed. +type countingObjectStorage struct { + objectstorage.ObjectStorage + gets atomic.Int32 +} + +func (c *countingObjectStorage) GetObject(ctx context.Context, bucket, object string) (io.ReadCloser, error) { + c.gets.Add(1) + return c.ObjectStorage.GetObject(ctx, bucket, object) +} + +// newSnapshotCacheHerder returns an atelet whose plugin serves a snapshot +// holding the named files with the given contents, with a snapshot cache +// open under base, plus base itself, which is on the cache's filesystem, +// for restore directories. +func newSnapshotCacheHerder(t *testing.T, files map[string]string) (*Atelet, *countingObjectStorage, string) { + t.Helper() + base := t.TempDir() + store := &countingObjectStorage{ObjectStorage: &recordingObjectStorage{}} + for name, content := range files { + payload := filepath.Join(base, "payload-"+filepath.Base(name)) + if err := os.WriteFile(payload, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + if err := objectstorage.SendLocalFileToGCSWithZstd(context.Background(), store.ObjectStorage, testSnapshotURI+"/"+name+".zstd", payload); err != nil { + t.Fatal(err) + } + } + s := newPluginHerder(t, store) + cache, err := filecache.New(filepath.Join(base, "cache")) + if err != nil { + t.Fatal(err) + } + s.snapshotCache = cache + return s, store, base +} + +// restoreDir makes a restore directory under base, as Restore does before +// it downloads. +func restoreDir(t *testing.T, base, name string) string { + t.Helper() + dir := filepath.Join(base, name) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + return dir +} + +// TestFetchManifestCached pins that a shared snapshot's manifest is read +// through the cache, one download for any number of restores, with the +// scratch link dropped again, while a private one is read fresh each time. +func TestFetchManifestCached(t *testing.T) { + const manifest = `{"sandboxClass":"gvisor"}` + s, store, _ := newSnapshotCacheHerder(t, nil) + ctx := context.Background() + // The plugin stores the manifest as is, not zstd-compressed like the + // data files the herder's helper uploads. + if err := store.ObjectStorage.PutObject(ctx, "bucket", strings.TrimPrefix(testSnapshotPath, "bucket/")+"/"+sandboxManifestName, strings.NewReader(manifest)); err != nil { + t.Fatal(err) + } + + for i := range 2 { + got, err := s.fetchManifestCached(ctx, testActor, testSnapshotURI, cacheModeLink) + if err != nil { + t.Fatalf("fetchManifestCached #%d: %v", i+1, err) + } + if string(got) != manifest { + t.Fatalf("manifest #%d = %q, want %q", i+1, got, manifest) + } + } + if n := store.gets.Load(); n != 1 { + t.Errorf("two cached manifest reads downloaded %d times, want 1", n) + } + if left, err := os.ReadDir(s.snapshotScratchDir); err != nil || len(left) != 0 { + t.Errorf("scratch dir holds %v (%v) after cached manifest reads, want nothing", left, err) + } + + // Not shared: read from object storage, as before the cache existed. + if _, err := s.fetchManifestCached(ctx, testActor, testSnapshotURI, cacheModeOff); err != nil { + t.Fatal(err) + } + if n := store.gets.Load(); n != 2 { + t.Errorf("a private manifest read downloaded %d times in total, want 2", n) + } +} + +func TestDownloadExternalCheckpointServesFromCache(t *testing.T) { + s, store, base := newSnapshotCacheHerder(t, map[string]string{"memory-ranges": "golden memory"}) + ctx := context.Background() + files := []string{"memory-ranges"} + + // Two cached restores: one download, both destinations share the inode. + first, second := restoreDir(t, base, "first"), restoreDir(t, base, "second") + for _, dir := range []string{first, second} { + if err := s.downloadExternalCheckpoint(ctx, testActor, testSnapshotURI, dir, files, cacheModeLink); err != nil { + t.Fatalf("downloadExternalCheckpoint into %s: %v", dir, err) + } + if got, err := os.ReadFile(filepath.Join(dir, "memory-ranges")); err != nil || string(got) != "golden memory" { + t.Fatalf("staged in %s = %q, %v", dir, got, err) + } + } + if n := store.gets.Load(); n != 1 { + t.Errorf("two cached restores performed %d downloads, want 1", n) + } + fi1, err1 := os.Stat(filepath.Join(first, "memory-ranges")) + fi2, err2 := os.Stat(filepath.Join(second, "memory-ranges")) + if err1 != nil || err2 != nil || !os.SameFile(fi1, fi2) { + t.Errorf("cached destinations do not share an inode (%v, %v)", err1, err2) + } + if fi1.Mode().Perm()&0o222 != 0 { + t.Errorf("a linked restore file is writable (%v); the shared bytes must be read-only", fi1.Mode()) + } + + // A non-cacheable restore downloads fresh and shares nothing. + direct := restoreDir(t, base, "direct") + if err := s.downloadExternalCheckpoint(ctx, testActor, testSnapshotURI, direct, files, cacheModeOff); err != nil { + t.Fatalf("direct downloadExternalCheckpoint: %v", err) + } + if n := store.gets.Load(); n != 2 { + t.Errorf("direct restore did not download (gets=%d, want 2)", n) + } + if fi3, err := os.Stat(filepath.Join(direct, "memory-ranges")); err != nil || os.SameFile(fi1, fi3) { + t.Errorf("direct destination shares the cache inode (%v)", err) + } +} + +// TestDownloadExternalCheckpointCopyMode pins the micro-VM serving mode: one +// download per snapshot file, every destination a private writable inode, +// and a consumer's in-place mutation never reaches the next consumer. +func TestDownloadExternalCheckpointCopyMode(t *testing.T) { + s, store, base := newSnapshotCacheHerder(t, map[string]string{"memory-ranges": "golden memory"}) + ctx := context.Background() + files := []string{"memory-ranges"} + + first, second := restoreDir(t, base, "first"), restoreDir(t, base, "second") + for _, dir := range []string{first, second} { + if err := s.downloadExternalCheckpoint(ctx, testActor, testSnapshotURI, dir, files, cacheModeCopy); err != nil { + t.Fatalf("downloadExternalCheckpoint into %s: %v", dir, err) + } + } + if n := store.gets.Load(); n != 1 { + t.Errorf("two copy-mode restores performed %d downloads, want 1", n) + } + fi1, err1 := os.Stat(filepath.Join(first, "memory-ranges")) + fi2, err2 := os.Stat(filepath.Join(second, "memory-ranges")) + if err1 != nil || err2 != nil { + t.Fatal(err1, err2) + } + if os.SameFile(fi1, fi2) { + t.Error("copy-mode destinations share an inode; each consumer must own its own") + } + + // The consumer may rewrite its staged file (ateworker-microvm does, to + // config.json) without poisoning what the next restore receives. + if err := os.WriteFile(filepath.Join(first, "memory-ranges"), []byte("rewritten by the worker"), 0o600); err != nil { + t.Fatalf("consumer write to its copy: %v", err) + } + third := restoreDir(t, base, "third") + if err := s.downloadExternalCheckpoint(ctx, testActor, testSnapshotURI, third, files, cacheModeCopy); err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(filepath.Join(third, "memory-ranges")); err != nil || string(got) != "golden memory" { + t.Errorf("copy after consumer mutation = %q, %v; the cached bytes were poisoned", got, err) + } + if n := store.gets.Load(); n != 1 { + t.Errorf("gets=%d, want 1 (mutation must not invalidate the cache)", n) + } +} + +func TestDownloadExternalCheckpointWithoutCacheDownloads(t *testing.T) { + s, store, base := newSnapshotCacheHerder(t, map[string]string{"memory-ranges": "golden memory"}) + s.snapshotCache = nil // caching disabled (--snapshot-cache-dir="") + + dir := restoreDir(t, base, "out") + if err := s.downloadExternalCheckpoint(context.Background(), testActor, testSnapshotURI, dir, []string{"memory-ranges"}, cacheModeLink); err != nil { + t.Fatalf("downloadExternalCheckpoint with nil cache: %v", err) + } + if got, err := os.ReadFile(filepath.Join(dir, "memory-ranges")); err != nil || string(got) != "golden memory" { + t.Fatalf("staged content = %q, %v", got, err) + } + if n := store.gets.Load(); n != 1 { + t.Errorf("gets=%d, want 1", n) + } +} + +func TestDownloadExternalCheckpointCachedFetchErrorReachesCaller(t *testing.T) { + s, _, base := newSnapshotCacheHerder(t, map[string]string{"memory-ranges": "golden memory"}) + dir := restoreDir(t, base, "out") + err := s.downloadExternalCheckpoint(context.Background(), testActor, testSnapshotURI, dir, []string{"missing"}, cacheModeLink) + if err == nil { + t.Fatal("downloadExternalCheckpoint succeeded for a missing file") + } + if _, statErr := os.Stat(filepath.Join(dir, "missing")); !os.IsNotExist(statErr) { + t.Errorf("failed fetch left a destination file: %v", statErr) + } +} + +// TestDownloadExternalCheckpointSharesOneDownload pins the exit criterion at +// the unit level: concurrent restores staging the same shared snapshot +// perform its downloads once. +func TestDownloadExternalCheckpointSharesOneDownload(t *testing.T) { + s, store, base := newSnapshotCacheHerder(t, map[string]string{"memory-ranges": "golden memory"}) + files := []string{"memory-ranges"} + + const restores = 4 + errs := make([]error, restores) + var wg sync.WaitGroup + for i := range restores { + dir := restoreDir(t, base, fmt.Sprintf("actor-%d", i)) + wg.Go(func() { + errs[i] = s.downloadExternalCheckpoint(context.Background(), testActor, testSnapshotURI, dir, files, cacheModeLink) + }) + } + wg.Wait() + for i, err := range errs { + if err != nil { + t.Fatalf("restore %d: %v", i, err) + } + got, err := os.ReadFile(filepath.Join(base, fmt.Sprintf("actor-%d", i), "memory-ranges")) + if err != nil || string(got) != "golden memory" { + t.Fatalf("restore %d staged %q, %v", i, got, err) + } + } + if n := store.gets.Load(); n != 1 { + t.Errorf("%d concurrent restores performed %d downloads, want 1", restores, n) + } +} + +// TestDownloadExternalCheckpointCachedSubdirAndSymlinks covers the two ways +// the cached path differs from the plugin writing through an os.Root: a +// file in the checkpoint subdirectory is placed there, and a symlink waiting +// in the restore directory, as a directory or as the file itself, fails the +// restore instead of being followed outside it. +func TestDownloadExternalCheckpointCachedSubdirAndSymlinks(t *testing.T) { + s, _, base := newSnapshotCacheHerder(t, map[string]string{"memory-ranges": "golden memory", "fs/data.tar": "fs delta", "fs/rootfs-upper-app.tar": "rootfs delta"}) + ctx := context.Background() + + // Two files share the subdirectory and are fetched in parallel: the + // directory is made once, before either, not raced into by both. + dir := restoreDir(t, base, "subdir") + if err := s.downloadExternalCheckpoint(ctx, testActor, testSnapshotURI, dir, []string{"memory-ranges", "fs/data.tar", "fs/rootfs-upper-app.tar"}, cacheModeLink); err != nil { + t.Fatalf("downloadExternalCheckpoint with subdirectory files: %v", err) + } + for name, want := range map[string]string{"fs/data.tar": "fs delta", "fs/rootfs-upper-app.tar": "rootfs delta"} { + if got, err := os.ReadFile(filepath.Join(dir, name)); err != nil || string(got) != want { + t.Fatalf("subdirectory file %s = %q, %v", name, got, err) + } + } + + outside := filepath.Join(base, "outside") + if err := os.MkdirAll(outside, 0o700); err != nil { + t.Fatal(err) + } + keep := filepath.Join(outside, "memory-ranges") + if err := os.WriteFile(keep, []byte("keep me"), 0o600); err != nil { + t.Fatal(err) + } + fileLink := restoreDir(t, base, "file-link") + if err := os.Symlink(keep, filepath.Join(fileLink, "memory-ranges")); err != nil { + t.Fatal(err) + } + if err := s.downloadExternalCheckpoint(ctx, testActor, testSnapshotURI, fileLink, []string{"memory-ranges"}, cacheModeLink); err == nil { + t.Error("a symlink in place of the restore file was followed") + } + dirLink := restoreDir(t, base, "dir-link") + if err := os.Symlink(outside, filepath.Join(dirLink, "fs")); err != nil { + t.Fatal(err) + } + if err := s.downloadExternalCheckpoint(ctx, testActor, testSnapshotURI, dirLink, []string{"fs/data.tar"}, cacheModeCopy); err == nil { + t.Error("a symlink in place of the checkpoint subdirectory was followed") + } + if got, err := os.ReadFile(keep); err != nil || string(got) != "keep me" { + t.Errorf("file outside the restore directory = %q, %v; want untouched", got, err) + } + if entries, _ := os.ReadDir(outside); len(entries) != 1 { + t.Errorf("the directory outside the restore directory holds %d entries, want only the original file", len(entries)) + } +} diff --git a/cmd/atelet/snapshotreport_test.go b/cmd/atelet/snapshotreport_test.go index b74c290a92..4285864dc6 100644 --- a/cmd/atelet/snapshotreport_test.go +++ b/cmd/atelet/snapshotreport_test.go @@ -170,9 +170,10 @@ func nextReport(t *testing.T, clk *clocktesting.FakeClock, workers *fakeReportSe return got } -// TestBackgroundUploadAndDetachAreReported pauses an actor and leaves it -// paused: atelet reports the background upload of its snapshot, then its -// background detach, to the control plane, for the actor its Checkpoint named. +// TestBackgroundUploadAndDetachAreReported checkpoints an actor locally and +// leaves it suspended: atelet reports the background upload of its snapshot, +// then its background detach, to the control plane, for the actor its +// Checkpoint named. func TestBackgroundUploadAndDetachAreReported(t *testing.T) { a := runPluginActor(t) clk := clocktesting.NewFakeClock(time.Now()) diff --git a/cmd/atelet/volumes_test.go b/cmd/atelet/volumes_test.go index 4679f9d716..079e27debf 100644 --- a/cmd/atelet/volumes_test.go +++ b/cmd/atelet/volumes_test.go @@ -25,7 +25,6 @@ import ( "github.com/google/go-cmp/cmp" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" - "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/proto/ateletpb" "go.ate.dev/substrate/internal/volume" "go.ate.dev/substrate/pkg/api/v1alpha1" @@ -74,17 +73,6 @@ func (f *fakeWorkerPlugin) UnmountVolume(ctx context.Context, volumeID string, t var _ volume.VolumePluginWorkerPlane = (*fakeWorkerPlugin)(nil) -// withTempActorsDir redirects nodepath.ActorsDir at a temp dir for the -// duration of the test. Every path derived from ActorsDir moves with it, -// including the ones resetActorDirs and the bundle preparation compute -// independently of the volume mount code. -func withTempActorsDir(t *testing.T) { - t.Helper() - orig := nodepath.ActorsDir - nodepath.ActorsDir = filepath.Join(t.TempDir(), "actors") - t.Cleanup(func() { nodepath.ActorsDir = orig }) -} - func TestUnmountExternalVolumes(t *testing.T) { ctx := context.Background() actorUID := "test-actor-123" @@ -191,7 +179,7 @@ func TestMountExternalVolumes(t *testing.T) { } t.Run("success mounts external volumes and creates mount directory", func(t *testing.T) { - withTempActorsDir(t) + useTempNodeDirs(t) fake := &fakeWorkerPlugin{} s := &Atelet{ @@ -235,7 +223,7 @@ func TestMountExternalVolumes(t *testing.T) { }) t.Run("target directory already exists is handled cleanly", func(t *testing.T) { - withTempActorsDir(t) + useTempNodeDirs(t) expectedPath := ateletpath.ExternalVolumeHostPath(actorUID, "vol-1") if err := os.MkdirAll(expectedPath, 0o750); err != nil { @@ -259,7 +247,7 @@ func TestMountExternalVolumes(t *testing.T) { }) t.Run("plugin lookup failure returns error", func(t *testing.T) { - withTempActorsDir(t) + useTempNodeDirs(t) unknownVol := &ateletpb.Volume{ Name: "vol-unknown", @@ -283,7 +271,7 @@ func TestMountExternalVolumes(t *testing.T) { }) t.Run("plugin mount failure returns error", func(t *testing.T) { - withTempActorsDir(t) + useTempNodeDirs(t) fake := &fakeWorkerPlugin{ mountErr: errors.New("mount operation failed: device or resource busy"), @@ -304,7 +292,7 @@ func TestMountExternalVolumes(t *testing.T) { }) t.Run("multi-volume partial failure aborts on first failure", func(t *testing.T) { - withTempActorsDir(t) + useTempNodeDirs(t) fake := &fakeWorkerPlugin{ mountErr: errors.New("cannot mount volume"), @@ -325,7 +313,7 @@ func TestMountExternalVolumes(t *testing.T) { }) t.Run("multi-volume partial failure does not roll back already mounted volume", func(t *testing.T) { - withTempActorsDir(t) + useTempNodeDirs(t) expectedPath1 := ateletpath.ExternalVolumeHostPath(actorUID, "vol-1") expectedPath2 := ateletpath.ExternalVolumeHostPath(actorUID, "vol-2") @@ -394,7 +382,7 @@ func TestVolumeHostDirectoryCleanup(t *testing.T) { } t.Run("unmountExternalVolumes preserves host mount directories", func(t *testing.T) { - withTempActorsDir(t) + useTempNodeDirs(t) fake := &fakeWorkerPlugin{} s := &Atelet{ @@ -428,7 +416,7 @@ func TestVolumeHostDirectoryCleanup(t *testing.T) { }) t.Run("resetActorDirs removes empty unmounted volume host directories", func(t *testing.T) { - withTempActorsDir(t) + useTempNodeDirs(t) fake := &fakeWorkerPlugin{} s := &Atelet{ @@ -476,7 +464,7 @@ func TestVolumeHostDirectoryCleanup(t *testing.T) { }) t.Run("resetActorDirs preserves non-empty volume directories and returns error", func(t *testing.T) { - withTempActorsDir(t) + useTempNodeDirs(t) fake := &fakeWorkerPlugin{} s := &Atelet{ @@ -511,7 +499,7 @@ func TestVolumeHostDirectoryCleanup(t *testing.T) { }) t.Run("resetActorDirs succeeds cleanly when volumes directory does not exist", func(t *testing.T) { - withTempActorsDir(t) + useTempNodeDirs(t) if err := resetActorDirs(actorUID); err != nil { t.Fatalf("resetActorDirs failed when volumes dir does not exist: %v", err) diff --git a/cmd/atelet/worker_actor_jwt.go b/cmd/atelet/worker_actor_jwt.go new file mode 100644 index 0000000000..00b89c49c9 --- /dev/null +++ b/cmd/atelet/worker_actor_jwt.go @@ -0,0 +1,105 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/utils/lru" + + "go.ate.dev/substrate/internal/objectstoreplugin/exchange" + "go.ate.dev/substrate/pkg/proto/ateapipb" +) + +// actorRef identifies the actor a transfer is for, as atelet's RPCs carry it. +type actorRef struct { + atespace, name, uid string +} + +const ( + // workerActorJWTRefreshMargin is the least remaining life a cached JWT is used + // with. The JWT only has to be valid when the node plugin exchanges it, + // not for the duration of a transfer. + workerActorJWTRefreshMargin = 5 * time.Minute + // workerActorJWTCacheSize bounds the per-actor cache. A miss costs one mint, + // never correctness. + workerActorJWTCacheSize = 10_000 +) + +type cachedWorkerActorJWT struct { + jwt string + expiry time.Time +} + +// workerActorJWTs mints worker-for-actor JWTs through ate-api-server and caches +// them per actor UID. The node plugin trades one for storage credentials +// scoped to the actor, so atelet sends one with every snapshot transfer. +type workerActorJWTs struct { + workers ateapipb.WorkerServiceClient + now func() time.Time + cache *lru.Cache +} + +func newWorkerActorJWTs(workers ateapipb.WorkerServiceClient) *workerActorJWTs { + return &workerActorJWTs{workers: workers, now: time.Now, cache: lru.New(workerActorJWTCacheSize)} +} + +// get returns a JWT for the actor. The cache serves it while more than +// workerActorJWTRefreshMargin remains, otherwise ate-api-server mints a new +// one. A mint failure comes back as a gRPC status with the code a plugin +// failure would carry, so the transfer helpers handle both the same way: +// - Unavailable: ate-api-server could not be reached. Kept, so the callers +// that retry a transfer do. +// - FailedPrecondition: the actor is not this node's to transfer for, which +// the mint reports as NotFound or Aborted. Renamed because callers read +// NotFound from a transfer as a missing snapshot object. +// - Internal: anything else, including an empty JWT in the response. +func (j *workerActorJWTs) get(ctx context.Context, actor actorRef) (string, error) { + if v, ok := j.cache.Get(actor.uid); ok { + if c := v.(cachedWorkerActorJWT); c.expiry.Sub(j.now()) > workerActorJWTRefreshMargin { + return c.jwt, nil + } + } + resp, err := j.workers.MintWorkerActorJwt(ctx, &ateapipb.MintWorkerActorJwtRequest{ + Actor: &ateapipb.ObjectRef{Atespace: actor.atespace, Name: actor.name}, + ActorUid: actor.uid, + Audiences: []string{exchange.ActorJWTAudience}, + }) + switch status.Code(err) { + case codes.OK: + case codes.Unavailable: + return "", status.Errorf(codes.Unavailable, "while minting the actor JWT for %s/%s: %s", actor.atespace, actor.name, status.Convert(err).Message()) + case codes.NotFound, codes.Aborted: + return "", status.Errorf(codes.FailedPrecondition, "actor %s/%s cannot be transferred from this node: %s", actor.atespace, actor.name, status.Convert(err).Message()) + default: + return "", status.Errorf(codes.Internal, "while minting the actor JWT for %s/%s: %s", actor.atespace, actor.name, status.Convert(err).Message()) + } + if resp.GetActorJwt() == "" { + return "", status.Errorf(codes.Internal, "ate-api-server minted an empty actor JWT for %s/%s", actor.atespace, actor.name) + } + j.cache.Add(actor.uid, cachedWorkerActorJWT{jwt: resp.GetActorJwt(), expiry: resp.GetExpireTime().AsTime()}) + return resp.GetActorJwt(), nil +} + +// workerActorJWT is the JWT a transfer for actor carries. +func (s *Atelet) workerActorJWT(ctx context.Context, actor actorRef) (string, error) { + if s.workerActorJWTs == nil { + return "", status.Error(codes.Internal, "actor JWT source is not configured") + } + return s.workerActorJWTs.get(ctx, actor) +} diff --git a/cmd/atelet/worker_actor_jwt_test.go b/cmd/atelet/worker_actor_jwt_test.go new file mode 100644 index 0000000000..c0af8d883e --- /dev/null +++ b/cmd/atelet/worker_actor_jwt_test.go @@ -0,0 +1,179 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "sync" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" + + "go.ate.dev/substrate/internal/objectstoreplugin/exchange" + "go.ate.dev/substrate/pkg/proto/ateapipb" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" +) + +// testActor is the actor the transfer tests act for. +var testActor = actorRef{atespace: "ate-demo", name: "bot", uid: "9f1e0b2a-9e4f-4c3d-8a6b-5f2e1d0c9b8a"} + +// fakeJWTMinter answers MintWorkerActorJwt with a fixed token and records +// what it was asked for. A restore fetches its files in parallel, and each +// fetch mints while the cache is cold, so the record is locked. +type fakeJWTMinter struct { + ateapipb.WorkerServiceClient + jwt string + expiry time.Time + err error + + mu sync.Mutex + got []*ateapipb.MintWorkerActorJwtRequest +} + +func (m *fakeJWTMinter) MintWorkerActorJwt(_ context.Context, in *ateapipb.MintWorkerActorJwtRequest, _ ...grpc.CallOption) (*ateapipb.MintWorkerActorJwtResponse, error) { + m.mu.Lock() + m.got = append(m.got, in) + m.mu.Unlock() + if m.err != nil { + return nil, m.err + } + return &ateapipb.MintWorkerActorJwtResponse{ActorJwt: m.jwt, ExpireTime: timestamppb.New(m.expiry)}, nil +} + +// testWorkerActorJWTs is the JWT source transfer tests give an Atelet. +func testWorkerActorJWTs() *workerActorJWTs { + return newWorkerActorJWTs(&fakeJWTMinter{jwt: "test-actor-jwt", expiry: time.Now().Add(time.Hour)}) +} + +func TestWorkerActorJWTs_MintsOncePerActorUntilNearExpiry(t *testing.T) { + now := time.Date(2026, 10, 16, 10, 0, 0, 0, time.UTC) + clock := now + minter := &fakeJWTMinter{jwt: "jwt-1", expiry: now.Add(time.Hour)} + j := newWorkerActorJWTs(minter) + j.now = func() time.Time { return clock } + + for range 2 { + jwt, err := j.get(t.Context(), testActor) + if err != nil || jwt != "jwt-1" { + t.Fatalf("get = %q, %v", jwt, err) + } + } + if len(minter.got) != 1 { + t.Fatalf("minted %d times, want 1", len(minter.got)) + } + req := minter.got[0] + if req.GetActor().GetAtespace() != testActor.atespace || req.GetActor().GetName() != testActor.name || req.GetActorUid() != testActor.uid { + t.Errorf("mint request names %v uid=%s", req.GetActor(), req.GetActorUid()) + } + if len(req.GetAudiences()) != 1 || req.GetAudiences()[0] != exchange.ActorJWTAudience { + t.Errorf("mint audiences = %v, want the exchange's", req.GetAudiences()) + } + + other := testActor + other.uid = "another-uid" + if _, err := j.get(t.Context(), other); err != nil { + t.Fatal(err) + } + if len(minter.got) != 2 { + t.Fatalf("a different actor should mint, minted %d times", len(minter.got)) + } + + // Under the refresh margin the cached JWT is not used. + clock = now.Add(time.Hour - workerActorJWTRefreshMargin + time.Second) + minter.jwt = "jwt-2" + jwt, err := j.get(t.Context(), testActor) + if err != nil || jwt != "jwt-2" { + t.Fatalf("get near expiry = %q, %v, want a fresh mint", jwt, err) + } +} + +func TestWorkerActorJWTs_Errors(t *testing.T) { + for _, tc := range []struct { + name string + minter *fakeJWTMinter + want codes.Code + }{ + // NotFound and Aborted must not reach callers as NotFound, which they + // read from a transfer as a missing snapshot object. + {"not this node's actor", &fakeJWTMinter{err: status.Error(codes.NotFound, "actor not found")}, codes.FailedPrecondition}, + {"recreated actor", &fakeJWTMinter{err: status.Error(codes.Aborted, "uid changed")}, codes.FailedPrecondition}, + {"ate-api-server down", &fakeJWTMinter{err: status.Error(codes.Unavailable, "connection refused")}, codes.Unavailable}, + {"ate-api-server refuses atelet", &fakeJWTMinter{err: status.Error(codes.Unauthenticated, "bad cert")}, codes.Internal}, + {"empty token", &fakeJWTMinter{jwt: "", expiry: time.Now().Add(time.Hour)}, codes.Internal}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := newWorkerActorJWTs(tc.minter).get(t.Context(), testActor) + if status.Code(err) != tc.want { + t.Fatalf("code = %v (%v), want %v", status.Code(err), err, tc.want) + } + }) + } + t.Run("no source configured", func(t *testing.T) { + _, err := (&Atelet{}).workerActorJWT(t.Context(), testActor) + if status.Code(err) != codes.Internal { + t.Fatalf("code = %v, want Internal", status.Code(err)) + } + }) +} + +// recordingNodePlugin captures the transfer requests atelet sends. +type recordingNodePlugin struct { + objectstorev1.NodeProviderClient + fetches []*objectstorev1.FetchSnapshotRequest + uploads []*objectstorev1.UploadSnapshotRequest +} + +func (p *recordingNodePlugin) FetchSnapshot(_ context.Context, in *objectstorev1.FetchSnapshotRequest, _ ...grpc.CallOption) (*objectstorev1.FetchSnapshotResponse, error) { + p.fetches = append(p.fetches, in) + return &objectstorev1.FetchSnapshotResponse{}, nil +} + +func (p *recordingNodePlugin) UploadSnapshot(_ context.Context, in *objectstorev1.UploadSnapshotRequest, _ ...grpc.CallOption) (*objectstorev1.UploadSnapshotResponse, error) { + p.uploads = append(p.uploads, in) + return &objectstorev1.UploadSnapshotResponse{}, nil +} + +func TestTransfersCarryTheActorJWT(t *testing.T) { + plugin := &recordingNodePlugin{} + s := &Atelet{objectStorePlugin: plugin, workerActorJWTs: testWorkerActorJWTs()} + if err := s.fetchSnapshotFiles(t.Context(), testActor, pausedSnapshotURI, t.TempDir(), []string{"a"}); err != nil { + t.Fatal(err) + } + if err := s.uploadSnapshotFiles(t.Context(), testActor, pausedSnapshotURI, t.TempDir(), []string{"a"}); err != nil { + t.Fatal(err) + } + if len(plugin.fetches) != 1 || plugin.fetches[0].GetActorJwt() != "test-actor-jwt" { + t.Errorf("fetch carried actor_jwt %q", plugin.fetches[0].GetActorJwt()) + } + if len(plugin.uploads) != 1 || plugin.uploads[0].GetActorJwt() != "test-actor-jwt" { + t.Errorf("upload carried actor_jwt %q", plugin.uploads[0].GetActorJwt()) + } +} + +func TestTransfersStopWhenTheMintFails(t *testing.T) { + plugin := &recordingNodePlugin{} + s := &Atelet{objectStorePlugin: plugin, workerActorJWTs: newWorkerActorJWTs(&fakeJWTMinter{err: status.Error(codes.NotFound, "actor not found")})} + err := s.fetchSnapshotFiles(t.Context(), testActor, pausedSnapshotURI, t.TempDir(), []string{"a"}) + if status.Code(err) != codes.FailedPrecondition { + t.Fatalf("err = %v, want FailedPrecondition for an actor this node cannot transfer for", err) + } + if len(plugin.fetches) != 0 { + t.Fatal("the plugin was called without a JWT") + } +} diff --git a/cmd/atelet/worker_sandbox_syncer.go b/cmd/atelet/worker_sandbox_syncer.go index ac2b7582f8..81ff49866f 100644 --- a/cmd/atelet/worker_sandbox_syncer.go +++ b/cmd/atelet/worker_sandbox_syncer.go @@ -369,10 +369,7 @@ func (s *workerSandboxSyncer) registerAhead(pool *v1alpha1.WorkerPool) { if err != nil { return } - hashed, err := hashVersions(context.Background(), plan) - if err != nil { - return - } + hashed := hashVersions(plan) if err := s.index.registerRuntimes(plan.config, hashed); err != nil { // The syncs write it again, and report it if it keeps failing. slog.Warn("Could not index a changed pool's runtimes ahead of its sync", slog.String("pool", pool.Name), slog.Any("err", err)) diff --git a/cmd/atelet/worker_sandbox_syncer_test.go b/cmd/atelet/worker_sandbox_syncer_test.go index 38cb0a5b3b..d139fd804b 100644 --- a/cmd/atelet/worker_sandbox_syncer_test.go +++ b/cmd/atelet/worker_sandbox_syncer_test.go @@ -394,11 +394,7 @@ func TestWorkerSandboxSyncerRegistersAheadOfJitter(t *testing.T) { if err != nil { t.Fatal(err) } - hashed, err := hashVersions(context.Background(), plan) - if err != nil { - t.Fatal(err) - } - for _, v := range hashed { + for _, v := range hashVersions(plan) { if _, ok := s.index.lookup(v.hash); !ok { t.Errorf("runtime %s (%s) does not resolve before the jittered sync", v.name, v.hash) } @@ -424,7 +420,7 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { deleting := workerPod("pod-a", "pool") deleting.DeletionTimestamp = &metav1.Time{Time: time.Now()} stale := wantVersions() - stale.DefaultRuntime = wantRuntime("v0") + stale.SandboxClasses[0].DefaultRuntime = sandboxruntimes.Runtime{Version: wantRuntime("v0")} tests := []struct { name string @@ -535,7 +531,7 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { }, }, priorRetries: syncMaxRetries, - want: &sandboxruntimes.File{DefaultRuntime: wantRuntime("v2")}, + want: gvisorFile(wantRuntime("v2")), wantDir: true, wantRetries: syncMaxRetries + 1, }, { diff --git a/cmd/atelet/worker_sandbox_versions.go b/cmd/atelet/worker_sandbox_versions.go index d964c9c4de..cb5e468a34 100644 --- a/cmd/atelet/worker_sandbox_versions.go +++ b/cmd/atelet/worker_sandbox_versions.go @@ -18,10 +18,10 @@ package main // in the steps syncPod runs: // - resolve: find the SandboxConfig of the pod's WorkerPool // - plan: pick the config's versions this node can run -// - build: hash each version's assets and pause image digests into its -// sandbox_bundle_hash, and probe what a snapshot taken on the runtime -// must find again: the host CPU identity for microVM, the runsc -// binary's CPU features for gVisor +// - build: hash each version's asset digests into its sandbox_bundle_hash, +// and probe what a snapshot taken on the runtime must find again: the +// host CPU identity for microVM, the runsc binary's CPU features for +// gVisor // - write: replace the file in the worker directory if its content changed // // worker_sandbox_syncer.go decides when each pod is synced, and creates and @@ -42,7 +42,6 @@ import ( "strings" "time" - "github.com/google/go-containerregistry/pkg/name" corev1 "k8s.io/api/core/v1" "go.ate.dev/substrate/internal/hardware" @@ -55,8 +54,8 @@ import ( // errPermanent marks a failure that retrying cannot fix, so the syncer drops // the item instead of backing off until the next event for the pod: // - the pod has no worker pool label -// - the config's default version is missing, has no assets for this -// node's architecture, or has a pause image not pinned by digest +// - the config's default version is missing or has no assets for this +// node's architecture // - the default version's runsc runs but fails, prints no features, or // prints one this atelet does not know; the binary is content-addressed, // so running it again gives the same answer @@ -197,7 +196,6 @@ func recordForArch(cfg *v1alpha1.SandboxConfig, ver *v1alpha1.SandboxVersionConf } rec := &sandboxAssetsRecord{ SandboxClass: string(cfg.Spec.SandboxClass), - PauseImage: ver.PauseImage, Assets: make(map[string]assetEntry, len(files)), } for name, f := range files { @@ -213,24 +211,13 @@ type hashedVersion struct { } // hashVersions hashes each version of plan. The hashes come from the -// SandboxConfig spec alone, so it does no IO. A version that cannot be -// hashed is left out, unless it is the default, without which no worker of -// the pool could start actors on this node. -func hashVersions(ctx context.Context, plan *sandboxPlan) ([]hashedVersion, error) { +// SandboxConfig spec alone, so it does no IO. +func hashVersions(plan *sandboxPlan) []hashedVersion { hashed := make([]hashedVersion, 0, len(plan.versions)) for _, v := range plan.versions { - h, err := versionBundleHash(v.rec) - if err != nil { - err = fmt.Errorf("SandboxConfig %q, version %q: %w", plan.config, v.name, err) - if v.isDefault { - return nil, fmt.Errorf("%w: %w", errPermanent, err) - } - slog.WarnContext(ctx, "Leaving a sandbox version out of the worker sandbox runtimes file", slog.Any("err", err)) - continue - } - hashed = append(hashed, hashedVersion{versionPlan: v, hash: h}) + hashed = append(hashed, hashedVersion{versionPlan: v, hash: bundleHash(v.rec)}) } - return hashed, nil + return hashed } // buildFile returns the file content naming the runtimes of the hashed @@ -248,12 +235,12 @@ func hashVersions(ctx context.Context, plan *sandboxPlan) ([]hashedVersion, erro // fetched and probed, serially. // // Listing a runtime does not promise its assets are on the node: probing -// fetches a gVisor version's runsc the first time this atelet sees it, but -// not its pause image, and nothing of a microVM version. The syncer has -// the default runtime prewarmed once the file is written, and Run/Restore -// fetch any other version on demand. +// fetches a gVisor version's runsc the first time this atelet sees it, and +// nothing of a microVM version. The syncer has the default runtime prewarmed +// once the file is written, and Run/Restore fetch any other version on +// demand. func (s *workerSandboxSyncer) buildFile(ctx context.Context, plan *sandboxPlan, hashed []hashedVersion) (*sandboxruntimes.File, error) { - out := &sandboxruntimes.File{} + class := sandboxruntimes.Class{SandboxClass: string(plan.sandboxClass)} seen := make(map[string]bool, len(hashed)) var incomplete error for _, v := range hashed { @@ -278,11 +265,13 @@ func (s *workerSandboxSyncer) buildFile(ctx context.Context, plan *sandboxPlan, } seen[v.hash] = true if v.isDefault { - out.DefaultRuntime = attrs + class.DefaultRuntime = sandboxruntimes.Runtime{Version: attrs} } else { - out.RestorableRuntimes = append(out.RestorableRuntimes, attrs) + class.RestorableRuntimes = append(class.RestorableRuntimes, sandboxruntimes.Runtime{Version: attrs}) } } + // One class per pool today. + out := &sandboxruntimes.File{SandboxClasses: []sandboxruntimes.Class{class}} if incomplete != nil { return out, fmt.Errorf("%w: %w", errIncomplete, incomplete) } @@ -424,34 +413,15 @@ func runRunscCPUFeatures(ctx context.Context, runscPath string) ([]string, error return strings.Split(printed, ","), nil } -// versionBundleHash returns the bundle hash of one version. The CRD requires -// every pause image to be pinned by digest, which is the digest the image -// cache resolves the ref to, so the hash needs no pull. An unpinned one is -// errPermanent: only a spec change fixes it. -func versionBundleHash(rec *sandboxAssetsRecord) (string, error) { - var pauseDigest string - if rec.PauseImage != "" { - ref, err := name.NewDigest(rec.PauseImage) - if err != nil { - return "", fmt.Errorf("%w: pause image %q is not pinned by digest: %w", errPermanent, rec.PauseImage, err) - } - pauseDigest = ref.DigestStr() - } - return bundleHash(rec, pauseDigest), nil -} - // bundleHash is the sandbox_bundle_hash of a version: a sha256 over one -// sorted "key=value" line per asset digest and pause image digest. Version -// names and URLs are left out, so the same binaries under another name or in -// another bucket are the same runtime. -func bundleHash(rec *sandboxAssetsRecord, pauseDigest string) string { - lines := make([]string, 0, len(rec.Assets)+1) +// sorted "asset/=" line per asset. Version names and URLs are +// left out, so the same binaries under another name or in another bucket are +// the same runtime. +func bundleHash(rec *sandboxAssetsRecord) string { + lines := make([]string, 0, len(rec.Assets)) for name, a := range rec.Assets { lines = append(lines, "asset/"+name+"="+a.SHA256) } - if pauseDigest != "" { - lines = append(lines, "pause_image="+pauseDigest) - } slices.Sort(lines) sum := sha256.Sum256([]byte(strings.Join(lines, "\n"))) return "sha256:" + hex.EncodeToString(sum[:]) @@ -475,10 +445,7 @@ func (s *workerSandboxSyncer) syncPod(ctx context.Context, pod *corev1.Pod) erro if err != nil { return err } - hashed, err := hashVersions(ctx, plan) - if err != nil { - return err - } + hashed := hashVersions(plan) // Resolvable, and on disk, before the file can let the worker register // any of them. if err := s.index.registerRuntimes(plan.config, hashed); err != nil { @@ -501,7 +468,7 @@ func (s *workerSandboxSyncer) syncPod(ctx context.Context, pod *corev1.Pod) erro // Only now, with the file naming its defaults: a sync that failed above // left the file, and so the directories, as they were. For a gVisor // default the build fetched the release for its probe, so the prewarm - // is links and the pause image; a micro-VM default downloads here. + // is links; a micro-VM default downloads here. if s.prewarm != nil { var defaults []string for _, v := range writtenVersions(hashed, want) { @@ -522,10 +489,12 @@ func (s *workerSandboxSyncer) syncPod(ctx context.Context, pod *corev1.Pod) erro // writtenVersions returns the versions of hashed whose runtime file names, // in hashed's order: every version whose attributes were built. func writtenVersions(hashed []hashedVersion, file *sandboxruntimes.File) []hashedVersion { - written := make(map[string]bool, 1+len(file.RestorableRuntimes)) - written[file.DefaultRuntime[hardware.AttrSandboxBundleHash]] = true - for _, attrs := range file.RestorableRuntimes { - written[attrs[hardware.AttrSandboxBundleHash]] = true + written := make(map[string]bool, len(hashed)) + for _, c := range file.SandboxClasses { + written[c.DefaultRuntime.Version[hardware.AttrSandboxBundleHash]] = true + for _, rt := range c.RestorableRuntimes { + written[rt.Version[hardware.AttrSandboxBundleHash]] = true + } } out := make([]hashedVersion, 0, len(hashed)) for _, v := range hashed { @@ -545,11 +514,14 @@ func (s *workerSandboxSyncer) write(ctx context.Context, uid, config string, wan return err } if wrote { - slog.InfoContext(ctx, "Wrote worker sandbox runtimes", - slog.String("pod_uid", uid), - slog.String("sandbox_config", config), - slog.String("default_bundle_hash", want.DefaultRuntime[hardware.AttrSandboxBundleHash]), - slog.Int("restorable_runtimes", len(want.RestorableRuntimes))) + for _, c := range want.SandboxClasses { + slog.InfoContext(ctx, "Wrote worker sandbox runtimes", + slog.String("pod_uid", uid), + slog.String("sandbox_config", config), + slog.String("sandbox_class", c.SandboxClass), + slog.String("default_bundle_hash", c.DefaultRuntime.Version[hardware.AttrSandboxBundleHash]), + slog.Int("restorable_runtimes", len(c.RestorableRuntimes))) + } } return nil } diff --git a/cmd/atelet/worker_sandbox_versions_test.go b/cmd/atelet/worker_sandbox_versions_test.go index 7941cc1915..abcbf9f3ed 100644 --- a/cmd/atelet/worker_sandbox_versions_test.go +++ b/cmd/atelet/worker_sandbox_versions_test.go @@ -37,21 +37,14 @@ import ( "go.ate.dev/substrate/pkg/api/v1alpha1" ) -// pauseDigest is the digest of the pause image of multiVersionConfig's -// version with the given name. -func pauseDigest(name string) string { - return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte("pause-"+name))) -} - // multiVersionConfig is a gVisor config whose default version "v2" is listed // after an Enabled "v1", a Disabled "v0", and an Enabled "other-arch" with no // assets for this node. func multiVersionConfig() *v1alpha1.SandboxConfig { version := func(name, sha string, state *v1alpha1.SandboxVersionState) v1alpha1.SandboxVersionConfig { return v1alpha1.SandboxVersionConfig{ - Name: name, - State: state, - PauseImage: "registry.k8s.io/pause@" + pauseDigest(name), + Name: name, + State: state, Assets: map[string]map[string]v1alpha1.AssetFile{ runtime.GOARCH: {gvisorAssetName: {URL: "gs://bucket/" + name, SHA256: sha}}, }, @@ -85,7 +78,6 @@ func TestRecordForArch(t *testing.T) { assets: map[string]map[string]v1alpha1.AssetFile{runtime.GOARCH: {gvisorAssetName: {URL: "gs://bucket/gvisor", SHA256: sha}}}, want: &sandboxAssetsRecord{ SandboxClass: string(v1alpha1.SandboxClassGvisor), - PauseImage: "registry.k8s.io/pause@sha256:abc", Assets: map[string]assetEntry{gvisorAssetName: {URL: "gs://bucket/gvisor", SHA256: sha}}, }, }, { @@ -109,35 +101,26 @@ func TestRecordForArch(t *testing.T) { func TestBundleHash(t *testing.T) { gvisor := map[string]assetEntry{gvisorAssetName: {URL: "gs://bucket/gvisor", SHA256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f"}} - pause := "sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" // Snapshots record the hash, so a change to any of these values strands // every snapshot taken before it. tests := []struct { - name string - rec *sandboxAssetsRecord - pauseDigest string - want string + name string + rec *sandboxAssetsRecord + want string }{{ - name: "gvisor asset and pause image", - rec: &sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: "registry.k8s.io/pause:3.10.2@" + pause, Assets: gvisor}, - pauseDigest: pause, - want: "sha256:7197896bb429f0b824accbc635a38c9ed5666c160070b833bbdc6e036ccd4f76", + name: "gvisor asset", + rec: &sandboxAssetsRecord{SandboxClass: "gvisor", Assets: gvisor}, + want: "sha256:f86e6f2bd8e25ac31a22e51e6e57e0eb6741eb0b0df77082989cc1e40f906fe1", }, { // Only the digests count, not the version's other fields. - name: "another URL and pause image ref", - rec: &sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: "mirror.example.com/pause@" + pause, Assets: map[string]assetEntry{gvisorAssetName: {URL: "gs://other/gvisor", SHA256: gvisor[gvisorAssetName].SHA256}}}, - pauseDigest: pause, - want: "sha256:7197896bb429f0b824accbc635a38c9ed5666c160070b833bbdc6e036ccd4f76", - }, { - name: "no pause image", - rec: &sandboxAssetsRecord{SandboxClass: "microvm", Assets: gvisor}, + name: "another URL and class", + rec: &sandboxAssetsRecord{SandboxClass: "microvm", Assets: map[string]assetEntry{gvisorAssetName: {URL: "gs://other/gvisor", SHA256: gvisor[gvisorAssetName].SHA256}}}, want: "sha256:f86e6f2bd8e25ac31a22e51e6e57e0eb6741eb0b0df77082989cc1e40f906fe1", }, { - name: "the asset under another name", - rec: &sandboxAssetsRecord{SandboxClass: "gvisor", Assets: map[string]assetEntry{"renamed": gvisor[gvisorAssetName]}}, - pauseDigest: pause, - want: "sha256:04bf18376ca0c7227ae9fa65675127b1045d7de33f05ea5d2375eb11a1a8e62e", + name: "the asset under another name", + rec: &sandboxAssetsRecord{SandboxClass: "gvisor", Assets: map[string]assetEntry{"renamed": gvisor[gvisorAssetName]}}, + want: "sha256:510948cc43e084528a7c76c1627c5bfb2d140be0b620f33282c03fc2debd65af", }, { name: "several assets", rec: &sandboxAssetsRecord{SandboxClass: "microvm", Assets: map[string]assetEntry{ @@ -148,7 +131,7 @@ func TestBundleHash(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := bundleHash(tt.rec, tt.pauseDigest); got != tt.want { + if got := bundleHash(tt.rec); got != tt.want { t.Errorf("bundleHash() = %q, want %q", got, tt.want) } }) @@ -245,48 +228,6 @@ func TestWorkerSandboxResolverResolve(t *testing.T) { } } -func TestVersionBundleHash(t *testing.T) { - gvisor := map[string]assetEntry{gvisorAssetName: {URL: "gs://bucket/gvisor", SHA256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f"}} - pause := "sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" - - tests := []struct { - name string - pauseImage string - want string - wantErr bool - }{{ - name: "pause image pinned by digest", - pauseImage: "registry.k8s.io/pause:3.10.2@" + pause, - want: bundleHash(&sandboxAssetsRecord{Assets: gvisor}, pause), - }, { - name: "pause image pinned by digest only", - pauseImage: "gcr.io/gke-release/pause@" + pause, - want: bundleHash(&sandboxAssetsRecord{Assets: gvisor}, pause), - }, { - name: "no pause image", - want: bundleHash(&sandboxAssetsRecord{Assets: gvisor}, ""), - }, { - // The CRD rejects this shape. - name: "pause image by tag", - pauseImage: "registry.k8s.io/pause:3.10.2", - wantErr: true, - }} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := versionBundleHash(&sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: tt.pauseImage, Assets: gvisor}) - if (err != nil) != tt.wantErr { - t.Fatalf("versionBundleHash() error = %v, want error %v", err, tt.wantErr) - } - if err != nil && !errors.Is(err, errPermanent) { - t.Errorf("versionBundleHash() error = %v, want errPermanent", err) - } - if got != tt.want { - t.Errorf("versionBundleHash() = %q, want %q", got, tt.want) - } - }) - } -} - // wantRuntime is the runtime buildFile builds from multiVersionConfig's // version with the given name when runsc reports testCPUFeatures() ("3"). func wantRuntime(name string) sandboxruntimes.Attributes { @@ -298,16 +239,28 @@ func wantRuntimeWithFeatures(name, cpuFeatures string) sandboxruntimes.Attribute return sandboxruntimes.Attributes{ hardware.AttrArchitecture: runtime.GOARCH, hardware.AttrCPUFeatures: cpuFeatures, - hardware.AttrSandboxBundleHash: bundleHash(rec, pauseDigest(name)), + hardware.AttrSandboxBundleHash: bundleHash(rec), + } +} + +// classFile is the file content of a worker in a pool of the given class: +// one entry, with the given default and restorable runtimes. +func classFile(class v1alpha1.SandboxClass, def sandboxruntimes.Attributes, restorable ...sandboxruntimes.Attributes) *sandboxruntimes.File { + c := sandboxruntimes.Class{SandboxClass: string(class), DefaultRuntime: sandboxruntimes.Runtime{Version: def}} + for _, r := range restorable { + c.RestorableRuntimes = append(c.RestorableRuntimes, sandboxruntimes.Runtime{Version: r}) } + return &sandboxruntimes.File{SandboxClasses: []sandboxruntimes.Class{c}} +} + +// gvisorFile is classFile for a gVisor pool. +func gvisorFile(def sandboxruntimes.Attributes, restorable ...sandboxruntimes.Attributes) *sandboxruntimes.File { + return classFile(v1alpha1.SandboxClassGvisor, def, restorable...) } // wantVersions is the file content for multiVersionConfig. func wantVersions() *sandboxruntimes.File { - return &sandboxruntimes.File{ - DefaultRuntime: wantRuntime("v2"), - RestorableRuntimes: []sandboxruntimes.Attributes{wantRuntime("v1")}, - } + return gvisorFile(wantRuntime("v2"), wantRuntime("v1")) } // planAndBuild runs the plan and build steps of syncPod for pod. @@ -316,22 +269,17 @@ func planAndBuild(s *workerSandboxSyncer, pod *corev1.Pod) (*sandboxruntimes.Fil if err != nil { return nil, err } - hashed, err := hashVersions(context.Background(), plan) - if err != nil { - return nil, err - } - return s.buildFile(context.Background(), plan, hashed) + return s.buildFile(context.Background(), plan, hashVersions(plan)) } func TestPlanAndBuild(t *testing.T) { defaultNoArch := multiVersionConfig() defaultNoArch.Spec.Versions[3].Assets = map[string]map[string]v1alpha1.AssetFile{"other-arch": defaultNoArch.Spec.Versions[3].Assets[runtime.GOARCH]} - onlyDefault := &sandboxruntimes.File{DefaultRuntime: wantRuntime("v2")} + onlyDefault := gvisorFile(wantRuntime("v2")) // sameAsDefault has v1 run the default version's binaries. sameAsDefault := multiVersionConfig() - sameAsDefault.Spec.Versions[1].PauseImage = sameAsDefault.Spec.Versions[3].PauseImage sameAsDefault.Spec.Versions[1].Assets = sameAsDefault.Spec.Versions[3].Assets // v1Twice adds an Enabled "v3" running v1's binaries. @@ -340,13 +288,6 @@ func TestPlanAndBuild(t *testing.T) { v3.Name = "v3" v1Twice.Spec.Versions = append(v1Twice.Spec.Versions, v3) - // The CRD rejects a pause image not pinned by digest. - tagged := func(i int) *v1alpha1.SandboxConfig { - cfg := multiVersionConfig() - cfg.Spec.Versions[i].PauseImage = "registry.k8s.io/pause:3.10.2" - return cfg - } - unsupportedClass := multiVersionConfig() unsupportedClass.Spec.SandboxClass = "unknown" @@ -381,16 +322,6 @@ func TestPlanAndBuild(t *testing.T) { cfg: defaultNoArch, wantErr: true, wantPermanent: true, - }, { - // No worker of the pool could start actors on this node. - name: "default version pause image not pinned by digest", - cfg: tagged(3), - wantErr: true, - wantPermanent: true, - }, { - name: "non-default version pause image not pinned by digest is left out", - cfg: tagged(1), - want: onlyDefault, }, { name: "unsupported sandbox class", cfg: unsupportedClass, @@ -454,7 +385,7 @@ func TestPlanAndBuildMicroVM(t *testing.T) { hardware.AttrArchitecture: runtime.GOARCH, hardware.AttrCPUVendor: testCPUVendor, hardware.AttrCPUModel: testCPUModel, - hardware.AttrSandboxBundleHash: bundleHash(rec, ""), + hardware.AttrSandboxBundleHash: bundleHash(rec), } } @@ -483,10 +414,7 @@ func TestPlanAndBuildMicroVM(t *testing.T) { // Once a probe succeeds, every later sync reuses it. probeErr = nil - want := &sandboxruntimes.File{ - DefaultRuntime: wantMicroVM("sha-v2"), - RestorableRuntimes: []sandboxruntimes.Attributes{wantMicroVM("sha-v1")}, - } + want := classFile(v1alpha1.SandboxClassMicroVM, wantMicroVM("sha-v2"), wantMicroVM("sha-v1")) for _, uid := range []string{"pod-a", "pod-b"} { got, err := planAndBuild(s, workerPod(uid, "microvm-pool")) if err != nil { @@ -534,11 +462,11 @@ func TestGVisorCPUFeaturesMemoizationAndErrors(t *testing.T) { t.Run("memoizes runsc cpu-features by asset digest across versions and syncs", func(t *testing.T) { cfg := multiVersionConfig() // Add a third version "v3" that shares v1's gvisor asset digest ("sha-v1") - // but has its own pause image digest, so it has a distinct bundle hash - // while reusing the same runsc binary. + // but has one more asset, so it has a distinct bundle hash while it + // uses the same runsc binary. v3 := *cfg.Spec.Versions[1].DeepCopy() v3.Name = "v3" - v3.PauseImage = "registry.k8s.io/pause@" + pauseDigest("v3") + v3.Assets[runtime.GOARCH]["extra"] = v1alpha1.AssetFile{URL: "gs://bucket/extra", SHA256: "sha-extra"} cfg.Spec.Versions = append(cfg.Spec.Versions, v3) s := newTestSyncer(t, resolvedPool("pool", "gvisor-release"), cfg) @@ -560,18 +488,16 @@ func TestGVisorCPUFeaturesMemoizationAndErrors(t *testing.T) { if err != nil { t.Fatalf("first planAndBuild() error = %v", err) } - v3Rec := &sandboxAssetsRecord{Assets: map[string]assetEntry{gvisorAssetName: {SHA256: "sha-v1"}}} - want := &sandboxruntimes.File{ - DefaultRuntime: wantRuntimeWithFeatures("v2", "3"), - RestorableRuntimes: []sandboxruntimes.Attributes{ - wantRuntimeWithFeatures("v1", "1"), - { - hardware.AttrArchitecture: runtime.GOARCH, - hardware.AttrCPUFeatures: "1", - hardware.AttrSandboxBundleHash: bundleHash(v3Rec, pauseDigest("v3")), - }, + v3Rec := &sandboxAssetsRecord{Assets: map[string]assetEntry{gvisorAssetName: {SHA256: "sha-v1"}, "extra": {SHA256: "sha-extra"}}} + want := gvisorFile( + wantRuntimeWithFeatures("v2", "3"), + wantRuntimeWithFeatures("v1", "1"), + sandboxruntimes.Attributes{ + hardware.AttrArchitecture: runtime.GOARCH, + hardware.AttrCPUFeatures: "1", + hardware.AttrSandboxBundleHash: bundleHash(v3Rec), }, - } + ) if diff := cmp.Diff(want, got); diff != "" { t.Errorf("planAndBuild() mismatch (-want +got):\n%s", diff) } @@ -630,7 +556,7 @@ func TestGVisorCPUFeaturesMemoizationAndErrors(t *testing.T) { if !errors.Is(err, errIncomplete) || errors.Is(err, errPermanent) { t.Errorf("planAndBuild() on a restorable version's fetch error = %v, want errIncomplete and not errPermanent", err) } - want := &sandboxruntimes.File{DefaultRuntime: wantRuntime("v2")} + want := gvisorFile(wantRuntime("v2")) if diff := cmp.Diff(want, got); diff != "" { t.Errorf("planAndBuild() partial file mismatch (-want +got):\n%s", diff) } @@ -661,7 +587,7 @@ func TestGVisorCPUFeaturesMemoizationAndErrors(t *testing.T) { if err != nil { t.Fatalf("planAndBuild() with unknown feature on non-default error = %v", err) } - want := &sandboxruntimes.File{DefaultRuntime: wantRuntime("v2")} + want := gvisorFile(wantRuntime("v2")) if diff := cmp.Diff(want, got); diff != "" { t.Errorf("planAndBuild() mismatch (-want +got):\n%s", diff) } diff --git a/cmd/atelet/workersupport.go b/cmd/atelet/workersupport.go index be9f3c32de..85cdc2e724 100644 --- a/cmd/atelet/workersupport.go +++ b/cmd/atelet/workersupport.go @@ -125,18 +125,16 @@ func (s *workerSupportServer) RegisterWorker(ctx context.Context, req *ateletpb. } if _, err := s.workers.RegisterWorker(ctx, &ateapipb.RegisterWorkerRequest{ // Workers are global-scoped and named by their pod UID. - Worker: &ateapipb.ObjectRef{Name: workerIdentity.PodUID}, - Capacity: toWorkerResources(req.GetCapacity()), - DefaultRuntime: toAteAPISandboxRuntime(req.GetDefaultRuntime()), - RestorableRuntimes: toAteAPISandboxRuntimes(req.GetRestorableRuntimes()), + Worker: &ateapipb.ObjectRef{Name: workerIdentity.PodUID}, + Capacity: toWorkerResources(req.GetCapacity()), + SandboxClasses: toAteAPISandboxClasses(req.GetSandboxClasses()), }); err != nil { return nil, err } - slog.InfoContext(ctx, "Registered worker capacity and sandbox runtimes", + slog.InfoContext(ctx, "Registered worker capacity and sandbox classes", slog.String("pod_uid", workerIdentity.PodUID), slog.Any("capacity", req.GetCapacity()), - slog.Any("default_runtime", req.GetDefaultRuntime()), - slog.Any("restorable_runtimes", req.GetRestorableRuntimes())) + slog.Any("sandbox_classes", req.GetSandboxClasses())) return &ateletpb.RegisterWorkerResponse{}, nil } @@ -153,29 +151,52 @@ func toWorkerResources(in *ateletpb.WorkerResources) *ateapipb.WorkerResources { } } -// toAteAPISandboxRuntime converts atelet's SandboxRuntime to the control +// toAteAPISandboxClasses converts atelet's WorkerSandboxClass list to the +// control plane's, which it mirrors field for field. +func toAteAPISandboxClasses(in []*ateletpb.WorkerSandboxClass) []*ateapipb.WorkerSandboxClass { + if in == nil { + return nil + } + out := make([]*ateapipb.WorkerSandboxClass, 0, len(in)) + for _, c := range in { + out = append(out, &ateapipb.WorkerSandboxClass{ + SandboxClass: c.GetSandboxClass(), + DefaultRuntime: toAteAPISandboxRuntime(c.GetDefaultRuntime()), + RestorableRuntimes: toAteAPISandboxRuntimes(c.GetRestorableRuntimes()), + }) + } + return out +} + +// toAteAPISandboxRuntime converts atelet's WorkerSandboxRuntime to the control // plane's, which it mirrors field for field. -func toAteAPISandboxRuntime(in *ateletpb.SandboxRuntime) *ateapipb.SandboxRuntime { +func toAteAPISandboxRuntime(in *ateletpb.WorkerSandboxRuntime) *ateapipb.WorkerSandboxRuntime { if in == nil { return nil } - out := &ateapipb.SandboxRuntime{SandboxClass: in.GetSandboxClass()} - if v := in.GetVersion(); v != nil { - out.Version = &ateapipb.VersionedSandboxCompat{SchemaVersion: v.GetSchemaVersion()} - for _, a := range v.GetAttributes() { - out.Version.Attributes = append(out.Version.Attributes, &ateapipb.AttributeEntry{Key: a.GetKey(), Value: a.GetValue()}) - } + return &ateapipb.WorkerSandboxRuntime{Version: toAteAPICompat(in.GetVersion())} +} + +func toAteAPISandboxRuntimes(in []*ateletpb.WorkerSandboxRuntime) []*ateapipb.WorkerSandboxRuntime { + if in == nil { + return nil + } + out := make([]*ateapipb.WorkerSandboxRuntime, 0, len(in)) + for _, rt := range in { + out = append(out, toAteAPISandboxRuntime(rt)) } return out } -func toAteAPISandboxRuntimes(in []*ateletpb.SandboxRuntime) []*ateapipb.SandboxRuntime { +// toAteAPICompat converts atelet's VersionedSandboxCompat to the control +// plane's, which it mirrors field for field. +func toAteAPICompat(in *ateletpb.VersionedSandboxCompat) *ateapipb.VersionedSandboxCompat { if in == nil { return nil } - out := make([]*ateapipb.SandboxRuntime, 0, len(in)) - for _, r := range in { - out = append(out, toAteAPISandboxRuntime(r)) + out := &ateapipb.VersionedSandboxCompat{SchemaVersion: in.GetSchemaVersion()} + for _, a := range in.GetAttributes() { + out.Attributes = append(out.Attributes, &ateapipb.AttributeEntry{Key: a.GetKey(), Value: a.GetValue()}) } return out } @@ -186,8 +207,8 @@ func toAteAPISandboxRuntimes(in []*ateletpb.SandboxRuntime) []*ateapipb.SandboxR // // The control plane's error is returned unwrapped: a refusal is a real answer // here -- the actor is no longer assigned to this worker, or the suspend lost a -// race to a resume, pause, or delete -- and the worker needs to tell those from -// a transport failure it should retry. +// race to a resume, suspend, or delete -- and the worker needs to tell those +// from a transport failure it should retry. func (s *workerSupportServer) RequestActorSuspend(ctx context.Context, req *ateletpb.RequestActorSuspendRequest) (*ateletpb.RequestActorSuspendResponse, error) { // Identity comes only from the mTLS certificate, never from the request: a // worker can speak for the actors it hosts and no others. Which those are diff --git a/cmd/atelet/workersupport_test.go b/cmd/atelet/workersupport_test.go index 974b9e9d14..f649c9e899 100644 --- a/cmd/atelet/workersupport_test.go +++ b/cmd/atelet/workersupport_test.go @@ -114,40 +114,52 @@ func (s *fakeWorkerService) MintWorkerActorCertificate(_ context.Context, in *at } func TestRegisterWorker(t *testing.T) { - reqRuntime := func(hash string) *ateletpb.SandboxRuntime { - return &ateletpb.SandboxRuntime{ - SandboxClass: "gvisor", - Version: &ateletpb.VersionedSandboxCompat{ - SchemaVersion: "v1", - Attributes: []*ateletpb.AttributeEntry{ - {Key: "architecture", Value: "amd64"}, - {Key: "sandbox_bundle_hash", Value: hash}, - }, + reqCompat := func(hash string) *ateletpb.VersionedSandboxCompat { + return &ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "sandbox_bundle_hash", Value: hash}, }, } } - wantRuntime := func(hash string) *ateapipb.SandboxRuntime { - return &ateapipb.SandboxRuntime{ - SandboxClass: "gvisor", - Version: &ateapipb.VersionedSandboxCompat{ - SchemaVersion: "v1", - Attributes: []*ateapipb.AttributeEntry{ - {Key: "architecture", Value: "amd64"}, - {Key: "sandbox_bundle_hash", Value: hash}, - }, + wantCompat := func(hash string) *ateapipb.VersionedSandboxCompat { + return &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "sandbox_bundle_hash", Value: hash}, }, } } - reqDefault, wantDefault := reqRuntime("9988"), wantRuntime("9988") - forwarded := func(capacity *ateapipb.WorkerResources, restorable ...*ateapipb.SandboxRuntime) []*ateapipb.RegisterWorkerRequest { + reqDefault, wantDefault := reqCompat("9988"), wantCompat("9988") + // reqClasses is the worker's one class, gvisor, with the default and + // the given restorable runtimes. + reqClasses := func(restorable ...*ateletpb.VersionedSandboxCompat) []*ateletpb.WorkerSandboxClass { + c := &ateletpb.WorkerSandboxClass{ + SandboxClass: "gvisor", + DefaultRuntime: &ateletpb.WorkerSandboxRuntime{Version: reqDefault}, + } + for _, r := range restorable { + c.RestorableRuntimes = append(c.RestorableRuntimes, &ateletpb.WorkerSandboxRuntime{Version: r}) + } + return []*ateletpb.WorkerSandboxClass{c} + } + forwarded := func(capacity *ateapipb.WorkerResources, restorable ...*ateapipb.VersionedSandboxCompat) []*ateapipb.RegisterWorkerRequest { // The Worker is named after the worker pod UID, taken from the - // certificate rather than the request; capacity and runtimes come - // from what the worker reported. + // certificate rather than the request; capacity and sandbox classes + // come from what the worker reported. + c := &ateapipb.WorkerSandboxClass{ + SandboxClass: "gvisor", + DefaultRuntime: &ateapipb.WorkerSandboxRuntime{Version: wantDefault}, + } + for _, r := range restorable { + c.RestorableRuntimes = append(c.RestorableRuntimes, &ateapipb.WorkerSandboxRuntime{Version: r}) + } return []*ateapipb.RegisterWorkerRequest{{ - Worker: &ateapipb.ObjectRef{Name: "pod-a"}, - Capacity: capacity, - DefaultRuntime: wantDefault, - RestorableRuntimes: restorable, + Worker: &ateapipb.ObjectRef{Name: "pod-a"}, + Capacity: capacity, + SandboxClasses: []*ateapipb.WorkerSandboxClass{c}, }} } @@ -164,45 +176,63 @@ func TestRegisterWorker(t *testing.T) { name: "records what the worker says", req: &ateletpb.RegisterWorkerRequest{ Capacity: &ateletpb.WorkerResources{Actors: 4, CpuMillicores: 2000, MemoryBytes: 4 << 30}, - DefaultRuntime: reqDefault, + SandboxClasses: reqClasses(), }, wantForwarded: forwarded(&ateapipb.WorkerResources{Actors: 4, CpuMillicores: 2000, MemoryBytes: 4294967296}), }, { name: "forwards restorable runtimes", req: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{Actors: 1}, - DefaultRuntime: reqDefault, - RestorableRuntimes: []*ateletpb.SandboxRuntime{reqRuntime("d547")}, + Capacity: &ateletpb.WorkerResources{Actors: 1}, + SandboxClasses: reqClasses(reqCompat("d547")), }, - wantForwarded: forwarded(&ateapipb.WorkerResources{Actors: 1}, wantRuntime("d547")), + wantForwarded: forwarded(&ateapipb.WorkerResources{Actors: 1}, wantCompat("d547")), }, { name: "omits undetermined compute", - req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, DefaultRuntime: reqDefault}, + req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, SandboxClasses: reqClasses()}, wantForwarded: forwarded(&ateapipb.WorkerResources{Actors: 1}), }, { name: "forwards empty capacity", - req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{}, DefaultRuntime: reqDefault}, + req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{}, SandboxClasses: reqClasses()}, wantForwarded: forwarded(&ateapipb.WorkerResources{}), }, { name: "rejects invalid capacity without forwarding", req: &ateletpb.RegisterWorkerRequest{ Capacity: &ateletpb.WorkerResources{CpuMillicores: -1}, - DefaultRuntime: reqDefault, + SandboxClasses: reqClasses(), }, wantCode: codes.InvalidArgument, }, { name: "rejects missing capacity without forwarding", - req: &ateletpb.RegisterWorkerRequest{DefaultRuntime: reqDefault}, + req: &ateletpb.RegisterWorkerRequest{SandboxClasses: reqClasses()}, wantCode: codes.InvalidArgument, }, { - name: "rejects missing default runtime without forwarding", + name: "rejects missing sandbox classes without forwarding", req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}}, wantCode: codes.InvalidArgument, + }, { + name: "rejects missing default runtime without forwarding", + req: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{Actors: 1}, + SandboxClasses: []*ateletpb.WorkerSandboxClass{{SandboxClass: "gvisor"}}, + }, + wantCode: codes.InvalidArgument, + }, { + // A worker runs one class until WorkerPoolSpec.sandboxClasses + // allows more. + name: "rejects two sandbox classes without forwarding", + req: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{Actors: 1}, + SandboxClasses: []*ateletpb.WorkerSandboxClass{ + {SandboxClass: "gvisor", DefaultRuntime: &ateletpb.WorkerSandboxRuntime{Version: reqDefault}}, + {SandboxClass: "microvm", DefaultRuntime: &ateletpb.WorkerSandboxRuntime{Version: reqDefault}}, + }, + }, + wantCode: codes.InvalidArgument, }, { // A worker may report only what its certificate proves it is. name: "requires a certificate", unauthenticated: true, - req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, DefaultRuntime: reqDefault}, + req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, SandboxClasses: reqClasses()}, wantCode: codes.Unauthenticated, }, { // The Worker record may not exist yet. The error must reach the @@ -210,7 +240,7 @@ func TestRegisterWorker(t *testing.T) { // leaves the Worker with no capacity forever. name: "surfaces the control plane's rejection", serviceErr: errors.New("no such worker"), - req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, DefaultRuntime: reqDefault}, + req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, SandboxClasses: reqClasses()}, wantCode: codes.Unknown, }} for _, tt := range tests { diff --git a/cmd/atenet/README.md b/cmd/atenet/README.md index d72b0f6a71..3abe9e1a0e 100644 --- a/cmd/atenet/README.md +++ b/cmd/atenet/README.md @@ -18,8 +18,8 @@ likely be split in the future for better scalability.) * Envoy * atenet router * Service will expose: - * Envoy port 80 and 443 for ordinary ingress traffic, plus a dedicated - CONNECT port (8081, or 8444 for TLS) for arbitrary-port ingress: a client + * Envoy port 443 for ordinary ingress traffic, plus a dedicated + CONNECT port (8444) for arbitrary-port ingress: a client reaches a port on the actor other than its default (80) with an HTTP `CONNECT` request naming the port in the authority. Only HTTP(S) traffic over the tunnel is supported today, not raw TCP. diff --git a/cmd/atenet/internal/router/README.md b/cmd/atenet/internal/router/README.md index 77be87f89b..2d7076bfa7 100644 --- a/cmd/atenet/internal/router/README.md +++ b/cmd/atenet/internal/router/README.md @@ -25,22 +25,22 @@ Router has several responsibilities: dataplane container's `preStop` hook. See `drain.go` and `envoydrain.go`. * Authenticates actor identity on egress: on every CONNECT, the egress gateway's ext_proc handler re-verifies the actor's client certificate against - the actor-identity CA, reads the `ActorIdentity` X.509 extension out of it, - and checks the certified UID against the ATE API. + the actor-identity CA, reads the actor from its SPIFFE URI SAN, and checks + with the ATE API that the actor exists and is running. * Authorizes egress against the actor's `EgressPolicy`: every request the - gateway can read is decided on its `Host` and the address the actor dialed, - rules in order, first match wins; what it cannot read is decided by address - at the CONNECT. An actor with no policy gets no tunnel. See + gateway can read is decided on its `Host` and the port the actor dialed, + the most specific rule winning; TLS it does not terminate is decided by SNI + at the ClientHello. An actor with no policy gets no tunnel. See [egress legs](#egress-legs). * Serves arbitrary-port ingress: a client reaches a port on the actor other than its default (80) by sending an HTTP CONNECT to - `:` on `--port-connect`/`--port-connect-tls`, rather than + `:` on `--port-connect-tls`, rather than naming the port some other way. Envoy terminates the CONNECT and reinjects the tunneled bytes into an internal listener that runs the same ext_proc path as ordinary traffic, so each request inside a long-lived tunnel still resumes the actor and re-routes independently if it moves workers. Only HTTP(S) traffic over the tunnel is supported today -- see `xds.go`'s - `connect_terminate`/`main_internal` listeners and + `connect_terminate_tls`/`main_internal` listeners and `ingress.Handler.HandleRequestHeaders`. ## packages @@ -95,13 +95,12 @@ the request legs match `ports` against it; a dataplane that does not share it gets no port enforcement. The CONNECT leg decides per connection: it refuses an actor without a policy -with rules, and for the rest computes hash of the policy SNI's rules for the -destination port, stores policy in the thread local store using hash as the -key and hands the inner listener the port the actor dialed, as -`envoy.upstream.dynamic_port` and the hash of the policy' as -`dev.ate.policy.egress.hash`. At the ClientHello the egress-policy module -reads the hash, and fetches egress policy from the thrad local store, -applies SNI rules and picks the chain: `egress_tls_mitm` under an `https` +with rules. For the rest it hashes the policy's SNI rules for the destination +port, stores them in the thread-local store under that hash, and hands the +inner listener the port the actor dialed, as `envoy.upstream.dynamic_port`, +and the hash, as `dev.ate.policy.egress.hash`. At the ClientHello the +egress-policy module reads the hash, fetches the rules from the thread-local +store, and picks the chain: `egress_tls_mitm` under an `https` rule, `egress_passthrough` under a `tls_passthrough` rule, and `denied`, which matches no chain and closes the connection, when neither covers the SNI. The passthrough chain resolves the SNI and relays the bytes to it on the dialed @@ -111,9 +110,10 @@ port; the address the actor dialed plays no part (see Identity on the request legs is `dev.ate.actor.identity`, the actor's SPIFFE ID that the outer chain set from the verified peer certificate and shares with the inner listener. Envoy builds every inner connection from the filter state -of the tunnel that created its pool, so `mitm_internal` keeps one pool per -downstream connection, which on the HTTP/1 egress listener is one per tunnel: -no tunnel inherits another's identity, authority or rules. Nothing inside the +of the tunnel that created its pool, so `mitm_internal` keys its pools on the +identity, the CONNECT authority and the policy hash: tunnels share a pool only +when all three match, and no tunnel inherits another's identity, authority or +rules. Nothing inside the tunnel can write any of this; a callout without an identity is refused. Policies are read through a per-actor cache (`--egress-policy-cache-ttl`, 10s @@ -122,24 +122,30 @@ create, update or delete is visible to new requests within one TTL, and a deleted policy becomes a deny. Every policy denial answers a fixed `egress denied` body; the reason is in the sidecar's log. -Credential injection (`replace_headers`) runs on the MITM leg: a header the -request carries is replaced with the credential from the provider, and a +Credential injection (`replace_headers`) runs on both request legs: a header +the request carries is replaced with the credential from the provider, and a request without it is forwarded unchanged. See `docs/egress-credential-injection.md`. ## adding a dataplane attribute The filter-state objects and request attributes a proxy carries alongside -a request are declared once, in `extproc/attributes.go`. +a request are declared once, in `extproc/attributes.go`. The exception is the +keys only the router's generated Envoy config reads, which stay in `xds.go`. | key | direction | purpose | | --- | --- | --- | -| `dev.ate.actor.name` | ingress | carries the actor name across CONNECT re-entry | -| `dev.ate.actor.atespace` | ingress | carries the atespace across CONNECT re-entry | +| `dev.ate.target.actor` | ingress | carries the actor routing target across CONNECT re-entry | | `dev.ate.connect.authority` | ingress, egress | carries the outer CONNECT authority across re-entry: target-port selection on ingress, the dialed port the request legs match `ports` against on egress | | `dev.ate.actor.identity` | egress | carries the authenticated actor identity to the policy ext_proc, the logs and additional ext_proc services | -| `dev.ate.egress:dial` | egress | dynamic metadata: a request leg's answer, `name` or `address`, which picks the route | +| `dev.ate.policy.egress` | egress | dynamic metadata: the CONNECT leg's SNI rules for the dialed port, which the egress-policy module stores by hash | +| `dev.ate.policy.egress.hash` | egress | filter state: the hash the inner listener looks those SNI rules up by | +| `dev.ate.egress:dialed_port` | egress | dynamic metadata: the CONNECT leg's answer, the port the passthrough chain dials the resolved SNI on | +| `dev.ate.egress:dial` | egress | dynamic metadata: a request leg's answer, `name`, which picks the route | +| `dev.ate.egress:host` | egress | dynamic metadata: the name the request was decided on, which the forward proxy dials on the port the actor dialed | +| `dev.ate.egress.filter_chain` | egress | filter state: the egress-policy module's verdict, which selects the inner filter chain | | `dev.ate.extproc.direction` | egress | selects the egress handler for dataplanes without Envoy filter chains | +| `dev.ate.tunnel.{key,worker,actor,port}` | ingress | filter state: the routing decision carried into `actor_connect` (declared in `xds.go`) | ### name it @@ -198,12 +204,9 @@ its own static configuration because ingress and egress scale independently. ## Ingress client authentication and authorization -The ingress router is migrating to require clients to present a SPIFFE certificate. -* With `--ingress-auth-mode=static-mtls`, the ingress router no longer offers any - plaintext listeners, and all clients *must* present a SPIFFE certificate. -* With `--ingress-auth-mode=deprecated-insecure`, the ingress router offers both - plaintext and secure listeners. If a client presents a SPIFFE certificate, it - must be valid. +The Envoy ingress router serves only TLS listeners, and every client must +present a SPIFFE certificate. The agentgateway dataplane does not authenticate +clients yet. Configure the CA that is used to validate client certificates with the `--ingress-client-ca-file` flag, which points at a PEM file containing root @@ -215,6 +218,55 @@ to contact any actor. Richer authorization policies may be added post-GA. Use the `--ingress-allowed-spiffe-ids` flag to set the allowlist (comma-separated list). +### Common setups + +The default install sets `--ingress-client-ca-file` to the trust bundle of +substrate's pod identity signer, `podidentity.podcert.ate.dev/identity`. That +signer issues certificates with the SPIFFE ID +`spiffe://cluster.local/ns//sa/`. + +The router's serving certificate comes from the +`servicedns.podcert.ate.dev/identity` signer. Clients verify it against that +signer's ClusterTrustBundle, with server name +`atenet-router.ate-system.svc`. + +**Custom CA.** An installer can point `--ingress-client-ca-file` at a CA of +their own choosing. Clients then get certificates from that CA however the +installer provides them, and the allowlist holds the SPIFFE IDs it issues. + +**Client in the cluster, pod identity CA.** Mount a client certificate with a +`podCertificate` projected volume, and the router's serving CA with a +`clusterTrustBundle` projected volume: + +```yaml +volumes: +- name: router-client + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: router-ca.pem +``` + +Add the pod's SPIFFE ID to `--ingress-allowed-spiffe-ids`. + +**Client outside the cluster, pod identity CA.** A demo on your machine can +follow the e2e tests in `internal/e2e/ingressauth.go`. It creates a +PodCertificateRequest for the podidentity signer that names a pod, node and +ServiceAccount that don't exist. kube-apiserver checks those only for requests +from a kubelet, and the signer issues the SPIFFE ID of the named namespace and +ServiceAccount. The client keeps the private key, reads the chain from the +request's status, and reads the servicedns ClusterTrustBundle to verify the +router. Creating PodCertificateRequests needs RBAC for that resource in the +namespace. + ## status page The unauthenticated `/statusz` dashboard binds to `127.0.0.1:4040` by default diff --git a/cmd/atenet/internal/router/cmd.go b/cmd/atenet/internal/router/cmd.go index bf52fe9144..17620d57fe 100644 --- a/cmd/atenet/internal/router/cmd.go +++ b/cmd/atenet/internal/router/cmd.go @@ -60,8 +60,6 @@ func bindRouterFlags(fs *pflag.FlagSet, cfg *routerConfig) { fs.StringVar(&cfg.RouterServiceName, "router-service-name", installdefaults.RouterServiceName, "Service name of this atenet-router in the operations namespace. Override when the deployment renames the Service.") fs.StringVar(&cfg.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig configuration file") fs.StringVar(&cfg.AteapiAddr, "ateapi-address", "k8s:///api.ate-system.svc:443", "gRPC dial target for the cluster ateapi Control instance.") - fs.IntVar(&cfg.HttpPort, "port-http", 8080, "TCP port for workload traffic entering through the Envoy Router") - fs.IntVar(&cfg.ConnectPlainTextPort, "port-connect", 8081, "TCP port for CONNECT-tunneled traffic entering through the router dataplane") fs.IntVar(&cfg.ConnectTLSPort, "port-connect-tls", 8444, "TCP port for CONNECT-tunneled traffic entering through the router dataplane over TLS. --port-https also defaults to 8443, and both listeners are commonly enabled at once, so --port-connect-tls defaults to a different port (8444) rather than colliding with it") fs.IntVar(&cfg.XdsPort, "port-xds", 18000, "TCP port listening for the xDS dynamic Envoy connections") fs.IntVar(&cfg.ExtprocPort, "port-extproc", 50051, "Listen port for the External Processing (ext_proc) server the dataplane calls") @@ -73,14 +71,8 @@ func bindRouterFlags(fs *pflag.FlagSet, cfg *routerConfig) { fs.StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file.") // Flags for client authentication and authorization in ingress mode. - fs.StringVar(&cfg.IngressAuth.ClientCAFile, "ingress-client-ca-file", "", "PEM bundle of the CAs allowed to sign ingress client certificates. Envoy reads the file, so it must be mounted at this path in the Envoy container as well. Required whenever a TLS ingress listener is enabled") - fs.StringSliceVar(&cfg.IngressAuth.AllowedSPIFFEIDs, "ingress-allowed-spiffe-ids", nil, "SPIFFE IDs (exact match) allowed to connect to actors through the ingress dataplane. Comma-separated or repeated. Required whenever a TLS ingress listener is enabled") - fs.StringVar( - (*string)(&cfg.IngressAuth.Mode), - "ingress-auth-mode", - string(IngressAuthDeprecatedInsecure), - "Ingress authentication and authorization mode. `deprecated-insecure` (both http and https served, client certificates optional on https) or `static-mtls` (only https served, client certificates required).", - ) + fs.StringVar(&cfg.IngressAuth.ClientCAFile, "ingress-client-ca-file", "", "PEM bundle of the CAs allowed to sign ingress client certificates. Envoy reads the file, so it must be mounted at this path in the Envoy container as well. Required for the Envoy ingress dataplane") + fs.StringSliceVar(&cfg.IngressAuth.AllowedSPIFFEIDs, "ingress-allowed-spiffe-ids", nil, "SPIFFE IDs (exact match) allowed to connect to actors through the ingress dataplane. Comma-separated or repeated. Required for the Envoy ingress dataplane") fs.StringVar(&cfg.UpstreamCredentialBundlePath, "upstream-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "PEM credential bundle (cert+key) the router presents as the client cert when dialing the actor's atunnel ingress server over mTLS. Empty disables upstream mTLS (legacy plaintext pod-IP:80) and needs --upstream-connect=false.") fs.StringVar(&cfg.UpstreamTrustBundlePath, "upstream-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "PEM trust bundle used to validate the actor's atunnel ingress server certificate.") diff --git a/cmd/atenet/internal/router/config.go b/cmd/atenet/internal/router/config.go index 84c43f4c6f..5345caef7d 100644 --- a/cmd/atenet/internal/router/config.go +++ b/cmd/atenet/internal/router/config.go @@ -63,51 +63,23 @@ func (m Mode) ServesIngress() bool { return m == ModeIngress } // ServesEgress reports whether this mode answers egress CONNECTs. func (m Mode) ServesEgress() bool { return m == ModeEgress } -// IngressAuthMode selects how the ingress dataplane authenticates and -// authorizes the clients that connect to it. -type IngressAuthMode string - -const ( - // IngressAuthDeprecatedInsecure serves the plaintext ingress listeners, - // which accept every client and let any of them reach any actor, - // alongside the TLS listeners, which authenticate clients exactly as in - // IngressAuthStaticMTLS. It exists so clients can move to mTLS one at a - // time, and will be removed once they have. - IngressAuthDeprecatedInsecure IngressAuthMode = "deprecated-insecure" - // IngressAuthStaticMTLS requires every client to present a certificate - // that chains to the configured client CA bundle and carries one of the - // allowed SPIFFE IDs as a URI SAN. Envoy enforces both during the TLS - // handshake, so only the TLS listeners may be enabled. - IngressAuthStaticMTLS IngressAuthMode = "static-mtls" -) - // ingressAuthConfig holds the client authentication settings for the ingress -// dataplane's listeners. +// dataplane's listeners. Every client must present a certificate that chains +// to ClientCAFile and carries one of AllowedSPIFFEIDs as a URI SAN. Envoy +// enforces both during the TLS handshake. type ingressAuthConfig struct { - // Mode is the authentication mode. Empty means - // IngressAuthDeprecatedInsecure. - Mode IngressAuthMode // ClientCAFile is the PEM bundle of CAs allowed to sign client // certificates. Envoy reads it, so it must be mounted in the Envoy - // container at the same path. Required whenever a TLS listener is - // enabled. + // container at the same path. ClientCAFile string // AllowedSPIFFEIDs is the exact set of SPIFFE IDs allowed to connect to - // actors through the TLS listeners. Required whenever a TLS listener is - // enabled. + // actors. // // Entries are complete SPIFFE IDs, like // `spiffe://mytrustdomain.com/foo/bar/baz`. AllowedSPIFFEIDs []string } -func (c ingressAuthConfig) mode() IngressAuthMode { - if c.Mode == "" { - return IngressAuthDeprecatedInsecure - } - return c.Mode -} - // authConfig holds the router's mTLS settings for dialing ateapi. type authConfig struct { AteapiCAFile string @@ -150,7 +122,6 @@ type routerConfig struct { RouterServiceName string Kubeconfig string AteapiAddr string - HttpPort int XdsPort int ExtprocPort int ExtprocAddr string @@ -158,12 +129,10 @@ type routerConfig struct { StatusAddr string HealthInterval time.Duration HttpsPort int - // ConnectPlainTextPort and ConnectTLSPort are the plaintext and TLS - // listener ports for CONNECT-tunneled traffic. Non-positive disables the - // corresponding listener. - ConnectPlainTextPort int - ConnectTLSPort int - EnvoyCertPath string + // ConnectTLSPort is the listener port for CONNECT-tunneled traffic. + // Non-positive disables the listener. + ConnectTLSPort int + EnvoyCertPath string // UpstreamCredentialBundlePath is the router's podidentity credential bundle // (cert+key) presented as the client cert when dialing the actor's atunnel @@ -340,60 +309,25 @@ func (c routerConfig) validate() error { return nil } -// tlsIngressEnabled reports whether the Envoy dataplane builds at least one -// TLS ingress listener. -func (c routerConfig) tlsIngressEnabled() bool { - return c.EnvoyCertPath != "" && (c.HttpsPort > 0 || c.ConnectTLSPort > 0) -} - -// validateIngressAuth rejects a configuration that would leave a TLS listener -// without client authentication, a static-mtls configuration that would leave -// any path to an actor unauthenticated, and one the dataplane cannot enforce. +// validateIngressAuth rejects an Envoy ingress configuration with no TLS +// listener, or one whose clients cannot be authenticated. Agentgateway's +// listeners come from its static configuration, so the router has none to +// check. func (c routerConfig) validateIngressAuth() error { - mode := c.IngressAuth.mode() - switch mode { - case IngressAuthDeprecatedInsecure, IngressAuthStaticMTLS: - default: - return fmt.Errorf("--ingress-auth-mode must be %q or %q, got %q", IngressAuthDeprecatedInsecure, IngressAuthStaticMTLS, c.IngressAuth.Mode) - } - // The egress gateway never builds the ingress listeners. - if !c.Mode.ServesIngress() { + if !c.Mode.ServesIngress() || c.atenetRouter() != atenetRouterEnvoy { return nil } - if mode == IngressAuthDeprecatedInsecure { - // Agentgateway's listeners come from its static configuration, so - // the router has none to authenticate. - if c.atenetRouter() != atenetRouterEnvoy || !c.tlsIngressEnabled() { - return nil - } - return c.validateIngressClientAuth() - } - if c.atenetRouter() != atenetRouterEnvoy { - return fmt.Errorf("--ingress-auth-mode=%s requires --atenet-dataplane=%s: the %s dataplane is configured statically, not by the router", IngressAuthStaticMTLS, atenetRouterEnvoy, c.atenetRouter()) - } - if c.HttpPort > 0 { - return fmt.Errorf("--ingress-auth-mode=%s requires --port-http=0: the plaintext listener cannot authenticate clients", IngressAuthStaticMTLS) - } - if c.ConnectPlainTextPort > 0 { - return fmt.Errorf("--ingress-auth-mode=%s requires --port-connect=0: the plaintext listener cannot authenticate clients", IngressAuthStaticMTLS) - } if c.EnvoyCertPath == "" { - return fmt.Errorf("--ingress-auth-mode=%s requires --envoy-cert-path: without a serving certificate there is no TLS listener", IngressAuthStaticMTLS) + return fmt.Errorf("--envoy-cert-path is required: without a serving certificate there is no TLS listener") } if c.HttpsPort <= 0 && c.ConnectTLSPort <= 0 { - return fmt.Errorf("--ingress-auth-mode=%s requires --port-https or --port-connect-tls: there is no TLS listener", IngressAuthStaticMTLS) + return fmt.Errorf("--port-https or --port-connect-tls is required: there is no TLS listener") } - return c.validateIngressClientAuth() -} - -// validateIngressClientAuth checks the settings the TLS listeners authenticate -// clients with. -func (c routerConfig) validateIngressClientAuth() error { if c.IngressAuth.ClientCAFile == "" { - return fmt.Errorf("TLS ingress listeners (--port-https, --port-connect-tls) require --ingress-client-ca-file") + return fmt.Errorf("--ingress-client-ca-file is required") } if len(c.IngressAuth.AllowedSPIFFEIDs) == 0 { - return fmt.Errorf("TLS ingress listeners (--port-https, --port-connect-tls) require at least one --ingress-allowed-spiffe-ids entry") + return fmt.Errorf("at least one --ingress-allowed-spiffe-ids entry is required") } for _, id := range c.IngressAuth.AllowedSPIFFEIDs { if _, err := spiffeid.FromString(id); err != nil { diff --git a/cmd/atenet/internal/router/config_test.go b/cmd/atenet/internal/router/config_test.go index ece06c928b..d047cee62a 100644 --- a/cmd/atenet/internal/router/config_test.go +++ b/cmd/atenet/internal/router/config_test.go @@ -126,7 +126,13 @@ func TestRouterConfigValidate(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - err := tc.cfg.validate() + // TestRouterConfigValidateIngressAuth covers the ingress + // listener settings, so every row here gets valid ones. + cfg := tc.cfg + cfg.HttpsPort = 8443 + cfg.EnvoyCertPath = "/run/servicedns.podcert.ate.dev/credential-bundle.pem" + cfg.IngressAuth = validIngressAuthConfig() + err := cfg.validate() if tc.wantErr == "" { if err != nil { t.Fatalf("expected valid, got %v", err) @@ -140,20 +146,23 @@ func TestRouterConfigValidate(t *testing.T) { } } -// staticMTLSConfig returns a valid static-mtls ingress configuration for +func validIngressAuthConfig() ingressAuthConfig { + return ingressAuthConfig{ + ClientCAFile: "/run/podidentity.podcert.ate.dev/trust-bundle.pem", + AllowedSPIFFEIDs: []string{"spiffe://cluster.local/ns/demo/sa/client"}, + } +} + +// validIngressConfig returns a valid ingress configuration for // TestRouterConfigValidateIngressAuth to break one field at a time. -func staticMTLSConfig() routerConfig { +func validIngressConfig() routerConfig { return routerConfig{ Mode: ModeIngress, ParkedRequest: ingress.ParkedRequestConfig{Max: ingress.DefaultParkedRequestMax}, HttpsPort: 8443, ConnectTLSPort: 8444, EnvoyCertPath: "/run/servicedns.podcert.ate.dev/credential-bundle.pem", - IngressAuth: ingressAuthConfig{ - Mode: IngressAuthStaticMTLS, - ClientCAFile: "/run/podidentity.podcert.ate.dev/trust-bundle.pem", - AllowedSPIFFEIDs: []string{"spiffe://cluster.local/ns/demo/sa/client"}, - }, + IngressAuth: validIngressAuthConfig(), } } @@ -164,121 +173,37 @@ func TestRouterConfigValidateIngressAuth(t *testing.T) { wantErr string // substring; empty means valid }{ { - name: "static-mtls with only TLS listeners accepted", + name: "both TLS listeners accepted", mutate: func(*routerConfig) {}, }, { - name: "static-mtls with only the HTTPS listener accepted", + name: "only the HTTPS listener accepted", mutate: func(c *routerConfig) { c.ConnectTLSPort = 0 }, }, { - name: "static-mtls with only the CONNECT-TLS listener accepted", + name: "only the CONNECT-TLS listener accepted", mutate: func(c *routerConfig) { c.HttpsPort = 0 }, }, - { - name: "unknown mode rejected", - mutate: func(c *routerConfig) { c.IngressAuth.Mode = "mtls" }, - wantErr: "--ingress-auth-mode must be", - }, - { - name: "old insecure mode name rejected", - mutate: func(c *routerConfig) { c.IngressAuth.Mode = "insecure" }, - wantErr: "--ingress-auth-mode must be", - }, { name: "egress-only instance ignores the ingress settings", mutate: func(c *routerConfig) { c.Mode = ModeEgress - c.HttpPort = 8080 - c.IngressAuth.ClientCAFile = "" - c.IngressAuth.AllowedSPIFFEIDs = nil - }, - }, - { - name: "deprecated-insecure accepts plaintext listeners alongside mTLS ones", - mutate: func(c *routerConfig) { - c.IngressAuth.Mode = IngressAuthDeprecatedInsecure - c.HttpPort = 8080 - c.ConnectPlainTextPort = 8081 - }, - }, - { - name: "empty mode means deprecated-insecure", - mutate: func(c *routerConfig) { - c.IngressAuth.Mode = "" - c.HttpPort = 8080 - }, - }, - { - name: "deprecated-insecure with only plaintext listeners needs no client auth", - mutate: func(c *routerConfig) { - c.IngressAuth = ingressAuthConfig{Mode: IngressAuthDeprecatedInsecure, AllowedSPIFFEIDs: []string{"not a spiffe id"}} - c.HttpPort = 8080 - c.HttpsPort = 0 - c.ConnectTLSPort = 0 - }, - }, - { - name: "deprecated-insecure without a serving cert needs no client auth", - mutate: func(c *routerConfig) { - c.IngressAuth = ingressAuthConfig{Mode: IngressAuthDeprecatedInsecure} - c.HttpPort = 8080 c.EnvoyCertPath = "" + c.IngressAuth = ingressAuthConfig{} }, }, { - name: "deprecated-insecure on agentgateway needs no client auth", + name: "agentgateway ignores the ingress settings", mutate: func(c *routerConfig) { - c.IngressAuth = ingressAuthConfig{Mode: IngressAuthDeprecatedInsecure} c.AtenetRouter = string(atenetRouterAgentgateway) - c.HttpPort = 8080 - }, - }, - { - name: "deprecated-insecure TLS listener without client CA rejected", - mutate: func(c *routerConfig) { - c.IngressAuth.Mode = IngressAuthDeprecatedInsecure - c.IngressAuth.ClientCAFile = "" - c.HttpPort = 8080 - }, - wantErr: "require --ingress-client-ca-file", - }, - { - name: "deprecated-insecure TLS listener without allowlist rejected", - mutate: func(c *routerConfig) { - c.IngressAuth.Mode = IngressAuthDeprecatedInsecure - c.IngressAuth.AllowedSPIFFEIDs = nil - c.HttpPort = 8080 - }, - wantErr: "require at least one --ingress-allowed-spiffe-ids", - }, - { - name: "deprecated-insecure non-SPIFFE ID rejected", - mutate: func(c *routerConfig) { - c.IngressAuth.Mode = IngressAuthDeprecatedInsecure - c.IngressAuth.AllowedSPIFFEIDs = []string{"https://cluster.local/ns/demo/sa/client"} + c.EnvoyCertPath = "" + c.IngressAuth = ingressAuthConfig{} }, - wantErr: "is not a valid SPIFFE ID", - }, - { - name: "agentgateway rejected", - mutate: func(c *routerConfig) { c.AtenetRouter = string(atenetRouterAgentgateway) }, - wantErr: "requires --atenet-dataplane=envoy", - }, - { - name: "plaintext HTTP listener rejected", - mutate: func(c *routerConfig) { c.HttpPort = 8080 }, - wantErr: "requires --port-http=0", - }, - { - name: "plaintext CONNECT listener rejected", - mutate: func(c *routerConfig) { c.ConnectPlainTextPort = 8081 }, - wantErr: "requires --port-connect=0", }, { name: "missing serving cert rejected", mutate: func(c *routerConfig) { c.EnvoyCertPath = "" }, - wantErr: "requires --envoy-cert-path", + wantErr: "--envoy-cert-path is required", }, { name: "no TLS listener rejected", @@ -286,17 +211,17 @@ func TestRouterConfigValidateIngressAuth(t *testing.T) { c.HttpsPort = 0 c.ConnectTLSPort = 0 }, - wantErr: "requires --port-https or --port-connect-tls", + wantErr: "--port-https or --port-connect-tls is required", }, { name: "missing client CA rejected", mutate: func(c *routerConfig) { c.IngressAuth.ClientCAFile = "" }, - wantErr: "require --ingress-client-ca-file", + wantErr: "--ingress-client-ca-file is required", }, { name: "empty allowlist rejected", mutate: func(c *routerConfig) { c.IngressAuth.AllowedSPIFFEIDs = nil }, - wantErr: "require at least one --ingress-allowed-spiffe-ids", + wantErr: "at least one --ingress-allowed-spiffe-ids entry is required", }, { name: "non-SPIFFE ID rejected", @@ -315,7 +240,7 @@ func TestRouterConfigValidateIngressAuth(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - cfg := staticMTLSConfig() + cfg := validIngressConfig() tc.mutate(&cfg) err := cfg.validate() if tc.wantErr == "" { diff --git a/cmd/atenet/internal/router/dashboard.html b/cmd/atenet/internal/router/dashboard.html index 856f2224d7..9b4e38badc 100644 --- a/cmd/atenet/internal/router/dashboard.html +++ b/cmd/atenet/internal/router/dashboard.html @@ -257,8 +257,8 @@

atenet Router Status

Component Network Allocation