From ebeeae0f49050440c9848ab86e81e728a4242a13 Mon Sep 17 00:00:00 2001 From: "Haowei Cai (Roy)" Date: Wed, 21 Oct 2026 22:06:28 +0000 Subject: [PATCH 01/12] ateworker-microvm, atelet: classify pre-serving restore failures as Unavailable (#2744) When a micro-VM restore in `restoreMemoryFidelity` fails to resume the guest, reach `kata-agent`, or reseed the guest CRNG before serving any inbound traffic, the worker's deferred cleanup already stops cloud-hypervisor and virtiofsd and unhosts the actor. However, `restoreMemoryFidelity` returned a plain `fmt.Errorf` (which the worker reports as `codes.Internal`, or `DeadlineExceeded` when the cause was a timeout) and `Atelet.Restore` wrapped all worker `RestoreWorkload` errors with `fmt.Errorf` (`codes.Internal`). So `ateapi`'s `handleAteletError` treated transient post-resume guest failures as fatal and moved the actor to `ACTOR_STATE_CRASHED` instead of keeping `ACTOR_STATE_RESUMING` for `ActorResumer` to retry. This PR: - updates the `apierror.Unavailable` doc comment in `internal/apierror/apierror.go` to cover an attempt that was rolled back so retrying it is safe, - adds `restoreGuestReadyError` in `cmd/ateworker-microvm/restore.go`, so pre-serving guest readiness failures (`client.Resume`, `dialAgentRetry`, `reseedGuestCRNG`) report `apierror.Unavailable` when the caller's context is still active. When it wraps `ctx.Err()`, it formats the cause with `%v`, so a child `context.DeadlineExceeded` does not shadow a caller `context.Canceled`. "Pre-serving" means before inbound traffic (not even the wakeup probe has connected yet). After `Resume`, the guest and its workload do run until the agent is dialed and the CRNG reseeded: atunnel's egress proxy closes their outbound TCP connections until `tunnel.Activate`, and the retry re-extracts the rootfs upper and durable-dir volumes from the snapshot, whereas any background writes made to a CSI volume during that window stay while guest memory and its page cache resume from the snapshot, - moves the `restoreMemoryFidelity` failure cleanup `defer` right after `PrepareEgress`, so it runs last (after the untar drain and the cloud-hypervisor and virtiofsd kills). It now sweeps the sandbox state, removes the rootfs upper dir, and detaches the bundle rootfs overlays (`cleanupFailedRestoreRootfs`, mirroring `teardownActor`), so nothing stays mounted across a retry. Two side effects: - failures from `buildActorContainers` through `stageMergedRootfs`, which used to leave the bundle overlays mounted, now detach them too; - like `teardownActor`, the sweep removes `/run/vc/vm/`, so a failed restore no longer leaves the guest's console, serial and virtiofsd logs on the node, - adds `workerRestoreError` in `cmd/atelet/main.go` (resolving the `TODO` in `Atelet.Restore`). It propagates a worker `codes.Unavailable`, which a transport error also arrives as, as `apierror.Unavailable`, so `ateapi` preserves `ACTOR_STATE_RESUMING` and `WorkerAssignment` for retry. Other worker errors still crash the actor, including a worker-side `Canceled` or `DeadlineExceeded`, which can come from any step, - bounds retries of a restore that keeps failing, so it does not keep the actor `RESUMING` forever: - a guest found stopped while dialing the agent (`errGuestStopped`, e.g. a guest kernel panic after resume or a corrupt memory image) is reported as an error rather than `Unavailable` (`coldBootActorRetrying` likewise caps retries of that case); - `atelet` reports an actor's third consecutive retryable worker failure to restore the same snapshot as `Internal`; - a successful restore, a restore error that crashes the actor, or a successful `Terminate` ends the streak, while an error that keeps the actor `RESUMING` (a context error, or another `Unavailable`) leaves it as is, and - adds unit tests in `cmd/ateworker-microvm/restore_test.go`, `cmd/ateworker-microvm/restore_linux_test.go` (the overlay case is root-gated), and `cmd/atelet/lifecycle_test.go`. #### How I tested it - `go test -race ./internal/apierror/... ./cmd/ateworker-microvm/... ./cmd/atelet/...` - `go build` and `go vet` at each commit, and `go vet ./...` on the head - The root-gated `TestCleanupFailedRestoreRootfs` overlay case, run in a user and mount namespace (`unshare -r -m`) - `./hack/verify/gofmt.sh` and `./hack/verify/boilerplate.sh` - [x] Tests pass - [ ] Appropriate changes to documentation are included in the PR --- cmd/atelet/lifecycle_test.go | 180 ++++++++++++++++++++ cmd/atelet/main.go | 122 ++++++++++++- cmd/ateworker-microvm/restore.go | 92 ++++++++-- cmd/ateworker-microvm/restore_linux_test.go | 119 +++++++++++++ cmd/ateworker-microvm/restore_test.go | 41 +++++ internal/apierror/apierror.go | 5 +- 6 files changed, 538 insertions(+), 21 deletions(-) create mode 100644 cmd/ateworker-microvm/restore_linux_test.go diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index d94bb2fe49..4fb7c222b3 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -42,6 +42,7 @@ import ( "go.ate.dev/substrate/internal/proto/ateletpb" "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/volume" "go.ate.dev/substrate/pkg/api/v1alpha1" "go.opentelemetry.io/otel/sdk/metric/metricdata" "google.golang.org/grpc" @@ -97,6 +98,9 @@ type fakeWorker struct { // onTerminate, if non-nil, runs inside TerminateWorkload, which fails with // the error it returns. onTerminate func() error + // onRestore, if non-nil, runs inside RestoreWorkload after it recorded + // the request, before it fails with restoreErr or succeeds. + onRestore func() } func (f *fakeWorker) recordFidelity(rpc string, fidelity ateworkerpb.SnapshotFidelity) { @@ -143,6 +147,9 @@ func (f *fakeWorker) RestoreWorkload(_ context.Context, req *ateworkerpb.Restore f.recordActorDirs("RestoreWorkload", req.GetActorDirs()) f.recordFidelity("RestoreWorkload", req.GetFidelity()) f.preserveRestoreDir = req.GetPreserveRestoreDir() + if f.onRestore != nil { + f.onRestore() + } if f.restoreErr != nil { return nil, f.restoreErr } @@ -2952,3 +2959,176 @@ func TestCheckpointSkipsStaleCommitAndKeepsNewerLocal(t *testing.T) { t.Errorf("stale local snapshot pause-snap-2: stat = %v, want pruned", err) } } + +func TestWorkerRestoreError(t *testing.T) { + canceledCtx, cancel := context.WithCancel(t.Context()) + cancel() + + for _, tc := range []struct { + name string + ctx context.Context + err error + wantCode codes.Code + }{ + { + name: "worker Unavailable maps to apierror.Unavailable", + ctx: t.Context(), + err: status.Error(codes.Unavailable, "while reseeding guest CRNG: context deadline exceeded"), + wantCode: codes.Unavailable, + }, + { + // The worker reports any step's internal timeout this way, not + // only one before the workload served traffic. + name: "worker DeadlineExceeded with live caller context maps to codes.Internal", + ctx: t.Context(), + err: status.Error(codes.DeadlineExceeded, "context deadline exceeded"), + wantCode: codes.Internal, + }, + { + name: "worker Canceled with live caller context maps to codes.Internal", + ctx: t.Context(), + err: status.Error(codes.Canceled, "context canceled"), + wantCode: codes.Internal, + }, + { + name: "worker Internal maps to codes.Internal", + ctx: t.Context(), + err: status.Error(codes.Internal, "corrupt snapshot config"), + wantCode: codes.Internal, + }, + { + name: "caller context canceled reports Canceled", + ctx: canceledCtx, + err: status.Error(codes.Internal, "worker aborted"), + wantCode: codes.Canceled, + }, + } { + t.Run(tc.name, func(t *testing.T) { + got := workerRestoreError(tc.ctx, tc.err) + if code := apierror.Code(got); code != tc.wantCode { + t.Errorf("apierror.Code(workerRestoreError()) = %v, want %v (err: %v)", code, tc.wantCode, got) + } + if !errors.Is(got, tc.err) { + t.Errorf("workerRestoreError() = %v, want it to wrap %v", got, tc.err) + } + }) + } +} + +// withExternalVolumes gives the actor's spec the external volumes ids, served +// by a fake volume plugin, and returns the plugin. +func (a *pluginActor) withExternalVolumes(ids ...string) *fakeWorkerPlugin { + volPlugin := &fakeWorkerPlugin{} + a.s.volumePlugins = map[string]volume.VolumePluginWorkerPlane{ + "mock-driver": volPlugin, + } + a.spec.Volumes = nil + for _, id := range ids { + a.spec.Volumes = append(a.spec.Volumes, &ateletpb.Volume{ + Name: "ext-" + id, + External: &ateletpb.ExternalVolumeSource{ + StorageVolumeId: id, + VolumeType: "mock-driver", + }, + }) + } + return volPlugin +} + +// restoreRequest is the control plane's request to restore the actor from +// snap. +func (a *pluginActor) restoreRequest(t *testing.T, snap *ateletpb.Snapshot) *ateletpb.RestoreRequest { + t.Helper() + return &ateletpb.RestoreRequest{ + Atespace: "ate-demo", + ActorName: "counter", + ActorUid: pluginActorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + WorkerPodUid: pluginWorkerPodUID, + SandboxRuntime: runtimeFor(t, a.rec), + Spec: a.spec, + Snapshot: snap, + } +} + +// checkpointed checkpoints the actor as snapshot name and returns the +// snapshot. +func (a *pluginActor) checkpointed(t *testing.T, name string, generation int32) *ateletpb.Snapshot { + t.Helper() + snap := a.snapshot(name, generation) + if _, err := a.checkpoint(t, snap, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); err != nil { + t.Fatalf("Checkpoint: %v", err) + } + return snap +} + +var errWorkerGuestNotReady = status.Error(codes.Unavailable, "while reseeding guest CRNG: context deadline exceeded") + +func TestRestoreStopsRetryingAfterConsecutiveUnavailableFailures(t *testing.T) { + a := runPluginActor(t) + snap := a.checkpointed(t, "pause-snap-1", 1) + restoreReq := a.restoreRequest(t, snap) + restoreCode := func() codes.Code { + _, err := a.s.Restore(t.Context(), restoreReq) + return apierror.Code(err) + } + failRetryably := func(times int, after string) { + t.Helper() + a.worker.restoreErr = errWorkerGuestNotReady + for i := 1; i <= times; i++ { + if got := restoreCode(); got != codes.Unavailable { + t.Fatalf("Restore %d %s: code = %v, want %v", i, after, got, codes.Unavailable) + } + } + } + + failRetryably(maxConsecutiveRetryableRestoreFailures-1, "at first") + if got := restoreCode(); got != codes.Internal { + t.Fatalf("Restore %d code = %v, want %v", maxConsecutiveRetryableRestoreFailures, got, codes.Internal) + } + + // Giving up starts the count over, and so does a successful restore. + failRetryably(maxConsecutiveRetryableRestoreFailures-1, "after giving up") + a.worker.restoreErr = nil + if got := restoreCode(); got != codes.OK { + t.Fatalf("successful Restore code = %v, want %v", got, codes.OK) + } + failRetryably(maxConsecutiveRetryableRestoreFailures-1, "after a success") + + // Restoring another snapshot starts a new streak. + restoreReq = a.restoreRequest(t, a.checkpointed(t, "pause-snap-2", 2)) + failRetryably(maxConsecutiveRetryableRestoreFailures-1, "of another snapshot") + + // So does a failure that crashes the actor: the control plane's crash + // handling does not call Terminate, so nothing else would clear the count + // before the actor is restored here again. + a.worker.restoreErr = status.Error(codes.Internal, "corrupt snapshot config") + if got := restoreCode(); got != codes.Internal { + t.Fatalf("Restore with a worker Internal error: code = %v, want %v", got, codes.Internal) + } + failRetryably(maxConsecutiveRetryableRestoreFailures-1, "after a crash") + + // So does a failure in atelet's own steps before the worker call, here a + // volume that does not mount. + volPlugin := a.withExternalVolumes("vol-1") + volPlugin.mountErr = errors.New("attach failed") + if got := restoreCode(); got != codes.Internal { + t.Fatalf("Restore with a failing volume mount: code = %v, want %v", got, codes.Internal) + } + volPlugin.mountErr = nil + failRetryably(maxConsecutiveRetryableRestoreFailures-1, "after a failed mount") + + // An attempt atelet stopped waiting on leaves the actor RESUMING, and the + // count as is, so the next retryable failure is the last. + ctx, cancel := context.WithCancel(t.Context()) + a.worker.onRestore = cancel + _, err := a.s.Restore(ctx, restoreReq) + if got := apierror.Code(err); got != codes.Canceled { + t.Fatalf("Restore that atelet stopped waiting on: code = %v, want %v (err: %v)", got, codes.Canceled, err) + } + a.worker.onRestore = nil + if got := restoreCode(); got != codes.Internal { + t.Fatalf("Restore after a canceled attempt: code = %v, want %v", got, codes.Internal) + } +} diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 22c1ebabd5..1aebee24e9 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -718,6 +718,11 @@ type Atelet struct { actorLocks actorlock.Locks // workerActorJWTs mints the JWT every snapshot transfer carries. workerActorJWTs *workerActorJWTs + + // restoreFailures counts each actor's consecutive retryable worker restore + // failures, so a restore that can never succeed stops being retried (see + // boundRetryableRestoreError). + restoreFailures restoreFailureCounter } var _ ateletpb.AteletServer = (*Atelet)(nil) @@ -1569,6 +1574,15 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res return nil, err } defer unlockActor() + // End the actor's streak of retryable worker failures (see + // boundRetryableRestoreError) if this restore succeeds or fails with an + // error that crashes the actor. Registered after unlockActor, so it runs + // while the actor's lock is still held. + defer func() { + if endsRestoreFailureStreak(err) { + s.restoreFailures.reset(actorUID) + } + }() // A detach in progress finishes first. If it ran, its local copy is gone, // and the restore reads snapshot.object. @@ -1765,8 +1779,7 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res }) dWorker = time.Since(tWorker) if err != nil { - // TODO: classify the errors returned by Worker and crash the actor if needed. - return nil, fmt.Errorf("while calling worker RestoreWorkload: %w", err) + return nil, s.boundRetryableRestoreError(actorUID, req.GetSnapshot().GetSnapshotUuid(), workerRestoreError(ctx, err)) } // Record the sandbox binaries actually running the guest on-node so a @@ -1799,6 +1812,110 @@ func (s *Atelet) DetachActor(ctx context.Context, req *ateletpb.DetachActorReque return &ateletpb.DetachActorResponse{State: state}, nil } +// workerRestoreError translates a failed worker RestoreWorkload call into the +// error Atelet.Restore reports to the control plane. When atelet's own context +// ended, wrap ctx.Err() so the server reports the context status. When the +// worker reports codes.Unavailable, the restore was not acted on, or it failed +// before the workload served any inbound traffic and the worker tore down its +// local state (see apierror.Unavailable and the worker's +// restoreGuestReadyError), so report apierror.Unavailable to let the control +// plane keep the actor in ACTOR_STATE_RESUMING and retry. A transport error, +// such as from a worker pod that is restarting, arrives as codes.Unavailable +// too. All other worker errors, including a worker-side Canceled or +// DeadlineExceeded that may have come from any step of the restore, reach the +// control plane as codes.Internal, crashing the actor. +func workerRestoreError(ctx context.Context, err error) error { + if ctx.Err() != nil { + return fmt.Errorf("while calling worker RestoreWorkload: %w: %w", err, ctx.Err()) + } + if status.Code(err) == codes.Unavailable { + return apierror.Unavailable("while calling worker RestoreWorkload: %w", err) + } + return fmt.Errorf("while calling worker RestoreWorkload: %w", err) +} + +// maxConsecutiveRetryableRestoreFailures is the number of consecutive retryable +// worker failures to restore one actor's snapshot at which atelet gives up and +// reports that failure as Internal, crashing the actor. Contention on a busy +// node should clear within a retry or two; a restore that keeps failing (a +// guest that hangs on every resume of its snapshot, say) would otherwise keep +// the actor RESUMING forever, visible only as 503s. +const maxConsecutiveRetryableRestoreFailures = 3 + +// restoreFailureCounter counts each actor's consecutive retryable worker +// restore failures on this node, per snapshot. The zero value is ready to use. +// Entries for actors that left this node stay until the actor is restored or +// terminated here again. +type restoreFailureCounter struct { + mu sync.Mutex + streaks map[string]restoreFailureStreak +} + +// restoreFailureStreak is one actor's run of consecutive retryable restore +// failures of the snapshot snapshotUUID. +type restoreFailureStreak struct { + snapshotUUID string + failures int +} + +// add records another consecutive failure to restore snapshotUUID for actorUID +// and returns the count. A failure for another snapshot than the actor's +// current streak starts a new streak. +func (c *restoreFailureCounter) add(actorUID, snapshotUUID string) int { + c.mu.Lock() + defer c.mu.Unlock() + if c.streaks == nil { + c.streaks = map[string]restoreFailureStreak{} + } + streak := c.streaks[actorUID] + if streak.snapshotUUID != snapshotUUID { + streak = restoreFailureStreak{snapshotUUID: snapshotUUID} + } + streak.failures++ + c.streaks[actorUID] = streak + return streak.failures +} + +// reset forgets actorUID's consecutive failures. +func (c *restoreFailureCounter) reset(actorUID string) { + c.mu.Lock() + defer c.mu.Unlock() + delete(c.streaks, actorUID) +} + +// boundRetryableRestoreError counts a retryable (Unavailable) worker failure to +// restore snapshotUUID for actorUID and, once the actor has failed +// maxConsecutiveRetryableRestoreFailures times in a row, reports it as Internal +// instead so the control plane crashes the actor rather than retrying again. +// Any other error passes through uncounted. +func (s *Atelet) boundRetryableRestoreError(actorUID, snapshotUUID string, err error) error { + if apierror.Code(err) != codes.Unavailable { + return err + } + n := s.restoreFailures.add(actorUID, snapshotUUID) + if n < maxConsecutiveRetryableRestoreFailures { + return err + } + s.restoreFailures.reset(actorUID) + return apierror.Internal("worker restore failed with a retryable error %d times in a row, giving up: %v", n, err) +} + +// endsRestoreFailureStreak reports whether a Restore that returned err ends the +// actor's streak of retryable worker failures: it does if the restore +// succeeded or failed with an error that crashes the actor. The control plane +// keeps the actor RESUMING for another attempt on Unavailable, Canceled and +// DeadlineExceeded (see ateletErrorCrashesActor in ateapi), so those leave the +// streak as is: only a worker's Unavailable adds to it (see +// boundRetryableRestoreError), and an attempt cut short adds nothing. +func endsRestoreFailureStreak(err error) bool { + switch apierror.Code(err) { + case codes.Unavailable, codes.Canceled, codes.DeadlineExceeded: + return false + default: + return true + } +} + // Terminate terminates any running workload on worker, unmounts external volumes, // and resets actor directories on the node. func (s *Atelet) Terminate(ctx context.Context, req *ateletpb.TerminateRequest) (*ateletpb.TerminateResponse, error) { @@ -1884,6 +2001,7 @@ func (s *Atelet) Terminate(ctx context.Context, req *ateletpb.TerminateRequest) if err := removeActorDirs(actorUID); err != nil { return nil, fmt.Errorf("failed to remove actor directories during terminate (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) } + s.restoreFailures.reset(actorUID) return &ateletpb.TerminateResponse{}, nil } diff --git a/cmd/ateworker-microvm/restore.go b/cmd/ateworker-microvm/restore.go index 25243ab3dc..61a8bbc086 100644 --- a/cmd/ateworker-microvm/restore.go +++ b/cmd/ateworker-microvm/restore.go @@ -20,6 +20,7 @@ import ( "context" "crypto/rand" "encoding/json" + "errors" "fmt" "log/slog" "os" @@ -257,6 +258,23 @@ func (s *AteWorkerService) restoreMemoryFidelity(ctx context.Context, p actorBoo if err != nil { return err } + // On failure, undo everything below that outlives the processes. This defer + // MUST stay registered first (right after PrepareEgress): defers run last + // in, first out, so it runs after the defers below have drained the + // background untar and stopped cloud-hypervisor and virtiofsd. Only then is + // it safe to sweep the sandbox state (which detaches the merged rootfs + // overlays over the upper dir) and remove the upper dir; moving a defer + // above this one would remove the upper under a live untar or overlay. + defer func() { + if retErr != nil { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + if cleanupErr := s.tunnel.Deactivate(cleanupCtx, p.attribution()); cleanupErr != nil { + slog.WarnContext(cleanupCtx, "Failed to deactivate actor networking after Restore failure", slog.Any("err", cleanupErr)) + } + s.cleanupFailedRestoreRootfs(cleanupCtx, actorUID, p.actorDirs) + } + }() s.cleanupSandboxState(ctx, actorUID) // Repoint the snapshot's per-actor paths to this actor's: its VMDir @@ -339,20 +357,6 @@ func (s *AteWorkerService) restoreMemoryFidelity(ctx context.Context, p actorBoo // fd-backed, so CH needs fresh tap FDs (net_fds) on restore. The caller // unhosts on failure, once the defers here have stopped the actor's // processes. - defer func() { - if retErr != nil { - cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) - defer cancel() - if cleanupErr := s.tunnel.Deactivate(cleanupCtx, p.attribution()); cleanupErr != nil { - slog.WarnContext(cleanupCtx, "Failed to deactivate actor networking after Restore failure", slog.Any("err", cleanupErr)) - } - // Detach any bundle rootfs overlays mounted by buildActorContainers - // before the failure, mirroring teardownActor's cleanup. - if err := imagecache.UnmountAllUnder(p.actorDirs.GetOciBundleDir()); err != nil { - slog.WarnContext(ctx, "Failed to unmount bundle rootfs overlays after Restore failure", slog.Any("err", err)) - } - } - }() netDevs, err := ch.SnapshotNetDevices(restoreDir) if err != nil { return fmt.Errorf("while reading snapshot net devices: %w", err) @@ -420,7 +424,7 @@ func (s *AteWorkerService) restoreMemoryFidelity(ctx context.Context, p actorBoo maybeDropStagedMemoryImage(ctx, restoreDir, memMode, preserveRestoreDir) dropActorSnapshotCacheAsync(restoreDir, memMode == ch.MemRestoreEager) if err := client.Resume(ctx); err != nil { - return fmt.Errorf("while resuming restored guest: %w", err) + return restoreGuestReadyError(ctx, "resuming restored guest", err) } tResume := time.Now() @@ -429,7 +433,7 @@ func (s *AteWorkerService) restoreMemoryFidelity(ctx context.Context, p actorBoo // fails the restore. A failing restore closes the connection on its way out. guestAC, err := dialAgentRetry(ctx, kata.VsockSocketPath(actorUID), 15*time.Second) if err != nil { - return fmt.Errorf("while dialing kata-agent after resume: %w", err) + return restoreGuestReadyError(ctx, "dialing kata-agent after resume", err) } defer func() { if retErr != nil { @@ -455,7 +459,7 @@ func (s *AteWorkerService) restoreMemoryFidelity(ctx context.Context, p actorBoo // CRNG on its own with no host round-trip, no per-restore RPC, and no post-Resume // race. At that point this agent-driven reseed can be dropped. if err := reseedGuestCRNG(ctx, guestAC); err != nil { - return fmt.Errorf("while reseeding guest CRNG: %w", err) + return restoreGuestReadyError(ctx, "reseeding guest CRNG", err) } tReseeded := time.Now() @@ -552,6 +556,60 @@ func maybeDropStagedMemoryImage(ctx context.Context, restoreDir, memMode string, dropStagedMemoryImage(ctx, restoreDir) } +// cleanupFailedRestoreRootfs removes what a failed memory restore assembled on +// the host for the actor's rootfs, mirroring teardownActor: it sweeps the +// sandbox state (detaching the merged rootfs overlays), removes the rootfs +// upper dir re-materialized from the snapshot, and detaches the bundle rootfs +// overlays mounted by buildActorContainers. The caller must have stopped the +// actor's VMM and virtiofsd and drained the upper dir's untar first. +func (s *AteWorkerService) cleanupFailedRestoreRootfs(ctx context.Context, actorUID string, actorDirs *ateworkerpb.ActorDirs) { + s.cleanupSandboxState(ctx, actorUID) + if err := os.RemoveAll(rootfsUpperDir(actorDirs)); err != nil { + slog.WarnContext(ctx, "Failed to remove rootfs upper dir after Restore failure", slog.String("actorUID", actorUID), slog.Any("err", err)) + } + if err := imagecache.UnmountAllUnder(actorDirs.GetOciBundleDir()); err != nil { + slog.WarnContext(ctx, "Failed to unmount bundle rootfs overlays after Restore failure", slog.String("actorUID", actorUID), slog.Any("err", err)) + } +} + +// restoreGuestReadyError classifies a failure to resume the restored micro-VM or +// reach its kata-agent before the workload serves any inbound traffic. Because +// the restoreMemoryFidelity and RestoreWorkload defers tear down the VMM, +// virtiofsd, rootfs mounts, network, and hosted-actor slot on the way out, the +// failed attempt leaves no worker-local state behind and is safe for the caller +// to retry (re-staging the snapshot if it was dropped above). When the caller's +// own context ended, wrap ctx.Err() so the server reports the context status +// instead. +// +// The guest did run, though: after Resume, it and the workload in it run while +// the agent is dialed and the CRNG reseeded (up to about 15s plus +// reseedTimeout). Nothing routes inbound traffic to it yet (not even the wakeup +// probe), and atunnel's egress proxy closes its outbound TCP connections until +// tunnel.Activate, but a background task in the guest can query the DNS relay +// or write to its filesystems. The retry re-extracts the rootfs upper and +// durable-dir volumes from the snapshot, whereas a CSI volume is outside the +// snapshot and keeps any writes made during that window while guest memory and +// its page cache resume from the snapshot; we accept that narrow window over +// crashing the actor on a transient host stall. When the reseed is what failed, +// the guest also ran on CRNG state shared with the snapshot's other clones. +// +// A guest found stopped while dialing the agent (errGuestStopped: +// cloud-hypervisor exited, e.g. a guest kernel panic after resume or a corrupt +// memory image) is not reported as Unavailable: the same snapshot would likely +// stop the guest again on every retry. coldBootActorRetrying likewise caps +// retries of that case, at coldBootAttempts. A stop that surfaces in Resume or +// the reseed instead is reported as Unavailable, and the caller's limit on +// consecutive retryable failures bounds it. +func restoreGuestReadyError(ctx context.Context, step string, err error) error { + if ctx.Err() != nil { + return fmt.Errorf("while %s: %v: %w", step, err, ctx.Err()) + } + if errors.Is(err, errGuestStopped) { + return fmt.Errorf("while %s: %w", step, err) + } + return apierror.Unavailable("while %s: %w", step, err) +} + // rewriteSnapshotPaths repoints the snapshot config.json's per-actor paths from // the source actor's to the restoring actor's. The file is CH's own: at // snapshot time it serializes the config the VM booted with, and at restore diff --git a/cmd/ateworker-microvm/restore_linux_test.go b/cmd/ateworker-microvm/restore_linux_test.go new file mode 100644 index 0000000000..b9d9d46075 --- /dev/null +++ b/cmd/ateworker-microvm/restore_linux_test.go @@ -0,0 +1,119 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/uuid" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/roottest" +) + +// failedRestoreRootfs lays out what a memory restore has assembled on the host +// by the time buildActorContainers returns: a rootfs upper dir re-materialized +// from the snapshot and each container's bundle rootfs. With overlays, the +// bundle rootfs is an overlay mount, as in production. +func failedRestoreRootfs(t *testing.T, actorUID string, overlays bool) *ateworkerpb.ActorDirs { + t.Helper() + t.Cleanup(func() { kata.CleanupSandboxState(context.Background(), actorUID) }) + dirs := &ateworkerpb.ActorDirs{RootDir: t.TempDir(), OciBundleDir: t.TempDir()} + if err := os.MkdirAll(filepath.Join(rootfsUpperDir(dirs), "app", "data"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(rootfsUpperDir(dirs), "app", "data", "state"), []byte("restored"), 0o644); err != nil { + t.Fatal(err) + } + containers := []*ateworkerpb.Container{{Name: "app", ContainerSpec: &ateworkerpb.ContainerSpec{Args: []string{"/app"}}}} + bundle := ociBundlePath(dirs, "app") + if err := os.MkdirAll(filepath.Join(bundle, "rootfs"), 0o755); err != nil { + t.Fatal(err) + } + if overlays { + layer := t.TempDir() + if err := os.MkdirAll(filepath.Join(layer, "fs", "etc"), 0o755); err != nil { + t.Fatal(err) + } + if err := imagecache.WriteSpec(bundle, &imagecache.OverlaySpec{Layers: []string{layer}}); err != nil { + t.Fatalf("WriteSpec: %v", err) + } + // Detach the overlays even if the cleanup under test leaves them, so + // t.TempDir can remove the bundle. + t.Cleanup(func() { _ = imagecache.UnmountAllUnder(dirs.GetOciBundleDir()) }) + } + if _, err := (&AteWorkerService{}).buildActorContainers(actorUID, dirs, containers); err != nil { + t.Fatalf("buildActorContainers: %v", err) + } + return dirs +} + +// mountsUnder lists the mount points at or below dir. +func mountsUnder(t *testing.T, dir string) []string { + t.Helper() + b, err := os.ReadFile("/proc/self/mountinfo") + if err != nil { + t.Fatal(err) + } + var mounts []string + for _, line := range strings.Split(string(b), "\n") { + fields := strings.Fields(line) + if len(fields) < 5 { + continue + } + if mp := fields[4]; mp == dir || strings.HasPrefix(mp, dir+"/") { + mounts = append(mounts, mp) + } + } + return mounts +} + +// cleanupFailedRestoreRootfs, which a memory restore that fails after +// buildActorContainers runs, leaves neither the rootfs upper dir nor any bundle +// rootfs overlay behind: a retried restore re-materializes the upper into the +// same dir and remounts the bundles. +func TestCleanupFailedRestoreRootfs(t *testing.T) { + t.Run("removes the rootfs upper dir", func(t *testing.T) { + actorUID := uuid.NewString() + dirs := failedRestoreRootfs(t, actorUID, false) + (&AteWorkerService{}).cleanupFailedRestoreRootfs(t.Context(), actorUID, dirs) + if _, err := os.Stat(rootfsUpperDir(dirs)); !os.IsNotExist(err) { + t.Errorf("rootfs upper dir after cleanup: stat = %v, want it removed", err) + } + }) + + t.Run("detaches the bundle rootfs overlays", func(t *testing.T) { + roottest.Require(t, "mounts the bundle rootfs overlay") + actorUID := uuid.NewString() + dirs := failedRestoreRootfs(t, actorUID, true) + if len(mountsUnder(t, dirs.GetOciBundleDir())) == 0 { + t.Fatal("buildActorContainers mounted no bundle rootfs overlay") + } + (&AteWorkerService{}).cleanupFailedRestoreRootfs(t.Context(), actorUID, dirs) + if got := mountsUnder(t, dirs.GetOciBundleDir()); len(got) != 0 { + t.Errorf("mounts under the bundle dir after cleanup = %v, want none", got) + } + if _, err := os.Stat(rootfsUpperDir(dirs)); !os.IsNotExist(err) { + t.Errorf("rootfs upper dir after cleanup: stat = %v, want it removed", err) + } + }) +} diff --git a/cmd/ateworker-microvm/restore_test.go b/cmd/ateworker-microvm/restore_test.go index 42caeb77df..d2641e2b74 100644 --- a/cmd/ateworker-microvm/restore_test.go +++ b/cmd/ateworker-microvm/restore_test.go @@ -21,6 +21,8 @@ import ( "context" "encoding/json" "errors" + "fmt" + "io/fs" "os" "path/filepath" "strings" @@ -547,6 +549,45 @@ func TestReseedGuestCRNG(t *testing.T) { }) } +func TestRestoreGuestReadyError(t *testing.T) { + // Child timeouts in dialAgentRetry and reseedGuestCRNG wrap a real + // context.DeadlineExceeded. + cause := fmt.Errorf("agent ReseedRandomDev: %w", context.DeadlineExceeded) + + t.Run("live caller context reports Unavailable even when cause wraps child DeadlineExceeded", func(t *testing.T) { + err := restoreGuestReadyError(t.Context(), "reseeding guest CRNG", cause) + if got := apierror.Code(err); got != codes.Unavailable { + t.Errorf("apierror.Code(err) = %v, want %v (err: %v)", got, codes.Unavailable, err) + } + if !errors.Is(err, cause) { + t.Errorf("err = %v, want it to wrap %v", err, cause) + } + }) + + t.Run("canceled caller context reports Canceled without child DeadlineExceeded shadowing it", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + err := restoreGuestReadyError(ctx, "dialing kata-agent after resume", cause) + if got := apierror.Code(err); got != codes.Canceled { + t.Errorf("apierror.Code(err) = %v, want %v (err: %v)", got, codes.Canceled, err) + } + if !errors.Is(err, context.Canceled) { + t.Errorf("err = %v, want it to wrap context.Canceled", err) + } + }) + + t.Run("stopped guest reports Internal, not Unavailable", func(t *testing.T) { + stopped := fmt.Errorf("%w (cloud-hypervisor removed %q): %w", errGuestStopped, "/run/vsock.sock", fs.ErrNotExist) + err := restoreGuestReadyError(t.Context(), "dialing kata-agent after resume", stopped) + if got := apierror.Code(err); got != codes.Internal { + t.Errorf("apierror.Code(err) = %v, want %v (err: %v)", got, codes.Internal, err) + } + if !errors.Is(err, errGuestStopped) { + t.Errorf("err = %v, want it to wrap errGuestStopped", err) + } + }) +} + func TestRPCsRejectUntrustedRuntimeAssetPaths(t *testing.T) { s := &AteWorkerService{} ctx := context.Background() diff --git a/internal/apierror/apierror.go b/internal/apierror/apierror.go index 300734260b..df0ef318fb 100644 --- a/internal/apierror/apierror.go +++ b/internal/apierror/apierror.go @@ -98,8 +98,9 @@ func Unimplemented(format string, args ...any) error { return newf(codes.Unimplemented, format, args...) } -// Unavailable reports that the request was not acted on, so retrying it, here -// or elsewhere, is safe. Never use it for a failure partway through. +// Unavailable reports that the request was not acted on, or that the attempt +// was rolled back so retrying it, here or elsewhere, is safe. Never use it for +// a failure that leaves partial state behind. func Unavailable(format string, args ...any) error { return newf(codes.Unavailable, format, args...) } From e4ae7a4efe2904086337fdec7246425d1817542d Mon Sep 17 00:00:00 2001 From: Sairaj Pokale Date: Wed, 21 Oct 2026 22:46:49 +0000 Subject: [PATCH 02/12] atelet: Clean up whole actor directory on detach (#2871) ## What this PR does and why we need it ? When detaching an actor (either via background timer detach or explicit `DetachActor` RPC), `atelet` previously only pruned entries inside `/var/lib/ate/snapshots//local-checkpoint/`, leaving the actor's snapshot directory (`/var/lib/ate/snapshots/`) and actor directory (`/var/lib/ate/actors/`) on the node indefinitely. Because `sandbox-assets/` holds hard links into the sandbox asset cache (`st_nlink > 1`), leaving the actor directory also permanently pinned sandbox binaries in the cache after detach. ### Changes - Move `ResetDir`, `ResetActorDirs`, `ResetActorDirsKeepingVolumes`, `RemoveActorFilesExceptState`, and `RemoveActorDirs` into `cmd/atelet/internal/actordirs` so `Atelet` and `snapshottracker.Tracker` share directory cleanup logic (preserving populated external volume mount directories, removing `ateletpath.ActorSnapshotPath`, and handling read-only bundle upper directories via `imagecache.RemoveAllWritable`). - Update `Tracker.finishDetach` to call `actordirs.RemoveActorFilesExceptState` (removing `/var/lib/ate/snapshots/` and all entries under `/var/lib/ate/actors/` except `snapshot-state.json`) while `opDetach` is active, and mark a detach report due (`markReportDueLocked`) for both background and foreground `Detach` calls. - Keep `snapshot-state.json` (with `ReportDueSince`) on disk until the detach report finishes so a retried `DetachActor` RPC still sees `LatestDurable` and a pending detach report survives an `atelet` restart. - Remove `snapshot-state.json` and the empty `/var/lib/ate/actors/` directory (via `removeDetachedActorState` in `finishReportDueLocked`) once `!Started && LatestLocal == nil && Uploading == nil` and the control plane acknowledges the detach report or refuses it with `NotFound`. On give-up or any other refusal (`Unimplemented`, `FailedPrecondition`, `InvalidArgument`), `snapshot-state.json` stays on disk with `ReportDueSince` cleared so `DetachActor` keeps returning `LatestDurable`. - Require `--snapshot-report-workers >= 1` in `validateSnapshotWorkerFlags` so background detach reports are always sent. - In `cleanUpAfterCheckpoint`, call `actordirs.RemoveActorFilesExceptState` instead of `actordirs.ResetActorDirs` when no local snapshot is kept on the node, so empty subdirectories are not recreated. - Document in `atelet.proto` (`ActorSnapshotState.latest_durable` and `DetachActorResponse.state`) that `DetachActor` queues a background detach report and forgets the actor once acknowledged. > **Merge order**: Land after #2797 so `Terminate` tolerates a missing `sandbox-assets.json` after `cleanUpAfterCheckpoint` calls `RemoveActorFilesExceptState`. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- cmd/atelet/internal/actordirs/actordirs.go | 163 ++++++++++ cmd/atelet/internal/ateletpath/ateletpath.go | 8 +- cmd/atelet/internal/snapshottracker/actor.go | 10 +- cmd/atelet/internal/snapshottracker/detach.go | 55 ++-- .../internal/snapshottracker/detach_test.go | 178 +++++++++- .../internal/snapshottracker/lifecycle.go | 9 +- .../internal/snapshottracker/metrics.go | 8 +- .../internal/snapshottracker/metrics_test.go | 9 +- cmd/atelet/internal/snapshottracker/report.go | 46 ++- .../internal/snapshottracker/report_test.go | 305 ++++++++++++------ cmd/atelet/internal/snapshottracker/state.go | 13 + .../internal/snapshottracker/tracker.go | 17 +- cmd/atelet/lifecycle_test.go | 93 +++++- cmd/atelet/main.go | 149 ++------- cmd/atelet/main_test.go | 55 ++-- cmd/atelet/oci.go | 5 +- cmd/atelet/sandbox_asset_cache.go | 3 +- cmd/atelet/volumes_test.go | 23 +- docs/metrics/registry/metrics.yaml | 11 +- docs/observability.md | 2 +- internal/proto/ateletpb/atelet.pb.go | 10 +- internal/proto/ateletpb/atelet.proto | 24 +- internal/proto/ateletpb/atelet_grpc.pb.go | 28 +- 23 files changed, 862 insertions(+), 362 deletions(-) create mode 100644 cmd/atelet/internal/actordirs/actordirs.go diff --git a/cmd/atelet/internal/actordirs/actordirs.go b/cmd/atelet/internal/actordirs/actordirs.go new file mode 100644 index 0000000000..d005817f1a --- /dev/null +++ b/cmd/atelet/internal/actordirs/actordirs.go @@ -0,0 +1,163 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package actordirs manages each actor's on-node directory tree. +package actordirs + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/internal/imagecache" +) + +// ResetDir empties dir with remove and recreates it with mode. what names the +// directory in errors. +func ResetDir(what, dir string, remove func(string) error, mode os.FileMode) error { + if err := remove(dir); err != nil { + return fmt.Errorf("while deleting %s: %w", what, err) + } + if err := os.MkdirAll(dir, mode); err != nil { + return fmt.Errorf("while creating %s: %w", what, err) + } + return nil +} + +// removeEmptyVolumeDirs removes each empty mountpoint directory under the +// actor's external volumes directory, refusing to delete through a populated +// directory in case an unmount failed. +func removeEmptyVolumeDirs(actorUID string) error { + volumesDir := ateletpath.ExternalVolumeMountsDir(actorUID) + entries, err := os.ReadDir(volumesDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("while reading volumes dir: %w", err) + } + for _, entry := range entries { + volPath := filepath.Join(volumesDir, entry.Name()) + if err := os.Remove(volPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("while removing volume dir: %w", err) + } + } + return nil +} + +// ResetActorDirs empties the actor's directories and leaves them in place for +// its next activation. Use RemoveActorDirs when the actor will not come back. +func ResetActorDirs(actorUID string) error { + if err := ResetActorDirsKeepingVolumes(actorUID); err != nil { + return err + } + // Do not call RemoveAll on volume directories in case the unmount failed. + // We do not want to delete mount content. + return removeEmptyVolumeDirs(actorUID) +} + +// ResetActorDirsKeepingVolumes is ResetActorDirs for Run and Restore, which +// leave the external volume mount points alone: a failed earlier attempt can +// leave a volume published there, and publishing it again is a no-op. +func ResetActorDirsKeepingVolumes(actorUID string) error { + // Explicitly leave runsc logs dir untouched. + + // RemoveAllWritable, not os.RemoveAll: the bundle's upper dir can hold + // copied-up actor-image directories keeping the image's (possibly + // read-only) modes, which atelet can't remove as plain root without first + // making them writable. (The rootfs itself is just an empty mountpoint + // here: the overlay is mounted in the worker pod's mount namespace, not + // atelet's, and is detached by worker at teardown.) + if err := ResetDir("bundle dir", ateletpath.OCIBundleDir(actorUID), imagecache.RemoveAllWritable, 0o700); err != nil { + return err + } + if err := ResetDir("checkpoint-state dir", ateletpath.CheckpointStateDir(actorUID), os.RemoveAll, 0o700); err != nil { + return err + } + if err := ResetDir("restore-state dir", ateletpath.RestoreStateDir(actorUID), os.RemoveAll, 0o700); err != nil { + return err + } + if err := ResetDir("durable-dir volumes mount dir", ateletpath.DurableDirVolumeMountsDir(actorUID), os.RemoveAll, 0o755); err != nil { + return err + } + // World-readable (0o755): bind-mounted read-only into the actor, whose + // workload reads it through the gofer. + if err := ResetDir("system-info volume roots dir", ateletpath.SystemInfoVolumeRootsDir(actorUID), os.RemoveAll, 0o755); err != nil { + return err + } + if err := os.MkdirAll(ateletpath.ExternalVolumeMountsDir(actorUID), 0o755); err != nil { + return fmt.Errorf("while creating volumes dir: %w", err) + } + return nil +} + +// RemoveActorFilesExceptState removes the actor's snapshot directory tree and +// all files and subdirectories under the actor's directory except +// snapshot-state.json, refusing to proceed while an external volume mount +// directory is still populated. +func RemoveActorFilesExceptState(actorUID string) error { + if err := removeEmptyVolumeDirs(actorUID); err != nil { + return err + } + volumesDir := ateletpath.ExternalVolumeMountsDir(actorUID) + if err := os.Remove(volumesDir); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("while removing volumes dir: %w", err) + } + if err := os.RemoveAll(ateletpath.ActorSnapshotPath(actorUID)); err != nil { + return fmt.Errorf("while deleting actor snapshot dir: %w", err) + } + actorDir := ateletpath.ActorPath(actorUID) + stateFile := filepath.Base(ateletpath.ActorSnapshotStateFile(actorUID)) + entries, err := os.ReadDir(actorDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("while reading actor dir: %w", err) + } + for _, entry := range entries { + if entry.Name() == stateFile { + continue + } + if err := imagecache.RemoveAllWritable(filepath.Join(actorDir, entry.Name())); err != nil { + return fmt.Errorf("while deleting actor dir: %w", err) + } + } + return nil +} + +// RemoveActorDirs reclaims both of the actor's directory trees, roots +// included: nothing else on the node deletes them, and no later activation +// will look there. +// +// It refuses to proceed while a volume directory is still populated, so a +// failed unmount cannot become a deletion of the mount's contents, and removes +// a bundle upper dir carrying an image's read-only modes without recreating +// directories. Terminate calls Forget first, which has already deleted +// snapshot-state.json; after a crash here, Start finds no state file, so only a +// retried Terminate finishes the cleanup. +func RemoveActorDirs(actorUID string) error { + if err := RemoveActorFilesExceptState(actorUID); err != nil { + return err + } + if err := os.Remove(ateletpath.ActorSnapshotStateFile(actorUID)); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("while deleting the actor's snapshot state: %w", err) + } + if err := os.Remove(ateletpath.ActorPath(actorUID)); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("while deleting actor dir: %w", err) + } + return nil +} diff --git a/cmd/atelet/internal/ateletpath/ateletpath.go b/cmd/atelet/internal/ateletpath/ateletpath.go index 55d1b4450f..d38ad6241e 100644 --- a/cmd/atelet/internal/ateletpath/ateletpath.go +++ b/cmd/atelet/internal/ateletpath/ateletpath.go @@ -87,8 +87,8 @@ func ActorSnapshotPath(actorUID string) string { // binaries (class + content-addressed asset set, for this node's architecture) // the actor is currently running. It is written at Run/Restore and read at // Checkpoint (when the request no longer carries the sandbox config). It lives -// directly under ActorPath — NOT under a subdir wiped by atelet's -// resetActorDirs — so it survives between Run and a later Checkpoint. +// directly under ActorPath — NOT under a subdir wiped by +// actordirs.ResetActorDirs — so it survives between Run and a later Checkpoint. func ActorSandboxAssetsFile(actorUID string) string { return filepath.Join( ActorPath(actorUID), @@ -99,7 +99,7 @@ func ActorSandboxAssetsFile(actorUID string) string { // ActorSnapshotStateFile is the per-actor file where atelet keeps what it // tracks of the actor's snapshots: its latest local and durable snapshots, so // a restarted atelet still knows them. Like ActorSandboxAssetsFile it lives -// directly under ActorPath, outside the subdirs resetActorDirs wipes. +// directly under ActorPath, outside the subdirs actordirs.ResetActorDirs wipes. func ActorSnapshotStateFile(actorUID string) string { return filepath.Join( ActorPath(actorUID), @@ -118,7 +118,7 @@ func SandboxAssetsDir(actorUID string) string { // SandboxAssetsStagingDir is where Run stages an activation's assets before // moving them into SandboxAssetsDir, so a failed fetch leaves whatever a // live activation has there untouched. Not among the directories -// resetActorDirs empties. +// actordirs.ResetActorDirs empties. func SandboxAssetsStagingDir(actorUID string) string { return filepath.Join( ActorPath(actorUID), diff --git a/cmd/atelet/internal/snapshottracker/actor.go b/cmd/atelet/internal/snapshottracker/actor.go index 9365f3ee74..6f4914e9ba 100644 --- a/cmd/atelet/internal/snapshottracker/actor.go +++ b/cmd/atelet/internal/snapshottracker/actor.go @@ -177,11 +177,11 @@ type actorSnapshotsRecord struct { // (CommitLocal or CommitDurable). While it is unset, the actor's // directories and volumes are still that Checkpoint's to clean up. Started bool `json:"started,omitempty"` - // ReportDueSince is when the latest background change atelet owes a - // report of (a new LatestDurable, or local snapshots cleared by a - // detach) finished. Zero once no report is due: the control plane - // acknowledged or refused it, a commit returned the state, or atelet gave - // up after reportQueue.giveUpAfter. + // ReportDueSince is when the latest change atelet owes a report of (a + // new LatestDurable, or local snapshots cleared by a detach) finished. + // Zero once no report is due: the control plane acknowledged or refused + // it, a commit returned the state, or atelet gave up after + // reportQueue.giveUpAfter. ReportDueSince time.Time `json:"reportDueSince,omitzero"` } diff --git a/cmd/atelet/internal/snapshottracker/detach.go b/cmd/atelet/internal/snapshottracker/detach.go index 378311162e..0817458a41 100644 --- a/cmd/atelet/internal/snapshottracker/detach.go +++ b/cmd/atelet/internal/snapshottracker/detach.go @@ -17,6 +17,7 @@ package snapshottracker import ( "context" + "go.ate.dev/substrate/cmd/atelet/internal/actordirs" "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/proto/ateletpb" ) @@ -38,14 +39,16 @@ func (t *Tracker) beginDetachLocked(ctx context.Context, a *actorSnapshots, back } // Detach makes the actor's latest local snapshot durable, then deletes the -// actor's local snapshots. It is a no-op when atelet holds no local snapshot -// for the actor. It fails with Aborted once a Run or Restore of the actor has -// begun since its latest local snapshot, and with FailedPrecondition when no -// detach can make the snapshot durable. It starts once the actor's operation -// in progress ends, and runs on the tracker's context, not ctx. If ctx ends -// first, Detach returns ctx's error and the detach runs on, recorded by its -// report. After Wait begins, Detach fails with Unavailable. Callers serialize -// it with the actor's other lifecycle RPCs (see Tracker). +// actor's local snapshots and its files on this node (keeping +// snapshot-state.json until the detach is reported). It is a no-op when atelet +// holds no local snapshot for the actor. It fails with Aborted once a Run or +// Restore of the actor has begun since its latest local snapshot, and with +// FailedPrecondition when no detach can make the snapshot durable. It starts +// once the actor's operation in progress ends, and runs on the tracker's +// context, not ctx. If ctx ends first, Detach returns ctx's error and the +// detach runs on, recorded by its report. After Wait begins, Detach fails with +// Unavailable. Callers serialize it with the actor's other lifecycle RPCs (see +// Tracker). func (t *Tracker) Detach(ctx context.Context, actorUID string) (*ateletpb.ActorSnapshotState, error) { if t.store == nil { return nil, apierror.Unimplemented("atelet has no snapshot store to detach actors with") @@ -62,10 +65,22 @@ func (t *Tracker) Detach(ctx context.Context, actorUID string) (*ateletpb.ActorS if a == nil || a.LatestLocal == nil { // No local snapshot is stored on this node, so there is no work to be // done. A report already due stays due, in case the caller doesn't - // record this state. + // record this state, and so a retried DetachActor call still sees + // LatestDurable until the report is acknowledged. When the actor has + // not started and no report is due (as after a report gave up or was + // refused with an error other than NotFound), make a report due again + // so the next acknowledgement removes snapshot-state.json and the + // actor directory. st := &ateletpb.ActorSnapshotState{} if a != nil { st = t.stateLocked(a) + if !a.Started && !a.reportDue() && a.LatestDurable != nil { + t.addLocked(a) + t.markReportDueLocked(a) + t.saveLocked(a) + t.queueReportLocked(a) + t.requeueLocked(a) + } } t.unlock(actorUID) return st, nil @@ -112,8 +127,11 @@ func (t *Tracker) detachCallError(ctx context.Context, err error) error { // finishDetach runs op, a detach of the actor's latest local snapshot: it // uploads the snapshot unless already durable, then deletes the actor's local -// snapshots. background reports whether a worker runs it, rather than a Detach -// call. Either way it makes a report of what it changed due. +// snapshots and its other files on this node except snapshot-state.json. +// background reports whether a worker runs it, rather than a Detach call. +// Either way it makes a report of what it changed due, so snapshot-state.json +// and the actor's directory are removed once the control plane acknowledges +// the detach (or refuses it with NotFound). func (t *Tracker) finishDetach(ctx context.Context, actorUID string, op *actorOp, alreadyDurable, background bool) (*ateletpb.ActorSnapshotState, error) { uploaded := false target := op.target @@ -134,12 +152,13 @@ func (t *Tracker) finishDetach(ctx context.Context, actorUID string, op *actorOp } // The upload is recorded before anything is deleted, and LatestLocal is - // cleared only once its files are gone. After a crash or a failed prune in - // between, the detach is retried, finds the snapshot durable, and only - // prunes. Until then LatestLocal can name a half-deleted copy, which is - // safe because the snapshot is durable: Restore reads snapshot.object - // when the local copy is unusable. The prune runs with op in progress - // (before op.done is closed), so Run, Restore, and Checkpoint wait for it. + // cleared only once its files and the actor's other files are gone. After + // a crash or a failed cleanup in between, the detach is retried, finds the + // snapshot durable, and only cleans up. Until then LatestLocal can name a + // half-deleted copy, which is safe because the snapshot is durable: + // Restore reads snapshot.object when the local copy is unusable. The + // cleanup runs with op in progress (before op.done is closed), so Run, + // Restore, and Checkpoint wait for it. recordDetach := err == nil if recordDetach { detachStart = t.clock.Now() @@ -148,7 +167,7 @@ func (t *Tracker) finishDetach(ctx context.Context, actorUID string, op *actorOp err = ctx.Err() } if err == nil { - err = PruneLocalCheckpoints(ctx, actorUID) + err = actordirs.RemoveActorFilesExceptState(actorUID) } t.updateOp(actorUID, func(a *actorSnapshots) { switch { diff --git a/cmd/atelet/internal/snapshottracker/detach_test.go b/cmd/atelet/internal/snapshottracker/detach_test.go index f9cd090aa8..5b073f86d5 100644 --- a/cmd/atelet/internal/snapshottracker/detach_test.go +++ b/cmd/atelet/internal/snapshottracker/detach_test.go @@ -19,10 +19,12 @@ import ( "errors" "log/slog" "os" + "path/filepath" "slices" "testing" "time" + "go.ate.dev/substrate/cmd/atelet/internal/actordirs" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/proto/ateletpb" @@ -611,33 +613,63 @@ func TestTrackerFailedDetach(t *testing.T) { }) } -// A detach whose prune fails keeps LatestLocal, now durable, and is retried: -// the retry finds the snapshot durable and only prunes. +// A detach whose cleanup fails (e.g. because an external volume mount directory +// is still populated) keeps LatestLocal and its local snapshot on disk, records +// the snapshot as durable, and is retried: the retry finds the snapshot durable +// and only cleans up. func TestTrackerDetachRetriesFailedPrune(t *testing.T) { store := newFakeSnapshotStore() - tr := newTestTracker(t, store) - tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) - // A file where the local checkpoint directory belongs fails the prune. - if err := os.WriteFile(ateletpath.LocalCheckpointsDir(trackedActorUID), nil, 0o600); err != nil { + tr := newReportingTracker(t, store, newFakeReporter()) + localSnap := ateletpath.LocalSnapshotDir(trackedActorUID, "s1") + if err := os.MkdirAll(localSnap, 0o700); err != nil { + t.Fatal(err) + } + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + // A populated volume mount directory fails RemoveActorFilesExceptState + // before it deletes the local snapshot tree. + stillMounted := filepath.Join(ateletpath.ExternalVolumeHostPath(trackedActorUID, "data"), "payload") + if err := os.MkdirAll(filepath.Dir(stillMounted), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(stillMounted, []byte("volume contents"), 0o600); err != nil { t.Fatal(err) } if _, err := tr.Detach(t.Context(), trackedActorUID); err == nil { - t.Fatal("detach succeeded, want the prune's error") + t.Fatal("detach succeeded, want RemoveActorFilesExceptState error") } if st := tr.State(trackedActorUID); uuidOf(st.GetLatestLocal()) != "s1" || uuidOf(st.GetLatestDurable()) != "s1" { t.Errorf("state = %v, want s1 still local and now durable", st) } + if _, err := os.Stat(localSnap); err != nil { + t.Errorf("local snapshot deleted on failed detach: %v", err) + } + if _, err := os.Stat(stillMounted); err != nil { + t.Errorf("stillMounted was deleted on failed detach: %v", err) + } + if _, err := os.Stat(ateletpath.OCIBundleDir(trackedActorUID)); !os.IsNotExist(err) { + t.Errorf("bundle dir was created during a failed detach cleanup: %v", err) + } - if err := os.Remove(ateletpath.LocalCheckpointsDir(trackedActorUID)); err != nil { + if err := os.Remove(stillMounted); err != nil { t.Fatal(err) } detachNow(t, tr) if got, want := store.uploads(), []string{"s1"}; !slices.Equal(got, want) { - t.Errorf("uploads = %v, want %v, as the retry only prunes", got, want) + t.Errorf("uploads = %v, want %v, as the retry only cleans up", got, want) } if got := tr.State(trackedActorUID).GetLatestLocal(); got != nil { t.Errorf("latest_local = %v after the retried detach, want unset", got) } + processNextReport(t, tr) + if inMemory(tr) { + t.Error("actor still in memory after retried detach") + } + if _, err := os.Stat(ateletpath.ActorPath(trackedActorUID)); !os.IsNotExist(err) { + t.Errorf("actor dir %s still exists: %v", ateletpath.ActorPath(trackedActorUID), err) + } + if _, err := os.Stat(ateletpath.ActorSnapshotPath(trackedActorUID)); !os.IsNotExist(err) { + t.Errorf("actor snapshot dir %s still exists: %v", ateletpath.ActorSnapshotPath(trackedActorUID), err) + } } // Detach fails on a tracker without a store, rather than uploading through it. @@ -699,12 +731,16 @@ func TestTrackerLiftsBackoffWithDetach(t *testing.T) { s1 := testRef("s1", 1, tr.clock.Now()) tr.CommitDurable(trackedActorUID, s1) tr.CommitLocal(trackedActorUID, s1) - // A file where the local checkpoint directory belongs fails the prune. - if err := os.WriteFile(ateletpath.LocalCheckpointsDir(trackedActorUID), nil, 0o600); err != nil { + // A populated volume mount directory fails RemoveActorFilesExceptState. + stillMounted := filepath.Join(ateletpath.ExternalVolumeHostPath(trackedActorUID, "data"), "payload") + if err := os.MkdirAll(filepath.Dir(stillMounted), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(stillMounted, nil, 0o600); err != nil { t.Fatal(err) } if _, err := tr.Detach(t.Context(), trackedActorUID); err == nil { - t.Fatal("detach succeeded, want the prune's error") + t.Fatal("detach succeeded, want the cleanup's error") } if !inMemory(tr) { t.Fatal("entry not in memory after a failed detach, want it kept for its backoff") @@ -1034,6 +1070,124 @@ func TestTrackerDetachActsForTheActor(t *testing.T) { }) } +// TestTrackerDetachCleansUpActorDirectory verifies that both explicit Detach and +// background timer detachNow remove the actor's files immediately, keep +// snapshot-state.json while the detach report is pending, and remove +// snapshot-state.json and the actor directory once the detach report is +// acknowledged. +func TestTrackerDetachCleansUpActorDirectory(t *testing.T) { + populateActorDirs := func(t *testing.T, actorUID string) { + t.Helper() + // Sandbox assets dir with dummy asset binary. + assetsDir := ateletpath.SandboxAssetsDir(actorUID) + if err := os.MkdirAll(assetsDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(assetsDir, "dummy-binary"), []byte("bin"), 0o755); err != nil { + t.Fatal(err) + } + // Sandbox assets JSON record. + if err := os.WriteFile(ateletpath.ActorSandboxAssetsFile(actorUID), []byte(`{"version":1}`), 0o600); err != nil { + t.Fatal(err) + } + // Runtime subdirs (bundles, durable-dir, etc.). + if err := actordirs.ResetActorDirs(actorUID); err != nil { + t.Fatal(err) + } + } + + assertActorFilesCleanedExceptState := func(t *testing.T, actorUID string) { + t.Helper() + if _, err := os.Stat(ateletpath.ActorSnapshotPath(actorUID)); !os.IsNotExist(err) { + t.Errorf("actor snapshot dir still exists before report: %v", err) + } + if _, err := os.Stat(ateletpath.SandboxAssetsDir(actorUID)); !os.IsNotExist(err) { + t.Errorf("sandbox assets dir still exists before report: %v", err) + } + if _, err := os.Stat(ateletpath.ActorSandboxAssetsFile(actorUID)); !os.IsNotExist(err) { + t.Errorf("sandbox assets file still exists before report: %v", err) + } + if !reportDueOnDisk() { + t.Fatal("snapshot-state.json not kept with report due before report is sent") + } + } + + assertActorDirCleanedUp := func(t *testing.T, actorUID string) { + t.Helper() + actorDir := ateletpath.ActorPath(actorUID) + if _, err := os.Stat(actorDir); !os.IsNotExist(err) { + t.Errorf("actor dir %s still exists: %v", actorDir, err) + } + snapshotDir := ateletpath.ActorSnapshotPath(actorUID) + if _, err := os.Stat(snapshotDir); !os.IsNotExist(err) { + t.Errorf("actor snapshot dir %s still exists: %v", snapshotDir, err) + } + if _, err := os.Stat(ateletpath.SandboxAssetsDir(actorUID)); !os.IsNotExist(err) { + t.Errorf("sandbox assets dir still exists: %v", err) + } + if _, err := os.Stat(ateletpath.ActorSandboxAssetsFile(actorUID)); !os.IsNotExist(err) { + t.Errorf("sandbox assets file still exists: %v", err) + } + if _, err := os.Stat(ateletpath.ActorSnapshotStateFile(actorUID)); !os.IsNotExist(err) { + t.Errorf("snapshot state file still exists: %v", err) + } + } + + t.Run("explicit Detach", func(t *testing.T) { + store := newFakeSnapshotStore() + tr := newReportingTracker(t, store, newFakeReporter()) + populateActorDirs(t, trackedActorUID) + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + + st, err := tr.Detach(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("Detach: %v", err) + } + if st.GetLatestLocal() != nil || uuidOf(st.GetLatestDurable()) != "s1" { + t.Errorf("state = %v, want s1 durable and nothing local", st) + } + assertActorFilesCleanedExceptState(t, trackedActorUID) + if inMemory(tr) { + t.Error("actor still in memory while detach report is queued on disk") + } + + // A retried DetachActor call before the report is acknowledged still + // returns LatestDurable from snapshot-state.json. + st, err = tr.Detach(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("second Detach: %v", err) + } + if st.GetLatestLocal() != nil || uuidOf(st.GetLatestDurable()) != "s1" { + t.Errorf("second Detach state = %v, want s1 durable and nothing local", st) + } + + processNextReport(t, tr) + if inMemory(tr) { + t.Error("actor still in memory after detach report is acknowledged") + } + assertActorDirCleanedUp(t, trackedActorUID) + }) + + t.Run("background timer detachNow", func(t *testing.T) { + store := newFakeSnapshotStore() + tr := newReportingTracker(t, store, newFakeReporter()) + populateActorDirs(t, trackedActorUID) + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + + detachNow(t, tr) + assertActorFilesCleanedExceptState(t, trackedActorUID) + if inMemory(tr) { + t.Error("actor still in memory while detach report is queued on disk") + } + + processNextReport(t, tr) + if inMemory(tr) { + t.Error("actor still in memory after detach report is acknowledged") + } + assertActorDirCleanedUp(t, trackedActorUID) + }) +} + // A detach whose upload runs past the upload timeout fails and keeps the local // snapshot to retry, whether a worker or a Detach call runs it. func TestTrackerDetachUploadTimeout(t *testing.T) { diff --git a/cmd/atelet/internal/snapshottracker/lifecycle.go b/cmd/atelet/internal/snapshottracker/lifecycle.go index 6965785486..8e9284002b 100644 --- a/cmd/atelet/internal/snapshottracker/lifecycle.go +++ b/cmd/atelet/internal/snapshottracker/lifecycle.go @@ -97,9 +97,10 @@ func (t *Tracker) CommitDurable(actorUID string, snap SnapshotRef) *ateletpb.Act } // CommittedCheckpointState returns the actor's state for a Checkpoint whose -// snapshot atelet committed already, and cancels any pending background -// report: a Checkpoint returns the state to the control plane, including a -// detach, in either upload mode. +// snapshot atelet committed already, and cancels any pending background upload +// report while a local snapshot is kept. Once LatestLocal is nil, a pending +// detach report stays due so the actor's state file and directory are removed +// once the detach report is acknowledged. func (t *Tracker) CommittedCheckpointState(actorUID string) *ateletpb.ActorSnapshotState { t.lock(actorUID) defer t.unlock(actorUID) @@ -107,7 +108,7 @@ func (t *Tracker) CommittedCheckpointState(actorUID string) *ateletpb.ActorSnaps if a == nil { return &ateletpb.ActorSnapshotState{} } - if a.reportDue() { + if a.LatestLocal != nil && a.reportDue() { t.clearReportDueLocked(a) } return t.stateLocked(a) diff --git a/cmd/atelet/internal/snapshottracker/metrics.go b/cmd/atelet/internal/snapshottracker/metrics.go index 736003fb15..f0f49557d5 100644 --- a/cmd/atelet/internal/snapshottracker/metrics.go +++ b/cmd/atelet/internal/snapshottracker/metrics.go @@ -77,9 +77,9 @@ func classifyOverdueBand(overdue time.Duration) overdueBand { // before its timeout, which is at least --snapshot-upload-timeout. var uploadDurationBuckets = []float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 60, 120, 300, 600} -// detachDurationBuckets cover deleting an actor's local snapshots during a -// background detach, which takes milliseconds to seconds. The upload a detach -// runs first is recorded on the upload histogram. +// detachDurationBuckets cover deleting an actor's local snapshots and node +// files during a background detach, which takes milliseconds to seconds. The +// upload a detach runs first is recorded on the upload histogram. var detachDurationBuckets = []float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 60} // overdueBuckets cover how late background work starts: within 100ms of its @@ -117,7 +117,7 @@ func NewInstruments(meter metric.Meter) (*Instruments, error) { detachDuration, err := meter.Float64Histogram( detachDurationMetric, metric.WithUnit("s"), - metric.WithDescription("Duration of deleting an actor's local snapshots during a background detach on atelet, excluding any snapshot upload."), + metric.WithDescription("Duration of deleting an actor's local snapshots and node files during a background detach on atelet, excluding any snapshot upload."), metric.WithExplicitBucketBoundaries(detachDurationBuckets...), ) if err != nil { diff --git a/cmd/atelet/internal/snapshottracker/metrics_test.go b/cmd/atelet/internal/snapshottracker/metrics_test.go index 3bbc97cbaa..e0b65d303e 100644 --- a/cmd/atelet/internal/snapshottracker/metrics_test.go +++ b/cmd/atelet/internal/snapshottracker/metrics_test.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "os" + "path/filepath" "slices" "testing" "time" @@ -407,8 +408,12 @@ func TestDetachDurationOutcomes(t *testing.T) { ref := testRef("s1", 1, tr.clock.Now()) tr.CommitDurable(trackedActorUID, ref) tr.CommitLocal(trackedActorUID, ref) - // A file where the local checkpoint directory belongs fails the prune. - if err := os.WriteFile(ateletpath.LocalCheckpointsDir(trackedActorUID), nil, 0o600); err != nil { + // A populated volume mount directory fails RemoveActorFilesExceptState. + stillMounted := filepath.Join(ateletpath.ExternalVolumeHostPath(trackedActorUID, "data"), "payload") + if err := os.MkdirAll(filepath.Dir(stillMounted), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(stillMounted, nil, 0o600); err != nil { t.Fatal(err) } stepTo(tr, detachDueOf(tr, trackedActorUID)) diff --git a/cmd/atelet/internal/snapshottracker/report.go b/cmd/atelet/internal/snapshottracker/report.go index 7831344872..ea669fb77c 100644 --- a/cmd/atelet/internal/snapshottracker/report.go +++ b/cmd/atelet/internal/snapshottracker/report.go @@ -68,7 +68,7 @@ type Reporter interface { // changes to the control plane. type reportQueue struct { // reporter tells the control plane what atelet holds of an actor's - // snapshots. Nil without WithReporter, or when workers is 0. + // snapshots. Nil without WithReporter. reporter Reporter // workers is how many report workers drain queue: DefaultReportWorkers // unless WithReportWorkers sets it. @@ -102,7 +102,7 @@ func newReportQueue(clk clock.WithTicker) reportQueue { } } -// markReportDueLocked records that a background upload or detach just made a +// markReportDueLocked records that a background upload or a detach just made a // change to report, starting a fresh giveUpAfter window and retry backoff. func (t *Tracker) markReportDueLocked(a *actorSnapshots) { a.ReportDueSince = t.clock.Now() @@ -182,9 +182,9 @@ func (t *Tracker) waitReportQueue(delay time.Duration) bool { // returns how long the report queue should pause before the next report. A // report the control plane acknowledges or refuses is no longer due, unless a // background change was made while it ran. Any other failure is retried after -// a backoff, until a commit cancels it or giveUpAfter has passed -// since the report became due; a failure other than Aborted also backs off -// the whole report queue. +// a backoff, until a commit cancels it or giveUpAfter has passed since the +// report became due; a failure other than Aborted also backs off the whole +// report queue. func (t *Tracker) report(actorUID string) time.Duration { actor, sent, dueSince, ok := t.beginReport(actorUID) if !ok { @@ -228,7 +228,7 @@ func (t *Tracker) report(actorUID string) time.Duration { } t.reporting.queueRetries.Forget("") t.reporting.retries.Forget(actorUID) - t.endReport(actorUID, sent) + t.endReport(actorUID, sent, err == nil || status.Code(err) == codes.NotFound) return 0 } @@ -267,10 +267,13 @@ func (t *Tracker) beginReport(actorUID string) (resources.ActorRef, *ateletpb.Ac } // endReport records that the control plane acknowledged or refused a report -// of sent, so the report is no longer due. If a background change modified -// the actor's state while sent was in flight, it queues the actor to report -// again instead. -func (t *Tracker) endReport(actorUID string, sent *ateletpb.ActorSnapshotState) { +// of sent, so the report is no longer due. When forget is true (the report +// was acknowledged or refused with NotFound) and the actor is detached with +// no upload in progress and has not started again, it removes the actor's +// snapshot state file and directory. If a background change modified the +// actor's state while sent was in flight, it queues the actor to report again +// instead. +func (t *Tracker) endReport(actorUID string, sent *ateletpb.ActorSnapshotState, forget bool) { t.lock(actorUID) defer t.unlock(actorUID) a := t.findLocked(actorUID) @@ -281,12 +284,12 @@ func (t *Tracker) endReport(actorUID string, sent *ateletpb.ActorSnapshotState) t.queueReportLocked(a) return } - t.clearReportDueLocked(a) + t.finishReportDueLocked(a, forget) } // retryReport schedules another attempt to send the report that became due at -// dueSince after err, unless a commit canceled it, a newer -// background change replaced it, or giveUpAfter has passed. +// dueSince after err, unless a commit canceled it, a newer background change +// replaced it, or giveUpAfter has passed. func (t *Tracker) retryReport(actorUID string, dueSince time.Time, attrs []any, err error) { t.lock(actorUID) a := t.findLocked(actorUID) @@ -322,9 +325,24 @@ func (t *Tracker) retryReport(actorUID string, dueSince time.Time, attrs []any, // clearReportDueLocked clears ReportDueSince, resets the report backoff, and // saves the actor's state. func (t *Tracker) clearReportDueLocked(a *actorSnapshots) { + t.finishReportDueLocked(a, false) +} + +// finishReportDueLocked clears ReportDueSince and resets the report backoff. +// When forget is true and the actor is detached with no upload in progress and +// has not started again, it removes the actor's state file and empty +// directory; otherwise it saves the actor's state so a running actor keeps its +// state and a later DetachActor call still sees LatestDurable. +func (t *Tracker) finishReportDueLocked(a *actorSnapshots, forget bool) { t.addLocked(a) t.cancelReportLocked(a) - t.saveLocked(a) + if forget && !a.Started && a.LatestLocal == nil && a.Uploading == nil { + a.unsaved = false + t.undurableActorsTally.forget(a.actorUID) + removeDetachedActorState(a.actorUID) + } else { + t.saveLocked(a) + } t.requeueLocked(a) } diff --git a/cmd/atelet/internal/snapshottracker/report_test.go b/cmd/atelet/internal/snapshottracker/report_test.go index c8c63e76d5..65d378e181 100644 --- a/cmd/atelet/internal/snapshottracker/report_test.go +++ b/cmd/atelet/internal/snapshottracker/report_test.go @@ -19,6 +19,7 @@ import ( "errors" "log/slog" "os" + "path/filepath" "slices" "sync" "testing" @@ -275,37 +276,150 @@ func TestTrackerReportsBackgroundDetach(t *testing.T) { }) } -// A Detach call makes a report of the detach due, as a background detach does. +// A Detach call marks a detach report due and keeps snapshot-state.json on +// disk until the report is acknowledged or refused with NotFound. On give-up +// or any other refusal, snapshot-state.json stays on disk so a later Detach +// call still returns LatestDurable. func TestTrackerReportsDetachCall(t *testing.T) { - store := newFakeSnapshotStore() - tr := newReportingTracker(t, store, newFakeReporter()) - tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) - if _, err := tr.Detach(t.Context(), trackedActorUID); err != nil { - t.Fatalf("detach: %v", err) - } - if !reportDue(tr) || !reportDueOnDisk() || tr.reporting.queue.Len() != 1 { - t.Errorf("report due = %v, on disk = %v, queued = %d after a Detach call, want due, saved, and queued", - reportDue(tr), reportDueOnDisk(), tr.reporting.queue.Len()) + for _, tc := range []struct { + name string + err error + giveUp bool + wantForget bool + }{ + {name: "acknowledged", wantForget: true}, + {name: "refused with NotFound", err: status.Error(codes.NotFound, "actor gone"), wantForget: true}, + {name: "refused with Unimplemented", err: status.Error(codes.Unimplemented, "older ateapi"), wantForget: false}, + {name: "refused with FailedPrecondition", err: status.Error(codes.FailedPrecondition, "conflict"), wantForget: false}, + {name: "gives up after timeout", err: status.Error(codes.Unavailable, "ateapi down"), giveUp: true, wantForget: false}, + } { + t.Run(tc.name, func(t *testing.T) { + store := newFakeSnapshotStore() + reporter := newFakeReporter() + if tc.err != nil { + if tc.giveUp { + reporter.failNext(tc.err, tc.err) + } else { + reporter.failNext(tc.err) + } + } + tr := newReportingTracker(t, store, reporter) + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + dueAt := tr.clock.Now() + if _, err := tr.Detach(t.Context(), trackedActorUID); err != nil { + t.Fatalf("detach: %v", err) + } + if !reportDue(tr) || !reportDueOnDisk() || tr.reporting.queue.Len() != 1 { + t.Fatalf("report due = %v, on disk = %v, queued = %d after Detach, want true, true, 1", reportDue(tr), reportDueOnDisk(), tr.reporting.queue.Len()) + } + + // A repeated Detach while the report is still pending returns + // LatestDurable and leaves the report due. + st, err := tr.Detach(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("second detach: %v", err) + } + if st.GetLatestLocal() != nil || uuidOf(st.GetLatestDurable()) != "s1" { + t.Errorf("second detach state = %v, want s1 durable and nothing local", st) + } + if !reportDue(tr) || !reportDueOnDisk() { + t.Fatal("second detach cleared the pending detach report") + } + + processNextReport(t, tr) + if tc.giveUp { + stepTo(tr, dueAt.Add(DefaultReportGiveUpAfter)) + processNextReport(t, tr) + } + if reportDue(tr) || reportDueOnDisk() { + t.Error("report still due once report ended") + } + if !tc.wantForget { + if _, err := os.Stat(ateletpath.ActorSnapshotStateFile(trackedActorUID)); err != nil { + t.Errorf("snapshot-state.json missing after %s: %v", tc.name, err) + } + } + st, err = tr.Detach(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("third detach: %v", err) + } + if tc.wantForget { + if _, err := os.Stat(ateletpath.ActorPath(trackedActorUID)); !os.IsNotExist(err) { + t.Errorf("actor dir %s still exists: %v", ateletpath.ActorPath(trackedActorUID), err) + } + if !proto.Equal(st, &ateletpb.ActorSnapshotState{}) { + t.Errorf("third detach state = %v, want empty", st) + } + } else { + if st.GetLatestLocal() != nil || uuidOf(st.GetLatestDurable()) != "s1" { + t.Errorf("third detach state = %v, want s1 durable kept", st) + } + if !reportDue(tr) || !reportDueOnDisk() { + t.Fatal("third detach after report failure did not re-queue a detach report") + } + processNextReport(t, tr) + if _, err := os.Stat(ateletpath.ActorPath(trackedActorUID)); !os.IsNotExist(err) { + t.Errorf("actor dir %s still exists after re-queued detach report acknowledged: %v", ateletpath.ActorPath(trackedActorUID), err) + } + } + }) } + + t.Run("started while detach report is due", func(t *testing.T) { + store := newFakeSnapshotStore() + reporter := newFakeReporter() + tr := newReportingTracker(t, store, reporter) + s1 := committedRef("s1", 1, tr.clock.Now()) + tr.CommitLocal(trackedActorUID, s1) + if _, err := tr.Detach(t.Context(), trackedActorUID); err != nil { + t.Fatalf("detach: %v", err) + } + + // BeginStart recreates the actor's state file on disk (which still + // exists while the detach report is due) with Started = true. + endStart, err := tr.BeginStart(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("BeginStart: %v", err) + } + endStart() + + processNextReport(t, tr) + if reportDue(tr) || reportDueOnDisk() { + t.Error("detach report still due once acknowledged") + } + if _, err := os.Stat(ateletpath.ActorSnapshotStateFile(trackedActorUID)); err != nil { + t.Errorf("snapshot-state.json removed for a started actor: %v", err) + } + if uuidOf(tr.State(trackedActorUID).GetLatestDurable()) != "s1" { + t.Errorf("state = %v, want s1 durable kept", tr.State(trackedActorUID)) + } + if newTracker(t).NotStartedSince(trackedActorUID, s1.Snapshot) { + t.Error("NotStartedSince on a fresh tracker = true, want false kept in snapshot-state.json") + } + }) } -// A Detach call whose upload succeeds and prune fails still reports the upload. +// A Detach call whose upload succeeds and cleanup fails still reports the upload. func TestTrackerReportsDetachCallUploadWhenPruneFails(t *testing.T) { store := newFakeSnapshotStore() tr := newReportingTracker(t, store, newFakeReporter()) tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) - // A file where the local checkpoint directory belongs fails the prune. - if err := os.WriteFile(ateletpath.LocalCheckpointsDir(trackedActorUID), nil, 0o600); err != nil { + // A populated volume mount directory fails RemoveActorFilesExceptState. + stillMounted := filepath.Join(ateletpath.ExternalVolumeHostPath(trackedActorUID, "data"), "payload") + if err := os.MkdirAll(filepath.Dir(stillMounted), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(stillMounted, nil, 0o600); err != nil { t.Fatal(err) } if _, err := tr.Detach(t.Context(), trackedActorUID); err == nil { - t.Fatal("detach succeeded, want the prune's error") + t.Fatal("detach succeeded, want the cleanup's error") } if st := tr.State(trackedActorUID); uuidOf(st.GetLatestDurable()) != "s1" { t.Fatalf("state = %v, want s1 durable", st) } if !reportDue(tr) || !reportDueOnDisk() { - t.Error("no report due for the upload after the prune failed") + t.Error("no report due for the upload after the cleanup failed") } } @@ -378,8 +492,12 @@ func TestTrackerDoesNotReportWhenNothingChanged(t *testing.T) { durable.UploadTime = t0.Add(time.Second) tr.CommitDurable(trackedActorUID, durable) tr.CommitLocal(trackedActorUID, local) - // A file where the local checkpoint directory belongs fails the prune. - if err := os.WriteFile(ateletpath.LocalCheckpointsDir(trackedActorUID), nil, 0o600); err != nil { + // A populated volume mount directory fails RemoveActorFilesExceptState. + stillMounted := filepath.Join(ateletpath.ExternalVolumeHostPath(trackedActorUID, "data"), "payload") + if err := os.MkdirAll(filepath.Dir(stillMounted), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(stillMounted, nil, 0o600); err != nil { t.Fatal(err) } @@ -549,38 +667,6 @@ func TestTrackerGivesUpOnReportAfterTimeout(t *testing.T) { }) } -// WithReportWorkers(0) disables background reporting, even when WithReporter -// is set: neither saved reports at startup nor fresh background changes are -// queued or sent. -func TestTrackerWithReportWorkersZeroDisablesReporting(t *testing.T) { - store := newFakeSnapshotStore() - store.setBlocking(true) - before := newTestTracker(t, store) - before.CommitLocal(trackedActorUID, committedRef("s1", 1, before.clock.Now())) - uploadNow(t, before, store, nil) - - reporter := newFakeReporter() - tr := New(t.Context(), store, clocktesting.NewFakeClock(time.Unix(1_700_000_000, 0)), - WithUploadDelay(testUploadDelay), WithDetachDelay(testDetachDelay), - WithReporter(reporter), WithReportWorkers(0)) - tr.Start() - t.Cleanup(func() { - if err := tr.Wait(context.Background()); err != nil { - t.Errorf("Wait: %v", err) - } - }) - - stepTo(tr, tr.clock.Now().Add(reportStartupJitter)) - tr.CommitLocal(trackedActorUID, committedRef("s2", 2, tr.clock.Now())) - uploadNow(t, tr, store, nil) - if n := tr.reporting.queue.Len(); n != 0 { - t.Errorf("reports queued = %d with 0 report workers, want none", n) - } - if got := len(reporter.reports()); got != 0 { - t.Errorf("reports sent = %d with 0 report workers, want none", got) - } -} - // A control-plane failure backs off the whole report queue, so other actors // waiting to report do not all hit an overloaded ateapi at once. Once an // attempt gets a response, the queue backoff resets; Aborted, which means @@ -813,6 +899,31 @@ func TestTrackerReportsAfterRestart(t *testing.T) { } }) + t.Run("sends saved detach report and removes actor directory", func(t *testing.T) { + store := newFakeSnapshotStore() + // The first tracker has no reporter: it saves the detach report due, + // as an atelet that restarts before sending it does, but queues + // nothing. + before := newTestTracker(t, store) + before.CommitLocal(trackedActorUID, committedRef("s1", 1, before.clock.Now())) + detachNow(t, before) + if !reportDueOnDisk() { + t.Fatal("saved detach report not due on disk before restart") + } + + reporter := newFakeReporter() + after := newStoreTracker(t, store) + startReporting(after, reporter) + processNextReport(t, after) + sent := reporter.reports() + if len(sent) != 1 || sent[0].state.GetLatestLocal() != nil || uuidOf(sent[0].state.GetLatestDurable()) != "s1" || sent[0].actor != testActor { + t.Fatalf("reports after the restart = %+v, want one detached report with s1 durable, for %+v", sent, testActor) + } + if _, err := os.Stat(ateletpath.ActorPath(trackedActorUID)); !os.IsNotExist(err) { + t.Errorf("actor dir %s after saved detach report acknowledged: %v", ateletpath.ActorPath(trackedActorUID), err) + } + }) + t.Run("jitters saved reports at startup", func(t *testing.T) { store := newFakeSnapshotStore() store.setBlocking(true) @@ -922,15 +1033,14 @@ func TestTrackerReportsStateChangedDuringReport(t *testing.T) { }) } -// A Checkpoint, a PromoteSnapshot while a local snapshot is kept, or a Detach -// call returns the actor's latest state to the control plane, so it cancels -// any pending background report, whether queued in retry backoff or already in +// A Checkpoint or PromoteSnapshot while a local snapshot is kept returns the +// actor's latest state to the control plane, so it cancels any pending +// background upload report, whether queued in retry backoff or already in // flight. func TestTrackerCancelsReportOnForegroundCall(t *testing.T) { for _, tc := range []struct { - name string - alreadyDetached bool - call func(t *testing.T, tr *Tracker) + name string + call func(t *testing.T, tr *Tracker) }{ { name: "CommitLocal", @@ -962,25 +1072,13 @@ func TestTrackerCancelsReportOnForegroundCall(t *testing.T) { tr.CommittedCheckpointState(trackedActorUID) }, }, - { - name: "CommittedCheckpointState when already detached", - alreadyDetached: true, - call: func(_ *testing.T, tr *Tracker) { - tr.CommittedCheckpointState(trackedActorUID) - }, - }, } { makeReportDue := func(t *testing.T, tr *Tracker, store *fakeSnapshotStore) { t.Helper() tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) - if tc.alreadyDetached { - store.setBlocking(false) - detachNow(t, tr) - } else { - store.setBlocking(true) - uploadNow(t, tr, store, nil) - store.setBlocking(false) - } + store.setBlocking(true) + uploadNow(t, tr, store, nil) + store.setBlocking(false) } t.Run(tc.name+"/queued in retry backoff", func(t *testing.T) { @@ -1050,30 +1148,53 @@ func TestTrackerCancelsReportOnForegroundCall(t *testing.T) { } } -// CommitPromoted after a background detach cleared LatestLocal leaves the -// pending detach report due, since the control plane does not read a promote's -// response as a detach. +// CommitPromoted or CommittedCheckpointState after a background detach cleared +// LatestLocal leaves the pending detach report due, so the report still +// completes and removes the actor's state file and directory. func TestTrackerKeepsDetachReportOnCommitPromotedWhenDetached(t *testing.T) { - store := newFakeSnapshotStore() - reporter := newFakeReporter() - tr := newReportingTracker(t, store, reporter) - tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) - detachNow(t, tr) - if !reportDue(tr) || !reportDueOnDisk() { - t.Fatal("detach report not due before CommitPromoted") - } + for _, tc := range []struct { + name string + call func(tr *Tracker) + }{ + { + name: "CommitPromoted", + call: func(tr *Tracker) { + tr.CommitPromoted(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + }, + }, + { + name: "CommittedCheckpointState", + call: func(tr *Tracker) { + tr.CommittedCheckpointState(trackedActorUID) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + store := newFakeSnapshotStore() + reporter := newFakeReporter() + tr := newReportingTracker(t, store, reporter) + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + detachNow(t, tr) + if !reportDue(tr) || !reportDueOnDisk() { + t.Fatal("detach report not due before foreground call") + } - tr.CommitPromoted(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) - if !reportDue(tr) || !reportDueOnDisk() { - t.Fatal("detach report canceled by CommitPromoted when LatestLocal was nil") - } - processNextReport(t, tr) - sent := reporter.reports() - if len(sent) != 1 || sent[0].state.GetLatestLocal() != nil || uuidOf(sent[0].state.GetLatestDurable()) != "s1" { - t.Errorf("reports = %+v, want 1 detach report with s1 durable", sent) - } - if reportDue(tr) || reportDueOnDisk() { - t.Error("detach report still due once acknowledged") + tc.call(tr) + if !reportDue(tr) || !reportDueOnDisk() { + t.Fatalf("detach report canceled by %s when LatestLocal was nil", tc.name) + } + processNextReport(t, tr) + sent := reporter.reports() + if len(sent) != 1 || sent[0].state.GetLatestLocal() != nil || uuidOf(sent[0].state.GetLatestDurable()) != "s1" { + t.Errorf("reports = %+v, want 1 detach report with s1 durable", sent) + } + if reportDue(tr) || reportDueOnDisk() { + t.Error("detach report still due once acknowledged") + } + if _, err := os.Stat(ateletpath.ActorPath(trackedActorUID)); !os.IsNotExist(err) { + t.Errorf("actor dir %s still exists after detach report acknowledged: %v", ateletpath.ActorPath(trackedActorUID), err) + } + }) } } diff --git a/cmd/atelet/internal/snapshottracker/state.go b/cmd/atelet/internal/snapshottracker/state.go index 95fa5d4e96..29a02e3548 100644 --- a/cmd/atelet/internal/snapshottracker/state.go +++ b/cmd/atelet/internal/snapshottracker/state.go @@ -20,6 +20,7 @@ import ( "fmt" "log/slog" "os" + "syscall" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" "go.ate.dev/substrate/cmd/atelet/internal/atomicfile" @@ -84,6 +85,18 @@ func removeSnapshotState(actorUID string) error { return nil } +// removeDetachedActorState deletes a detached actor's snapshot state file and +// its now-empty directory once its detach report is no longer due. +func removeDetachedActorState(actorUID string) { + if err := removeSnapshotState(actorUID); err != nil { + slog.Warn("Failed to remove the detached actor's snapshot state", slog.String("actorUID", actorUID), slog.Any("err", err)) + return + } + if err := os.Remove(ateletpath.ActorPath(actorUID)); err != nil && !errors.Is(err, os.ErrNotExist) && !errors.Is(err, syscall.ENOTEMPTY) && !errors.Is(err, syscall.EEXIST) { + slog.Warn("Failed to remove the detached actor's directory", slog.String("actorUID", actorUID), slog.Any("err", err)) + } +} + // savedActorUIDs returns the UIDs of the actors with a snapshot state file on // this node. A node with no actor directories yet has none. An actor whose // file cannot be checked is included, so reading the file reports why. diff --git a/cmd/atelet/internal/snapshottracker/tracker.go b/cmd/atelet/internal/snapshottracker/tracker.go index 1109db0046..a52c704166 100644 --- a/cmd/atelet/internal/snapshottracker/tracker.go +++ b/cmd/atelet/internal/snapshottracker/tracker.go @@ -77,7 +77,8 @@ type Store interface { // actor to that operation's end, which queues it again. // 5. An upload leaves LatestLocal on disk so a same-node Restore still // reads it. A detach, in the background or via Detach, makes LatestLocal -// durable and then deletes all the actor's local snapshots. +// durable and then deletes all the actor's local snapshots and its other +// files on this node except snapshot-state.json. // 6. A Run or Restore unschedules the actor's detach as it begins // (BeginStart), and saves that before the workload can start, so atelet // does not detach an actor whose start has begun, even after a restart or @@ -97,9 +98,13 @@ type Store interface { // startup window, send the actor's whole state through the Reporter, and // retry with per-actor and queue-wide backoff until the control plane // acknowledges or refuses it, a call cancels it, or reporting.giveUpAfter -// passes. A report waits while a Run, Restore, Checkpoint, -// PromoteSnapshot, or Terminate is in progress, and a background change -// made while a report ran is reported again. +// passes. Once a detach's report is acknowledged or refused with NotFound +// (and the actor has not started again), atelet removes the actor's +// snapshot state file and directory; on give-up or any other refusal the +// state file stays so Detach keeps returning LatestDurable. A report waits +// while a Run, Restore, Checkpoint, PromoteSnapshot, or Terminate is in +// progress, and a background change made while a report ran is reported +// again. // // Detach runs under the tracker's context rather than on a worker, so an // explicit control-plane call does not wait behind other actors' background @@ -289,7 +294,6 @@ func WithReporter(r Reporter) Option { } // WithReportWorkers sets how many background reports run at once on this node. -// Zero disables background reporting. func WithReportWorkers(n int) Option { return func(t *Tracker) { t.reporting.workers = n @@ -329,9 +333,6 @@ func New(ctx context.Context, store Store, clk clock.WithTicker, opts ...Option) for _, opt := range opts { opt(t) } - if t.reporting.workers <= 0 { - t.reporting.reporter = nil - } if t.instruments != nil { t.undurableActorsTally = newUndurableActorsTally(clk, t.uploadDelay, t.store != nil) t.instruments.watch(t.undurableActorsTally) diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index 4fb7c222b3..99f19ca59b 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -30,6 +30,7 @@ import ( "testing" "time" + "go.ate.dev/substrate/cmd/atelet/internal/actordirs" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" "go.ate.dev/substrate/cmd/atelet/internal/filecache" "go.ate.dev/substrate/cmd/atelet/internal/snapshottracker" @@ -1080,8 +1081,8 @@ func TestTerminateWithoutWorkerPodUID(t *testing.T) { snapshotName = "pause-snap-1" ) - if err := resetActorDirs(actorUID); err != nil { - t.Fatalf("resetActorDirs: %v", err) + if err := actordirs.ResetActorDirs(actorUID); err != nil { + t.Fatalf("ResetActorDirs: %v", err) } snapshotDir := ateletpath.LocalSnapshotDir(actorUID, snapshotName) if err := os.MkdirAll(snapshotDir, 0o700); err != nil { @@ -1320,6 +1321,9 @@ func TestImmediateCheckpointWithoutLocalCopy(t *testing.T) { a.store.mu.Unlock() // What the first call's cleanup leaves behind when it fails. leftover := filepath.Join(ateletpath.OCIBundleDir(pluginActorUID), "leftover") + if err := os.MkdirAll(filepath.Dir(leftover), 0o700); err != nil { + t.Fatal(err) + } if err := os.WriteFile(leftover, nil, 0o600); err != nil { t.Fatal(err) } @@ -1451,6 +1455,9 @@ func TestRetriedCheckpointAnswersFromCommittedSnapshot(t *testing.T) { } // What the first call's cleanup leaves behind when it fails. leftover := filepath.Join(ateletpath.OCIBundleDir(pluginActorUID), "leftover") + if err := os.MkdirAll(filepath.Dir(leftover), 0o700); err != nil { + t.Fatal(err) + } if err := os.WriteFile(leftover, nil, 0o600); err != nil { t.Fatal(err) } @@ -1467,6 +1474,9 @@ func TestRetriedCheckpointAnswersFromCommittedSnapshot(t *testing.T) { if _, err := os.Stat(leftover); !os.IsNotExist(err) { t.Errorf("leftover in the bundle dir after the retry: stat = %v, want it cleaned up", err) } + if _, err := os.Stat(ateletpath.OCIBundleDir(pluginActorUID)); (err == nil) != tc.wantLocal { + t.Errorf("bundle dir exists = %v (err = %v), want %v", err == nil, err, tc.wantLocal) + } st := resp.GetState() if got := st.GetLatestLocal().GetSnapshot(); proto.Equal(got, snap) != tc.wantLocal { t.Errorf("latest_local = %v, want local = %v", got, tc.wantLocal) @@ -1978,11 +1988,21 @@ func detachActorRequest() *ateletpb.DetachActorRequest { } // TestDetachActorDetachesPausedActor detaches a suspended actor through the -// RPC: atelet uploads its local snapshot, deletes the local copy, and returns -// the new state. +// RPC: atelet uploads its local snapshot, deletes the local copy, returns the +// new state, and removes the actor's directory once the detach report is +// acknowledged. func TestDetachActorDetachesPausedActor(t *testing.T) { a := runPluginActor(t) - a.s.snapshots = snapshottracker.New(t.Context(), &trackerStore{s: a.s}, clock.RealClock{}) + a.s.snapshots = snapshottracker.New(t.Context(), &trackerStore{s: a.s}, clock.RealClock{}, + snapshottracker.WithReporter(&trackerReporter{workers: newFakeReportService()})) + a.s.snapshots.Start() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if err := a.s.snapshots.Wait(ctx); err != nil { + t.Errorf("Wait: %v", err) + } + }) if _, err := a.checkpoint(t, a.snapshot("pause-snap-1", 1), ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); err != nil { t.Fatalf("Checkpoint (pause): %v", err) } @@ -1997,8 +2017,18 @@ func TestDetachActorDetachesPausedActor(t *testing.T) { if st := resp.GetState(); st.GetLatestLocal() != nil || st.GetLatestDurable().GetSnapshot().GetSnapshotUuid() != "pause-snap-1" { t.Errorf("state = %v, want pause-snap-1 durable and nothing local", st) } - if _, err := os.Stat(localDir); !errors.Is(err, os.ErrNotExist) { - t.Errorf("local snapshot dir after DetachActor: stat err = %v, want not exist", err) + if _, err := os.Stat(localDir); !os.IsNotExist(err) { + t.Errorf("local snapshot dir after DetachActor: stat = %v, want it deleted", err) + } + waitFor(t, "the actor dir to be removed after the detach report", func() bool { + return !exists(ateletpath.ActorPath(pluginActorUID)) + }) + resp, err = a.s.DetachActor(t.Context(), detachActorRequest()) + if err != nil { + t.Fatalf("second DetachActor: %v", err) + } + if !proto.Equal(resp.GetState(), &ateletpb.ActorSnapshotState{}) { + t.Errorf("second DetachActor state = %v, want empty", resp.GetState()) } } @@ -2243,6 +2273,55 @@ func TestPromoteSnapshotFoundAlreadyUploaded(t *testing.T) { } } +// TestPromoteSnapshotAfterDetach verifies that after an actor is detached +// (its entire directory removed), PromoteSnapshot still succeeds via its +// remote manifest check in object storage (with UploadLocalSnapshot returning +// uploaded = false). +func TestPromoteSnapshotAfterDetach(t *testing.T) { + a := runPluginActor(t) + store := &trackerStore{s: a.s} + a.s.snapshots = snapshottracker.New(t.Context(), store, clock.RealClock{}, + snapshottracker.WithReporter(&trackerReporter{workers: newFakeReportService()})) + a.s.snapshots.Start() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if err := a.s.snapshots.Wait(ctx); err != nil { + t.Errorf("Wait: %v", err) + } + }) + snap := a.snapshot("pause-snap-1", 1) + if _, err := a.checkpoint(t, snap, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); err != nil { + t.Fatalf("Checkpoint (pause): %v", err) + } + + // Detach uploads the snapshot and removes the actor directory once reported. + st, err := a.s.snapshots.Detach(t.Context(), pluginActorUID) + if err != nil { + t.Fatalf("Detach: %v", err) + } + if st.GetLatestLocal() != nil || st.GetLatestDurable() == nil { + t.Fatalf("state = %v, want durable set and local nil", st) + } + waitFor(t, "the actor dir to be removed after the detach report", func() bool { + return !exists(ateletpath.ActorPath(pluginActorUID)) + }) + + // TrackerStore.UploadLocalSnapshot after detach probes object storage and + // reports uploaded = false. + uploaded, err := store.UploadLocalSnapshot(t.Context(), pluginActorUID, snapshottracker.SnapshotRef{Snapshot: snap}) + if err != nil || uploaded { + t.Fatalf("UploadLocalSnapshot after detach = (%v, %v), want (false, nil)", uploaded, err) + } + + // PromoteSnapshot after detach finds the local snapshot gone, probes object + // storage for the manifest uploaded during detach, and succeeds without + // re-uploading. + if got := a.promote(t, snap).GetLatestDurable(); !proto.Equal(got.GetSnapshot(), snap) || got.GetDurableCompleteTime() != nil { + t.Errorf("latest_durable = %v, want %v with no upload time", got, snap) + } +} + // testSandboxRecord is the record of a gVisor runtime whose only asset is // release, as the syncer plans it from a SandboxConfig version. func testSandboxRecord(release gvisorTestRelease) *sandboxAssetsRecord { diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 1aebee24e9..2dc22fbad6 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -34,6 +34,7 @@ import ( "github.com/spf13/pflag" "github.com/spiffe/go-spiffe/v2/spiffeid" "github.com/spiffe/go-spiffe/v2/svid/x509svid" + "go.ate.dev/substrate/cmd/atelet/internal/actordirs" "go.ate.dev/substrate/cmd/atelet/internal/apivalidation" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" "go.ate.dev/substrate/cmd/atelet/internal/credentialprovider" @@ -132,7 +133,7 @@ var ( snapshotUploadEstMiBps = pflag.Int("snapshot-upload-est-throughput-mib", snapshottracker.DefaultUploadEstThroughput>>20, "Estimated snapshot upload throughput, in MiB per second of the snapshot's allocated size: each upload attempt may run --snapshot-upload-timeout plus the time the snapshot takes at this rate. 0 adds no time for the snapshot's size.") snapshotUploadWorkers = pflag.Int("snapshot-upload-workers", snapshottracker.DefaultUploadWorkers, "How many background snapshot uploads atelet runs at once.") snapshotDetachWorkers = pflag.Int("snapshot-detach-workers", snapshottracker.DefaultDetachWorkers, "How many background detaches atelet runs at once.") - snapshotReportWorkers = pflag.Int("snapshot-report-workers", snapshottracker.DefaultReportWorkers, "How many background snapshot state reports atelet runs at once. 0 disables background reporting.") + snapshotReportWorkers = pflag.Int("snapshot-report-workers", snapshottracker.DefaultReportWorkers, "How many background snapshot state reports atelet runs at once.") snapshotReportGiveUpAfter = pflag.Duration("snapshot-report-give-up-after", snapshottracker.DefaultReportGiveUpAfter, "How long after a background upload or detach atelet keeps retrying a failed snapshot state report before giving up on it.") ) @@ -617,8 +618,8 @@ func validateSnapshotDelayFlags() error { } // validateSnapshotWorkerFlags checks --snapshot-upload-workers, -// --snapshot-detach-workers, and --snapshot-report-workers: upload and detach -// must be at least 1, and report must be at least 0. +// --snapshot-detach-workers, and --snapshot-report-workers: each must be at +// least 1. func validateSnapshotWorkerFlags() error { if *snapshotUploadWorkers < 1 { return fmt.Errorf("--snapshot-upload-workers %d must be >= 1", *snapshotUploadWorkers) @@ -626,8 +627,8 @@ func validateSnapshotWorkerFlags() error { if *snapshotDetachWorkers < 1 { return fmt.Errorf("--snapshot-detach-workers %d must be >= 1", *snapshotDetachWorkers) } - if *snapshotReportWorkers < 0 { - return fmt.Errorf("--snapshot-report-workers %d must be >= 0", *snapshotReportWorkers) + if *snapshotReportWorkers < 1 { + return fmt.Errorf("--snapshot-report-workers %d must be >= 1", *snapshotReportWorkers) } return nil } @@ -828,7 +829,7 @@ func (s *Atelet) Run(ctx context.Context, req *ateletpb.RunRequest) (resp *atele }() s.systemInfoVolumes.Deregister(actorUID) - if err := resetActorDirsKeepingVolumes(actorUID); err != nil { + if err := actordirs.ResetActorDirsKeepingVolumes(actorUID); err != nil { return nil, fmt.Errorf("while resetting actor dirs: %w", err) } @@ -998,7 +999,7 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest // check holds through the cleanup. if s.snapshots.NotStartedSince(actorUID, req.GetSnapshot()) { tCleanup := time.Now() - err = s.cleanUpAfterCheckpoint(ctx, req) + err = s.cleanUpAfterCheckpoint(ctx, req, state.GetLatestLocal() != nil) dCleanup = time.Since(tCleanup) if err != nil { return nil, err @@ -1098,7 +1099,7 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest state = s.snapshots.CommitDurable(actorUID, durable) // The snapshot is durable now, so the local copy only saves a // download: failing to keep it must not fail the checkpoint. - // resetActorDirs cleans up checkpointDir if the move fails. + // RemoveActorFilesExceptState cleans up checkpointDir if the move fails. if err := s.moveLocalCheckpoint(ctx, req, sandboxRec); err != nil { slog.WarnContext(ctx, "Failed to keep a local copy of the uploaded snapshot", slog.Any("actor", actorRef), slog.Any("err", err)) } else { @@ -1110,7 +1111,7 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest dPersist = time.Since(tPersist) tCleanup := time.Now() - err = s.cleanUpAfterCheckpoint(ctx, req) + err = s.cleanUpAfterCheckpoint(ctx, req, state.GetLatestLocal() != nil) dCleanup = time.Since(tCleanup) if err != nil { return nil, err @@ -1120,10 +1121,10 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest // committedCheckpointState returns the actor's state for a retry of a // Checkpoint that committed req's snapshot, canceling any pending background -// report. Under UPLOAD_MODE_IMMEDIATE it first makes the snapshot durable, as -// PromoteSnapshot does, since the Checkpoint that committed it may have -// deferred its upload. It records in op and dPersist what promoteLocalSnapshot -// records. +// upload report while a local snapshot is kept. Under UPLOAD_MODE_IMMEDIATE it +// first makes the snapshot durable, as PromoteSnapshot does, since the +// Checkpoint that committed it may have deferred its upload. It records in op +// and dPersist what promoteLocalSnapshot records. func (s *Atelet) committedCheckpointState(ctx context.Context, req *ateletpb.CheckpointRequest, op *snapshotOp, dPersist *time.Duration) (*ateletpb.ActorSnapshotState, error) { if req.GetUploadConfig().GetMode() == ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE { uri, err := resources.ParseSnapshotURI(req.GetSnapshot().GetObject().GetSnapshotUri()) @@ -1146,9 +1147,11 @@ func (s *Atelet) committedCheckpointState(ctx context.Context, req *ateletpb.Che // cleanUpAfterCheckpoint finishes a Checkpoint that has committed its // snapshot: it prunes the actor's local snapshots, unmounts its external -// volumes, and empties its directories. Each step is idempotent, so a retried -// Checkpoint runs them again. -func (s *Atelet) cleanUpAfterCheckpoint(ctx context.Context, req *ateletpb.CheckpointRequest) error { +// volumes, and empties its directories (or removes its files except +// snapshot-state.json when keepLocal is false, as after a detach or a failed +// local copy). Each step is idempotent, so a retried Checkpoint runs them +// again. +func (s *Atelet) cleanUpAfterCheckpoint(ctx context.Context, req *ateletpb.CheckpointRequest, keepLocal bool) error { actorUID := req.GetActorUid() // Delete the actor's earlier local snapshots only now that this checkpoint @@ -1179,8 +1182,14 @@ func (s *Atelet) cleanUpAfterCheckpoint(ctx context.Context, req *ateletpb.Check } // Note: we do not crash the actor if resetting the directory fails. - if err := resetActorDirs(actorUID); err != nil { - return fmt.Errorf("while resetting actor dirs: %w", err) + if keepLocal { + if err := actordirs.ResetActorDirs(actorUID); err != nil { + return fmt.Errorf("while resetting actor dirs: %w", err) + } + } else { + if err := actordirs.RemoveActorFilesExceptState(actorUID); err != nil { + return fmt.Errorf("while removing actor files: %w", err) + } } return nil } @@ -1236,7 +1245,7 @@ func (s *Atelet) moveLocalCheckpoint(ctx context.Context, req *ateletpb.Checkpoi // Stage the files worker reported and the snapshot manifest inside // checkpointDir before renaming the directory into local-checkpoint/, so a - // failure leaves nothing under local-checkpoint/ and resetActorDirs cleans + // failure leaves nothing under local-checkpoint/ and ResetActorDirs cleans // checkpointDir up. stageDir := filepath.Join(checkpointDir, ".staged") if err := root.RemoveAll(stageDir); err != nil { @@ -1595,7 +1604,7 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res // Not crashing the actor, because terminal errors here indicate problems with atelet, // node or the disk itself. s.systemInfoVolumes.Deregister(actorUID) - if err := resetActorDirsKeepingVolumes(actorUID); err != nil { + if err := actordirs.ResetActorDirsKeepingVolumes(actorUID); err != nil { return nil, fmt.Errorf("while resetting actor dirs: %w", err) } // A restore may bring another runtime than the one staged here before, @@ -1998,7 +2007,7 @@ func (s *Atelet) Terminate(ctx context.Context, req *ateletpb.TerminateRequest) } // Reclaim the actor's directories on the node - if err := removeActorDirs(actorUID); err != nil { + if err := actordirs.RemoveActorDirs(actorUID); err != nil { return nil, fmt.Errorf("failed to remove actor directories during terminate (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) } s.restoreFailures.reset(actorUID) @@ -2275,73 +2284,16 @@ func (d *WorkerDialer) DialWorkerPod(ctx context.Context, podUID string) (*grpc. return conn, nil } -// resetActorDirs empties the actor's directories and leaves them in place for -// its next activation. Use removeActorDirs when the actor will not come back. -func resetActorDirs(actorUID string) error { - if err := resetActorDirsKeepingVolumes(actorUID); err != nil { - return err - } - // Do not call RemoveAll on volume directories in case the unmount failed. - // We do not want to delete mount content. - volumesDir := ateletpath.ExternalVolumeMountsDir(actorUID) - entries, err := os.ReadDir(volumesDir) - if err != nil { - return wrapFileSystemErr("while reading volumes dir", err) - } - for _, entry := range entries { - volPath := filepath.Join(volumesDir, entry.Name()) - if err := os.Remove(volPath); err != nil { - return wrapFileSystemErr("while removing volume dir", err) - } - } - return nil -} - -// resetActorDirsKeepingVolumes is resetActorDirs for Run and Restore, which -// leave the external volume mount points alone: a failed earlier attempt can -// leave a volume published there, and publishing it again is a no-op. -func resetActorDirsKeepingVolumes(actorUID string) error { - // Explicitly leave runsc logs dir untouched. - - // RemoveAllWritable, not os.RemoveAll: the bundle's upper dir can hold - // copied-up actor-image directories keeping the image's (possibly - // read-only) modes, which atelet can't remove as plain root without first - // making them writable. (The rootfs itself is just an empty mountpoint - // here: the overlay is mounted in the worker pod's mount namespace, not - // atelet's, and is detached by worker at teardown.) - if err := resetDir("bundle dir", ateletpath.OCIBundleDir(actorUID), imagecache.RemoveAllWritable, 0o700); err != nil { - return err - } - if err := resetDir("checkpoint-state dir", ateletpath.CheckpointStateDir(actorUID), os.RemoveAll, 0o700); err != nil { - return err - } - if err := resetDir("restore-state dir", ateletpath.RestoreStateDir(actorUID), os.RemoveAll, 0o700); err != nil { - return err - } - if err := resetDir("durable-dir volumes mount dir", ateletpath.DurableDirVolumeMountsDir(actorUID), os.RemoveAll, 0o755); err != nil { - return err - } - // World-readable (0o755): bind-mounted read-only into the actor, whose - // workload reads it through the gofer. - if err := resetDir("system-info volume roots dir", ateletpath.SystemInfoVolumeRootsDir(actorUID), os.RemoveAll, 0o755); err != nil { - return err - } - if err := os.MkdirAll(ateletpath.ExternalVolumeMountsDir(actorUID), 0o755); err != nil { - return wrapFileSystemErr("while creating volumes dir", err) - } - return nil -} - // resetSandboxAssetsDir empties the actor's sandbox assets dir for Restore -// to stage into. Not part of resetActorDirs: the links an activation holds -// stay while the actor remains resumable on this node, through Checkpoint +// to stage into. Not part of actordirs.ResetActorDirs: the links an activation +// holds stay while the actor remains resumable on this node, through Checkpoint // and its local snapshot, and keep its binaries in the cache meanwhile; // Run replaces them through its staging swap. The entries are hard links // into the asset cache, so removing them never touches the cached bytes. // Owner-only, like the bundle dir: the worker runs the binaries from here // as root. func resetSandboxAssetsDir(actorUID string) error { - return resetDir("sandbox assets dir", ateletpath.SandboxAssetsDir(actorUID), os.RemoveAll, 0o700) + return actordirs.ResetDir("sandbox assets dir", ateletpath.SandboxAssetsDir(actorUID), os.RemoveAll, 0o700) } // commitSandboxAssets replaces the actor's sandbox assets dir with the set @@ -2371,41 +2323,6 @@ func commitSandboxAssets(actorUID, staging string, paths map[string]string) erro return nil } -// resetDir empties dir with remove and recreates it with mode. what names the -// directory in errors. -func resetDir(what, dir string, remove func(string) error, mode os.FileMode) error { - if err := remove(dir); err != nil { - return wrapFileSystemErr("while deleting "+what, err) - } - if err := os.MkdirAll(dir, mode); err != nil { - return wrapFileSystemErr("while creating "+what, err) - } - return nil -} - -// removeActorDirs reclaims both of the actor's directory trees, roots -// included: nothing else on the node deletes them, and no later activation -// will look there. -// -// resetActorDirs runs first for the care a blanket RemoveAll lacks. It refuses -// to proceed while a volume directory is still populated, so a failed unmount -// cannot become a deletion of the mount's contents, and it can remove a bundle -// upper dir carrying an image's read-only modes. -func removeActorDirs(actorUID string) error { - if err := resetActorDirs(actorUID); err != nil { - return err - } - // The actor dir holds sandbox-assets.json, which a retried Terminate needs, - // so remove it last. - if err := os.RemoveAll(ateletpath.ActorSnapshotPath(actorUID)); err != nil { - return wrapFileSystemErr("while deleting actor snapshot dir", err) - } - if err := os.RemoveAll(ateletpath.ActorPath(actorUID)); err != nil { - return wrapFileSystemErr("while deleting actor dir", err) - } - return nil -} - // ateletServerTLSConfig builds a *tls.Config for a gRPC server that presents the // credential bundle at servingBundlePath, requires a client certificate // chaining to a CA in clientCAPath, and authorizes the client with diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index dcadeb47d5..d44079b368 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -35,6 +35,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/spf13/pflag" + "go.ate.dev/substrate/cmd/atelet/internal/actordirs" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" "go.ate.dev/substrate/cmd/atelet/internal/snapshottracker" "go.ate.dev/substrate/internal/apierror" @@ -143,8 +144,8 @@ func TestValidateSnapshotUploadEstThroughput(t *testing.T) { } } -// Upload and detach worker flags must be at least 1, report workers must be at -// least 0, and each flag defaults to the tracker's default count. +// Upload, detach, and report worker flags must be at least 1, and each flag +// defaults to the tracker's default count. func TestValidateSnapshotWorkerFlags(t *testing.T) { setFlags := func(upload, detach, report int) { *snapshotUploadWorkers, *snapshotDetachWorkers, *snapshotReportWorkers = upload, detach, report @@ -160,11 +161,11 @@ func TestValidateSnapshotWorkerFlags(t *testing.T) { }{ {"defaults", snapshottracker.DefaultUploadWorkers, snapshottracker.DefaultDetachWorkers, snapshottracker.DefaultReportWorkers, false}, {"single worker each", 1, 1, 1, false}, - {"zero report workers", 2, 2, 0, false}, {"zero upload workers", 0, 2, 1, true}, {"negative upload workers", -1, 2, 1, true}, {"zero detach workers", 2, 0, 1, true}, {"negative detach workers", 2, -1, 1, true}, + {"zero report workers", 2, 2, 0, true}, {"negative report workers", 2, 2, -1, true}, } { t.Run(tc.name, func(t *testing.T) { @@ -872,16 +873,16 @@ func TestRemoveActorDirsReclaimsTheRoot(t *testing.T) { useTempNodeDirs(t) const actorUID = "actor-uid-1" - if err := resetActorDirs(actorUID); err != nil { - t.Fatalf("resetActorDirs: %v", err) + if err := actordirs.ResetActorDirs(actorUID); err != nil { + t.Fatalf("ResetActorDirs: %v", err) } // Written at Run and deliberately kept out of the directories reset wipes. if err := os.WriteFile(ateletpath.ActorSandboxAssetsFile(actorUID), []byte("{}"), 0o600); err != nil { t.Fatalf("writing sandbox assets file: %v", err) } - if err := removeActorDirs(actorUID); err != nil { - t.Fatalf("removeActorDirs: %v", err) + if err := actordirs.RemoveActorDirs(actorUID); err != nil { + t.Fatalf("RemoveActorDirs: %v", err) } assertDirGone(t, "actor dir", ateletpath.ActorPath(actorUID)) @@ -911,8 +912,8 @@ func TestResetActorDirsCreatesBothTrees(t *testing.T) { useTempNodeDirs(t) const actorUID = "actor-uid-isolation" - if err := resetActorDirs(actorUID); err != nil { - t.Fatalf("resetActorDirs(%q) failed: %v", actorUID, err) + if err := actordirs.ResetActorDirs(actorUID); err != nil { + t.Fatalf("ResetActorDirs(%q) failed: %v", actorUID, err) } for root, dirs := range map[string]map[string]string{ ateletpath.ActorSnapshotPath(actorUID): { @@ -931,7 +932,7 @@ func TestResetActorDirsCreatesBothTrees(t *testing.T) { t.Errorf("%s = %q, want it under %q", name, dir, root) } if info, err := os.Stat(dir); err != nil || !info.IsDir() { - t.Errorf("resetActorDirs did not create %s (%q): err=%v", name, dir, err) + t.Errorf("ResetActorDirs did not create %s (%q): err=%v", name, dir, err) } } } @@ -944,8 +945,8 @@ func TestRemoveActorDirsKeepsActorDirWhenSnapshotTreeRemains(t *testing.T) { useTempNodeDirs(t) const actorUID = "actor-uid-1" - if err := resetActorDirs(actorUID); err != nil { - t.Fatalf("resetActorDirs(%q) failed: %v", actorUID, err) + if err := actordirs.ResetActorDirs(actorUID); err != nil { + t.Fatalf("ResetActorDirs(%q) failed: %v", actorUID, err) } record := ateletpath.ActorSandboxAssetsFile(actorUID) if err := os.WriteFile(record, []byte("{}"), 0o600); err != nil { @@ -964,8 +965,8 @@ func TestRemoveActorDirsKeepsActorDirWhenSnapshotTreeRemains(t *testing.T) { } t.Cleanup(func() { _ = os.Chmod(stuck, 0o700) }) - if err := removeActorDirs(actorUID); err == nil { - t.Fatal("removeActorDirs() = nil, want an error for the undeletable snapshot") + if err := actordirs.RemoveActorDirs(actorUID); err == nil { + t.Fatal("RemoveActorDirs() = nil, want an error for the undeletable snapshot") } if _, err := os.Stat(record); err != nil { t.Errorf("sandbox record %s is gone (err=%v), want it kept for a retried Terminate", record, err) @@ -973,9 +974,9 @@ func TestRemoveActorDirsKeepsActorDirWhenSnapshotTreeRemains(t *testing.T) { } // TestRemoveActorDirsKeepsPopulatedVolume checks that removing the root does -// not defeat the mount safety resetActorDirs provides: a volume directory with -// contents means the unmount did not take, and deleting it would take the -// mounted volume's data rather than the actor's. +// not defeat the mount safety actordirs.ResetActorDirs provides: a volume +// directory with contents means the unmount did not take, and deleting it would +// take the mounted volume's data rather than the actor's. func TestRemoveActorDirsKeepsPopulatedVolume(t *testing.T) { useTempNodeDirs(t) const actorUID = "actor-uid-1" @@ -988,15 +989,15 @@ func TestRemoveActorDirsKeepsPopulatedVolume(t *testing.T) { t.Fatalf("writing volume contents: %v", err) } - err := removeActorDirs(actorUID) + err := actordirs.RemoveActorDirs(actorUID) if err == nil { - t.Fatal("removeActorDirs succeeded with a populated volume dir, want an error") + t.Fatal("RemoveActorDirs succeeded with a populated volume dir, want an error") } if strings.Contains(err.Error(), "%w") { - t.Errorf("removeActorDirs error %q has a literal %%w", err) + t.Errorf("RemoveActorDirs error %q has a literal %%w", err) } if _, err := os.Stat(stillMounted); err != nil { - t.Errorf("removeActorDirs deleted the contents of a volume that was still populated: %v", err) + t.Errorf("RemoveActorDirs deleted the contents of a volume that was still populated: %v", err) } } @@ -1006,8 +1007,8 @@ func TestResetDir(t *testing.T) { if err := os.MkdirAll(filepath.Join(dir, "sub"), 0o700); err != nil { t.Fatal(err) } - if err := resetDir("state dir", dir, os.RemoveAll, 0o750); err != nil { - t.Fatalf("resetDir: %v", err) + if err := actordirs.ResetDir("state dir", dir, os.RemoveAll, 0o750); err != nil { + t.Fatalf("ResetDir: %v", err) } entries, err := os.ReadDir(dir) if err != nil || len(entries) != 0 { @@ -1020,9 +1021,9 @@ func TestResetDir(t *testing.T) { t.Run("names the directory when the remove fails", func(t *testing.T) { boom := errors.New("boom") - err := resetDir("state dir", t.TempDir(), func(string) error { return boom }, 0o700) + err := actordirs.ResetDir("state dir", t.TempDir(), func(string) error { return boom }, 0o700) if !errors.Is(err, boom) { - t.Fatalf("resetDir = %v, want it to wrap %v", err, boom) + t.Fatalf("ResetDir = %v, want it to wrap %v", err, boom) } if got, want := err.Error(), "while deleting state dir: boom"; got != want { t.Errorf("error = %q, want %q", got, want) @@ -1034,9 +1035,9 @@ func TestResetDir(t *testing.T) { if err := os.WriteFile(file, nil, 0o600); err != nil { t.Fatal(err) } - err := resetDir("state dir", filepath.Join(file, "child"), func(string) error { return nil }, 0o700) + err := actordirs.ResetDir("state dir", filepath.Join(file, "child"), func(string) error { return nil }, 0o700) if err == nil || !strings.HasPrefix(err.Error(), "while creating state dir: ") { - t.Errorf("resetDir = %v, want an error starting %q", err, "while creating state dir: ") + t.Errorf("ResetDir = %v, want an error starting %q", err, "while creating state dir: ") } if err != nil && strings.Contains(err.Error(), "%w") { t.Errorf("error %q has a literal %%w", err) diff --git a/cmd/atelet/oci.go b/cmd/atelet/oci.go index 32e4f39125..4cf7019008 100644 --- a/cmd/atelet/oci.go +++ b/cmd/atelet/oci.go @@ -83,8 +83,9 @@ func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, acto bundlePath := ateletpath.OCIBundlePath(actorUID, containerName) - // Clear any previous bundle contents (belt and suspenders: resetActorDirs - // already wiped the bundle dir on the Run/Restore path). + // Clear any previous bundle contents (belt and suspenders: + // actordirs.ResetActorDirs already wiped the bundle dir on the Run/Restore + // path). if err := imagecache.RemoveAllWritable(bundlePath); err != nil { return nil, fmt.Errorf("while clearing bundle %q: %w", bundlePath, err) } diff --git a/cmd/atelet/sandbox_asset_cache.go b/cmd/atelet/sandbox_asset_cache.go index 78e40b458e..f64d3cc4bd 100644 --- a/cmd/atelet/sandbox_asset_cache.go +++ b/cmd/atelet/sandbox_asset_cache.go @@ -26,6 +26,7 @@ import ( "github.com/spf13/pflag" + "go.ate.dev/substrate/cmd/atelet/internal/actordirs" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" "go.ate.dev/substrate/cmd/atelet/internal/filecache" "go.ate.dev/substrate/cmd/atelet/internal/filecachegc" @@ -73,7 +74,7 @@ func openSandboxAssetCache(ctx context.Context, dir string, minAge, fetchTimeout // is fully usable: log and carry on rather than failing startup. slog.WarnContext(ctx, "Sandbox asset cache debris sweep incomplete", slog.Any("err", err)) } - if err := resetDir("sandbox asset scratch dir", ateletpath.SandboxAssetScratchDir, os.RemoveAll, 0o700); err != nil { + if err := actordirs.ResetDir("sandbox asset scratch dir", ateletpath.SandboxAssetScratchDir, os.RemoveAll, 0o700); err != nil { return nil, err } slog.InfoContext(ctx, "Sandbox asset cache open", diff --git a/cmd/atelet/volumes_test.go b/cmd/atelet/volumes_test.go index 079e27debf..605a4c8d1d 100644 --- a/cmd/atelet/volumes_test.go +++ b/cmd/atelet/volumes_test.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/cmd/atelet/internal/actordirs" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" "go.ate.dev/substrate/internal/proto/ateletpb" "go.ate.dev/substrate/internal/volume" @@ -415,7 +416,7 @@ func TestVolumeHostDirectoryCleanup(t *testing.T) { } }) - t.Run("resetActorDirs removes empty unmounted volume host directories", func(t *testing.T) { + t.Run("ResetActorDirs removes empty unmounted volume host directories", func(t *testing.T) { useTempNodeDirs(t) fake := &fakeWorkerPlugin{} @@ -441,14 +442,14 @@ func TestVolumeHostDirectoryCleanup(t *testing.T) { } } - // resetActorDirs cleans up empty volume directories left after unmount. - if err := resetActorDirs(actorUID); err != nil { - t.Fatalf("resetActorDirs failed: %v", err) + // actordirs.ResetActorDirs cleans up empty volume directories left after unmount. + if err := actordirs.ResetActorDirs(actorUID); err != nil { + t.Fatalf("ResetActorDirs failed: %v", err) } for _, p := range []string{path1, path2} { if _, err := os.Stat(p); !os.IsNotExist(err) { - t.Errorf("expected mount path %q to be deleted by resetActorDirs, stat err = %v", p, err) + t.Errorf("expected mount path %q to be deleted by ResetActorDirs, stat err = %v", p, err) } } @@ -463,7 +464,7 @@ func TestVolumeHostDirectoryCleanup(t *testing.T) { } }) - t.Run("resetActorDirs preserves non-empty volume directories and returns error", func(t *testing.T) { + t.Run("ResetActorDirs preserves non-empty volume directories and returns error", func(t *testing.T) { useTempNodeDirs(t) fake := &fakeWorkerPlugin{} @@ -484,9 +485,9 @@ func TestVolumeHostDirectoryCleanup(t *testing.T) { t.Fatalf("writing test file: %v", err) } - err := resetActorDirs(actorUID) + err := actordirs.ResetActorDirs(actorUID) if err == nil { - t.Fatal("expected resetActorDirs to fail on non-empty volume dir, got nil") + t.Fatal("expected ResetActorDirs to fail on non-empty volume dir, got nil") } if !strings.Contains(err.Error(), "while removing volume dir") { t.Errorf("error = %v, want error containing 'while removing volume dir'", err) @@ -498,11 +499,11 @@ func TestVolumeHostDirectoryCleanup(t *testing.T) { } }) - t.Run("resetActorDirs succeeds cleanly when volumes directory does not exist", func(t *testing.T) { + t.Run("ResetActorDirs succeeds cleanly when volumes directory does not exist", func(t *testing.T) { useTempNodeDirs(t) - if err := resetActorDirs(actorUID); err != nil { - t.Fatalf("resetActorDirs failed when volumes dir does not exist: %v", err) + if err := actordirs.ResetActorDirs(actorUID); err != nil { + t.Fatalf("ResetActorDirs failed when volumes dir does not exist: %v", err) } volumesDir := ateletpath.ExternalVolumeMountsDir(actorUID) diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index a71507d993..491577f943 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -1105,17 +1105,18 @@ groups: instrument: histogram unit: s stability: development - brief: The time of deleting the local snapshots of an actor during a background detach on atelet, excluding any snapshot upload. + brief: The time of deleting the local snapshots and node files of an actor during a background detach on atelet, excluding any snapshot upload. note: > atelet detaches an actor once it stays suspended on the node for the detach delay. If object storage does not hold the latest local snapshot yet, the detach uploads it first and records that upload on atelet.snapshot.upload.duration; a detach whose upload fails stops there and is not recorded here. Once the snapshot is durable, the detach deletes - the local snapshots of the actor, which is what this histogram times. A - detach that Terminate or an atelet shutdown cancels records error.type - Canceled. A failed detach keeps the local snapshots, and atelet tries it - again later. A detach that a DetachActor call asks for is not recorded. + the local snapshots and other node files of the actor (except + snapshot-state.json), which is what this histogram times. A detach that + Terminate or an atelet shutdown cancels records error.type Canceled. A + failed detach keeps LatestLocal, and atelet tries it again later. A detach + that a DetachActor call asks for is not recorded. annotations: substrate: emitted_by: [atelet] diff --git a/docs/observability.md b/docs/observability.md index 54b73eca19..28d715d7a6 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -290,7 +290,7 @@ Agent Substrate emits foundational OpenTelemetry system and server metrics to mo | `ate.actor.restore.duration` | atelet | histogram | how long each phase of a restore takes on the worker node, which is where cold-start latency actually goes once ateapi hands off (labels `ate.snapshot.phase`, `ate.snapshot.kind`, `ate.snapshot.fidelity`, `ate.template.atespace`, `ate.template.name`, `ate.sandbox.class`) | | `ate.actor.checkpoint.duration` | atelet | histogram | the same phase breakdown for writing a snapshot, so a slow suspend can be attributed to ateworker or to the upload (same labels as the restore histogram) | | `atelet.snapshot.upload.duration` | atelet | histogram | how long a background upload of a local snapshot to object storage takes and whether it failed (`error.type` present = failure, absent = success; label `ate.snapshot.fidelity`) | -| `atelet.snapshot.detach.duration` | atelet | histogram | how long a background detach takes to delete an actor's local snapshots, excluding any snapshot upload, and whether it failed (`error.type` present = failure, absent = success; label `ate.snapshot.fidelity`) | +| `atelet.snapshot.detach.duration` | atelet | histogram | how long a background detach takes to delete an actor's local snapshots and node files, excluding any snapshot upload, and whether it failed (`error.type` present = failure, absent = success; label `ate.snapshot.fidelity`) | | `atelet.snapshot.upload.overdue.duration` | atelet | histogram | how long after a background snapshot upload came due a worker started it: near zero while an upload worker is free, growing when the node's upload workers fall behind (no labels) | | `atelet.snapshot.detach.overdue.duration` | atelet | histogram | how long after a background detach came due a worker started it: near zero while a detach worker is free, growing when the node's detach workers fall behind (no labels) | | `atelet.snapshot.upload.overdue.actors` | atelet | up/down counter | actors on the node with an undurable local snapshot past `--snapshot-upload-delay` waiting for an upload to finish, bucketed by time overdue (label `ate.snapshot.upload_overdue_band`: `under_1m`, `1m_to_5m`, `5m_to_15m`, `over_15m`) | diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index 695a5d2ba0..b9fd6030ec 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -3353,8 +3353,9 @@ type ActorSnapshotState struct { Uploading *SnapshotReport `protobuf:"bytes,2,opt,name=uploading,proto3" json:"uploading,omitempty"` // The newest snapshot atelet knows is durable, whether or not the control // plane has recorded it: one atelet uploaded since it started, or one a - // PromoteSnapshot found already uploaded. Unset if there is none, which says - // nothing about older durable snapshots. + // PromoteSnapshot found already uploaded, while atelet still tracks the + // actor. Unset if there is none (or once the actor is detached and + // reported), which says nothing about older durable snapshots. LatestDurable *SnapshotReport `protobuf:"bytes,3,opt,name=latest_durable,json=latestDurable,proto3" json:"latest_durable,omitempty"` // When atelet next plans to trigger a background upload, of `uploading` if // it is waiting to resume or retry, or of `latest_local` otherwise. Unset @@ -4087,8 +4088,9 @@ func (x *DetachActorRequest) GetActorUid() string { type DetachActorResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - // The actor's state once detached: latest_local is unset, and latest_durable - // is its latest snapshot, if atelet knows one. + // The actor's state from the detach: latest_local is unset, and + // latest_durable is its latest snapshot. Once detached and reported, atelet + // forgets the actor, so calling DetachActor again returns an empty state. State *ActorSnapshotState `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index 898d1e5a8c..28918fe34b 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -421,13 +421,13 @@ service Atelet { // atelet also detaches a suspended actor on its own after its detach delay. // It reports either detach with WorkerService.ReportActorSnapshotState, so // the control plane records a DetachActor call's detach even if the caller - // does not. The detach doesn't end with the call: if the caller's context - // ends first, DetachActor returns that error and the detach runs on. Waits - // for an operation already in progress on the actor, such as an upload or a - // detach. Does nothing when atelet holds no local snapshot for - // the actor. Returns ABORTED when the actor runs on this node, and - // FAILED_PRECONDITION when atelet cannot make the latest local snapshot - // durable, such as when it has no `snapshot.object`. + // does not, and once reported, forgets the actor. The detach doesn't end with + // the call: if the caller's context ends first, DetachActor returns that + // error and the detach runs on. Waits for an operation already in progress on + // the actor, such as an upload or a detach. Does nothing when atelet holds no + // local snapshot for the actor. Returns ABORTED when the actor runs on this + // node, and FAILED_PRECONDITION when atelet cannot make the latest local + // snapshot durable, such as when it has no `snapshot.object`. rpc DetachActor(DetachActorRequest) returns (DetachActorResponse) {} // Terminate tells atelet to terminate/kill any running workload for an actor, @@ -965,8 +965,9 @@ message ActorSnapshotState { // The newest snapshot atelet knows is durable, whether or not the control // plane has recorded it: one atelet uploaded since it started, or one a - // PromoteSnapshot found already uploaded. Unset if there is none, which says - // nothing about older durable snapshots. + // PromoteSnapshot found already uploaded, while atelet still tracks the + // actor. Unset if there is none (or once the actor is detached and + // reported), which says nothing about older durable snapshots. SnapshotReport latest_durable = 3; // When atelet next plans to trigger a background upload, of `uploading` if @@ -1201,7 +1202,8 @@ message DetachActorRequest { } message DetachActorResponse { - // The actor's state once detached: latest_local is unset, and latest_durable - // is its latest snapshot, if atelet knows one. + // The actor's state from the detach: latest_local is unset, and + // latest_durable is its latest snapshot. Once detached and reported, atelet + // forgets the actor, so calling DetachActor again returns an empty state. ActorSnapshotState state = 1; } diff --git a/internal/proto/ateletpb/atelet_grpc.pb.go b/internal/proto/ateletpb/atelet_grpc.pb.go index e2f893f352..1257667a81 100644 --- a/internal/proto/ateletpb/atelet_grpc.pb.go +++ b/internal/proto/ateletpb/atelet_grpc.pb.go @@ -395,13 +395,13 @@ type AteletClient interface { // atelet also detaches a suspended actor on its own after its detach delay. // It reports either detach with WorkerService.ReportActorSnapshotState, so // the control plane records a DetachActor call's detach even if the caller - // does not. The detach doesn't end with the call: if the caller's context - // ends first, DetachActor returns that error and the detach runs on. Waits - // for an operation already in progress on the actor, such as an upload or a - // detach. Does nothing when atelet holds no local snapshot for - // the actor. Returns ABORTED when the actor runs on this node, and - // FAILED_PRECONDITION when atelet cannot make the latest local snapshot - // durable, such as when it has no `snapshot.object`. + // does not, and once reported, forgets the actor. The detach doesn't end with + // the call: if the caller's context ends first, DetachActor returns that + // error and the detach runs on. Waits for an operation already in progress on + // the actor, such as an upload or a detach. Does nothing when atelet holds no + // local snapshot for the actor. Returns ABORTED when the actor runs on this + // node, and FAILED_PRECONDITION when atelet cannot make the latest local + // snapshot durable, such as when it has no `snapshot.object`. DetachActor(ctx context.Context, in *DetachActorRequest, opts ...grpc.CallOption) (*DetachActorResponse, error) // Terminate tells atelet to terminate/kill any running workload for an actor, // unmount its volumes, and clean up actor state on the node. @@ -522,13 +522,13 @@ type AteletServer interface { // atelet also detaches a suspended actor on its own after its detach delay. // It reports either detach with WorkerService.ReportActorSnapshotState, so // the control plane records a DetachActor call's detach even if the caller - // does not. The detach doesn't end with the call: if the caller's context - // ends first, DetachActor returns that error and the detach runs on. Waits - // for an operation already in progress on the actor, such as an upload or a - // detach. Does nothing when atelet holds no local snapshot for - // the actor. Returns ABORTED when the actor runs on this node, and - // FAILED_PRECONDITION when atelet cannot make the latest local snapshot - // durable, such as when it has no `snapshot.object`. + // does not, and once reported, forgets the actor. The detach doesn't end with + // the call: if the caller's context ends first, DetachActor returns that + // error and the detach runs on. Waits for an operation already in progress on + // the actor, such as an upload or a detach. Does nothing when atelet holds no + // local snapshot for the actor. Returns ABORTED when the actor runs on this + // node, and FAILED_PRECONDITION when atelet cannot make the latest local + // snapshot durable, such as when it has no `snapshot.object`. DetachActor(context.Context, *DetachActorRequest) (*DetachActorResponse, error) // Terminate tells atelet to terminate/kill any running workload for an actor, // unmount its volumes, and clean up actor state on the node. From e8a07194060bbeb7377f8d44dd98bc14d0d694b8 Mon Sep 17 00:00:00 2001 From: Sairaj Pokale Date: Wed, 21 Oct 2026 22:57:36 +0000 Subject: [PATCH 03/12] ateapi,atelet: reclaim actor node state when a worker's pod is gone (#2797) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What is this PR and Why we need it ? A deleted worker pod is deregistered by `workersync`, which calls `DeleteWorker` to mark its assigned actors `CRASHED` and clear their `WorkerAssignment`. Those actors are never terminated on the node, so `/var/lib/ate/actors/` and `/var/lib/ate/snapshots/` stay on disk until the actor is later deleted or reverted, if it ever is. (Note: this covers worker-pod removal via `DeleteWorker`; tearing down a crashed sandbox on a still-live worker is tracked separately in #2355.) ### Changes - **`atelet` workerless / gone-worker `Terminate`**: - Skips `TerminateWorkload` when `worker_pod_uid` is empty, the worker socket is gone, `sandbox-assets.json` is absent (`Run` and `Restore` write it before `RunWorkload`/`RestoreWorkload`, and an earlier `Terminate` removes it with the actor directory), or `TerminateWorkload` returns `Unavailable`/`NotFound`. Any other error from a live worker fails `Terminate` before touching disk. - Volume unmount failures in `Terminate` return `Internal` without `%w`-wrapping downstream gRPC errors, so a transient CSI failure does not surface to `ateapi` as `Unavailable` (and reaches `DeleteActor`/`RevertActor` callers as `Internal` instead of `Unavailable`). - **`ateapi` (`DeleteWorker`)**: - `DeleteWorker` takes each actor's lease (`ABORTED` if held, requeued by `workersync`), skips actors that already reached `SUSPENDED`, and calls best-effort `atelet.Terminate` with a 30s timeout to reclaim the actor's node state and local snapshots. Once a call times out (`DeadlineExceeded`) or returns `Unavailable`, remaining actors on that worker skip the node and leave cleanup to revert/delete. --- ### Testing - **Unit & functional tests**: `go test ./cmd/atelet/... ./cmd/ateapi/internal/controlapi/...` - **Live-cluster verification** (3 × `c3-standard-4` `gvisor` nodes, 3 × `c3-highcpu-4` `microvm` nodes): | Suite | `gvisor` (3 × `c3-standard-4`) | `microvm` (3 × `c3-highcpu-4`) | |---|---|---| | **`durdir_data_baseline` 1m smoke** (8 MiB `glutton-durdir-data`) | `20` reqs, `0` fails (`0.00%`) | `15` reqs, `0` fails (`0.00%`) | | **`DeleteWorker` scale-down reclaim** (3 actors, 256 MiB `durable-dir`/actor) | `/var/lib/ate/actors`: `0.00 GB -> 1.41 GB (3 dirs) -> 0.00 GB (0 dirs)`; `-0.25 GB`/node reclaimed (`+0.00 GB` vs baseline) | `/var/lib/ate/actors`: `0.00 GB -> 1.64 GB (3 dirs) -> 0.00 GB (0 dirs)`; `-0.25 GB`/node reclaimed (`+0.00 GB` vs baseline) | | **`DeleteActor` on paused actor with no worker** (`worker_pod_uid=""`) | `local-checkpoint` pruned & `/var/lib/ate/actors/` removed (`1.75 ms`) | `local-checkpoint` pruned & `/var/lib/ate/actors/` removed (`9.45 ms`) | Fixes #1688 - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- cmd/ateapi/internal/controlapi/service.go | 2 +- .../worker_assignment_reconciler.go | 2 +- cmd/ateapi/internal/controlapi/worker_test.go | 2 +- cmd/ateapi/internal/controlapi/workflow.go | 23 +- .../internal/controlapi/workflow_delete.go | 54 +-- .../controlapi/workflow_delete_test.go | 11 +- .../workflow_reconcile_assignments_test.go | 12 +- .../controlapi/workflow_resume_test.go | 22 +- .../controlapi/workflow_testutil_test.go | 16 +- .../controlapi/workflow_worker_delete.go | 143 +++++++- .../controlapi/workflow_worker_delete_test.go | 344 +++++++++++++++++- cmd/atelet/lifecycle_test.go | 271 +++++++++++++- cmd/atelet/main.go | 152 +++++--- 13 files changed, 941 insertions(+), 113 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/service.go b/cmd/ateapi/internal/controlapi/service.go index e86ef5bde2..6b94cabb40 100644 --- a/cmd/ateapi/internal/controlapi/service.go +++ b/cmd/ateapi/internal/controlapi/service.go @@ -107,7 +107,7 @@ func NewRPCService( } s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, storageClassLister, instruments, egressGatewayAddress, s, objectStorePlugin) s.actorWorkflow.clusterSnapshotLocation = snapshotLocation - s.workerWorkflow = NewWorkerWorkflow(impl) + s.workerWorkflow = NewWorkerWorkflow(impl, dialer) return s } diff --git a/cmd/ateapi/internal/controlapi/worker_assignment_reconciler.go b/cmd/ateapi/internal/controlapi/worker_assignment_reconciler.go index 3b5bca3ac9..e4f185eab9 100644 --- a/cmd/ateapi/internal/controlapi/worker_assignment_reconciler.go +++ b/cmd/ateapi/internal/controlapi/worker_assignment_reconciler.go @@ -57,7 +57,7 @@ type WorkerAssignmentReconciler struct { func NewWorkerAssignmentReconciler(persistence workerWorkflowStore, workers workerWatcher) *WorkerAssignmentReconciler { return &WorkerAssignmentReconciler{ persistence: persistence, - workflow: NewWorkerWorkflow(persistence), + workflow: NewWorkerWorkflow(persistence, nil), workers: workers, queue: workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[string]()), } diff --git a/cmd/ateapi/internal/controlapi/worker_test.go b/cmd/ateapi/internal/controlapi/worker_test.go index 1cd167e5ab..bce229aa69 100644 --- a/cmd/ateapi/internal/controlapi/worker_test.go +++ b/cmd/ateapi/internal/controlapi/worker_test.go @@ -73,7 +73,7 @@ func newWorkerAPIService(t *testing.T) (*RPCService, store.Interface) { persistence, cleanup := storetest.SetupTestStore(t) t.Cleanup(cleanup) impl := newServiceImpl(persistence, nil) - return &RPCService{impl: impl, workerWorkflow: NewWorkerWorkflow(persistence)}, persistence + return &RPCService{impl: impl, workerWorkflow: NewWorkerWorkflow(persistence, newNodeAteletDialer(t, &capturingTerminator{}))}, persistence } // seedAPIWorker registers a worker directly through the store and returns it as diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index d164bfc532..c94cd12aa4 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -19,6 +19,8 @@ import ( "errors" "fmt" "log/slog" + "sync" + "time" "go.ate.dev/substrate/cmd/ateapi/internal/scheduling" "go.ate.dev/substrate/cmd/ateapi/internal/store" @@ -198,12 +200,25 @@ type actorWorkflowStore interface { // does from the other side: releasing the Actor bound to a Worker stays // in-process because there is no bind/release RPC. type WorkerWorkflow struct { - store workerWorkflowStore + store workerWorkflowStore + dialer *AteletDialer + reclaimTimeout time.Duration + now func() time.Time + + mu sync.Mutex + gaveUp map[string]time.Time } -// NewWorkerWorkflow creates a new WorkerWorkflow. -func NewWorkerWorkflow(store workerWorkflowStore) *WorkerWorkflow { - return &WorkerWorkflow{store: store} +// NewWorkerWorkflow creates a new WorkerWorkflow. dialer may be nil when the +// caller only runs ReconcileAssignments. +func NewWorkerWorkflow(store workerWorkflowStore, dialer *AteletDialer) *WorkerWorkflow { + return &WorkerWorkflow{ + store: store, + dialer: dialer, + reclaimTimeout: defaultWorkerReclaimTimeout, + now: time.Now, + gaveUp: make(map[string]time.Time), + } } // workerWorkflowStore enumerates the exact storage methods needed by diff --git a/cmd/ateapi/internal/controlapi/workflow_delete.go b/cmd/ateapi/internal/controlapi/workflow_delete.go index 2dc3e09d37..070f1a2966 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete.go @@ -161,28 +161,7 @@ func (w *ActorWorkflow) ensureAteletTerminated(ctx context.Context, actorRef res slog.String("actor", actorRef.Name), slog.String("templateAtespace", actor.GetActorTemplate().GetAtespace()), slog.String("templateName", actor.GetActorTemplate().GetName())) - // Given that external volumes has been in the codebase for a while and - // its preview-ness is solely around the API, and that the impact of - // disabling the gate while in use would be an orphaned volume - // attachment, we are NOT checking the ExternalVolumes gate in the - // delete/suspend path. - workloadSpec = &ateletpb.WorkloadSpec{} - if preview.IsEnabled(preview.GateExternalVolumes) { - for _, vol := range actor.GetStatus().GetExternalVolumes().GetVolumes() { - // StorageVolumeId is only populated once the volume is provisioned. - // Skip volumes that were never created (e.g. failed during PENDING state). - if vol.GetStorageVolumeId() != "" { - workloadSpec.Volumes = append(workloadSpec.Volumes, &ateletpb.Volume{ - Name: vol.GetName(), - External: &ateletpb.ExternalVolumeSource{ - StorageVolumeId: vol.GetStorageVolumeId(), - VolumeType: vol.GetVolumeType(), - VolumeContext: vol.GetVolumeContext(), - }, - }) - } - } - } + workloadSpec = fallbackWorkloadSpec(actor) } req := &ateletpb.TerminateRequest{ @@ -206,6 +185,37 @@ func (w *ActorWorkflow) ensureAteletTerminated(ctx context.Context, actorRef res return nil } +// fallbackWorkloadSpec is the spec to terminate an actor with when its template +// cannot be resolved, whether deleted or never readable from the caller. Only +// the external volumes recorded on the actor are carried, because atelet still +// unmounts them from the node. A volume with no StorageVolumeId was never +// provisioned (e.g. the actor failed while PENDING), so there is nothing +// mounted to name. +func fallbackWorkloadSpec(actor *ateapipb.Actor) *ateletpb.WorkloadSpec { + // Given that external volumes has been in the codebase for a while and + // its preview-ness is solely around the API, and that the impact of + // disabling the gate while in use would be an orphaned volume + // attachment, we are NOT checking the ExternalVolumes gate in the + // delete/suspend path. + spec := &ateletpb.WorkloadSpec{} + if preview.IsEnabled(preview.GateExternalVolumes) { + for _, vol := range actor.GetStatus().GetExternalVolumes().GetVolumes() { + if vol.GetStorageVolumeId() == "" { + continue + } + spec.Volumes = append(spec.Volumes, &ateletpb.Volume{ + Name: vol.GetName(), + External: &ateletpb.ExternalVolumeSource{ + StorageVolumeId: vol.GetStorageVolumeId(), + VolumeType: vol.GetVolumeType(), + VolumeContext: vol.GetVolumeContext(), + }, + }) + } + } + return spec +} + // ensureVolumesDetachedForDelete detaches external volumes. func (w *ActorWorkflow) ensureVolumesDetachedForDelete(ctx context.Context, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (err error) { ctx, done := stepSpan(ctx, "DetachVolumesForDelete") diff --git a/cmd/ateapi/internal/controlapi/workflow_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_delete_test.go index 485b403c71..76bb5ff02f 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete_test.go @@ -603,17 +603,16 @@ func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) { } fresh = uri objects.PutSnapshot(t, fresh, "manifest.json") - // The worker's pod goes away with the commit still outstanding, so - // the suspend never gets to finish. - if _, err := NewWorkerWorkflow(persistence).DeleteWorker(ctx, workerName, store.DeletePreconditions{}); err != nil { - t.Errorf("DeleteWorker: %v", err) - return err - } return apierror.Unavailable("worker pod is gone") } if _, err := actorWorkflow.SuspendActor(ctx, actorRef, suspendSync); apierror.Code(err) != codes.Unavailable { t.Fatalf("SuspendActor err = %v, want Unavailable from the failed checkpoint", err) } + // SuspendActor failed mid-checkpoint and released the actor's lease, + // leaving the actor in SUSPENDING; DeleteWorker then crashes it. + if _, err := NewWorkerWorkflow(persistence, actorWorkflow.dialer).DeleteWorker(ctx, workerName, store.DeletePreconditions{}); err != nil { + t.Fatalf("DeleteWorker: %v", err) + } // The actor is CRASHED; DeleteActor deletes the in-progress snapshot // (in_progress_snapshot_uri). diff --git a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go index b4312d3191..b3a1b1a13a 100644 --- a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go @@ -67,7 +67,7 @@ func mustRaiseEpoch(t *testing.T, ctx context.Context, svc *RPCService, persiste func mustReconcileAssignments(t *testing.T, ctx context.Context, persistence store.Interface) { t.Helper() - if err := NewWorkerWorkflow(persistence).ReconcileAssignments(ctx, apiWorkerName); err != nil { + if err := NewWorkerWorkflow(persistence, nil).ReconcileAssignments(ctx, apiWorkerName); err != nil { t.Fatalf("ReconcileAssignments() failed: %v", err) } } @@ -378,7 +378,7 @@ func TestReconcileAssignments_WaitsForBusyActor(t *testing.T) { if err != nil { t.Fatalf("AcquireLease() failed: %v", err) } - err = NewWorkerWorkflow(persistence).ReconcileAssignments(ctx, apiWorkerName) + err = NewWorkerWorkflow(persistence, nil).ReconcileAssignments(ctx, apiWorkerName) if !errors.Is(err, errActorBusy) { t.Fatalf("ReconcileAssignments() = %v, want errActorBusy", err) } @@ -455,7 +455,7 @@ func TestReconcileAssignments_KeepsActorReboundSinceList(t *testing.T) { t.Fatalf("UpdateActor() failed: %v", err) } }} - if err := NewWorkerWorkflow(rebinding).ReconcileAssignments(ctx, apiWorkerName); err != nil { + if err := NewWorkerWorkflow(rebinding, nil).ReconcileAssignments(ctx, apiWorkerName); err != nil { t.Fatalf("ReconcileAssignments() failed: %v", err) } @@ -484,7 +484,7 @@ func TestReconcileAssignments_RetriesRecordAfterConcurrentBind(t *testing.T) { binding := &hookedStore{Interface: persistence, beforeUpdateWorker: func() { assignAPIWorker(t, ctx, persistence, apiWorkerName, boundUID) }} - if err := NewWorkerWorkflow(binding).ReconcileAssignments(ctx, apiWorkerName); err != nil { + if err := NewWorkerWorkflow(binding, nil).ReconcileAssignments(ctx, apiWorkerName); err != nil { t.Fatalf("ReconcileAssignments() failed: %v", err) } @@ -518,7 +518,7 @@ func TestReconcileAssignments_LeavesReplacedWorker(t *testing.T) { } seedEpochWorker(t, ctx, persistence, 1, 1) }} - if err := NewWorkerWorkflow(replacing).ReconcileAssignments(ctx, apiWorkerName); err != nil { + if err := NewWorkerWorkflow(replacing, nil).ReconcileAssignments(ctx, apiWorkerName); err != nil { t.Fatalf("ReconcileAssignments() failed: %v", err) } @@ -536,7 +536,7 @@ func TestReconcileAssignments_FailureKeepsObservedEpoch(t *testing.T) { seedRunningActor(t, ctx, persistence) mustRaiseEpoch(t, ctx, svc, persistence, 2) - failing := NewWorkerWorkflow(failingUpdateActorStore{Interface: persistence, err: errors.New("crash failed")}) + failing := NewWorkerWorkflow(failingUpdateActorStore{Interface: persistence, err: errors.New("crash failed")}, nil) if err := failing.ReconcileAssignments(ctx, apiWorkerName); err == nil { t.Fatal("ReconcileAssignments() = nil error, want the crash failure reported") } diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index 36d2eb8783..efd45a0fd5 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -1190,17 +1190,24 @@ func wireTestAssignment() *ateapipb.WorkerAssignment { } // newWireCaptureWorkflow builds an ActorWorkflow whose atelet dialer resolves -// to an in-process capturing fake. The dialer's conn cache is pre-warmed with -// a bufconn-backed connection for the atelet pod's UID and IP, so -// DialForAteletOnNode returns it without dialing the pod IP. The fake's server -// runs atelet's interceptor, so its errors reach the workflow with the code -// atelet would send. +// to an in-process capturing fake. func newWireCaptureWorkflow(t *testing.T, persistence store.Interface) (*ActorWorkflow, *capturingAtelet) { t.Helper() fake := &capturingAtelet{} + return &ActorWorkflow{store: persistence, dialer: newNodeAteletDialer(t, fake)}, fake +} + +// newNodeAteletDialer resolves node-1's atelet to an in-process fake. The +// dialer's conn cache is pre-warmed with a bufconn-backed connection for the +// atelet pod's UID and IP, so DialForAteletOnNode returns it without dialing +// the pod IP. The server runs atelet's interceptor, so its errors reach the +// workflow with the code atelet would send. +func newNodeAteletDialer(t *testing.T, srvImpl ateletpb.AteletServer) *AteletDialer { + t.Helper() + srv := grpc.NewServer(grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor)) - ateletpb.RegisterAteletServer(srv, fake) + ateletpb.RegisterAteletServer(srv, srvImpl) lis := bufconn.Listen(1 << 20) go func() { if err := srv.Serve(lis); err != nil { @@ -1227,8 +1234,7 @@ func newWireCaptureWorkflow(t *testing.T, persistence store.Interface) (*ActorWo } dialer := NewAteletDialer(newTestAteletIndexer(t, ateletPod), installdefaults.SystemNamespace, "", "") dialer.ateletConns.Add("atelet-uid", &ateletConn{ip: "10.0.0.1", conn: conn}) - - return &ActorWorkflow{store: persistence, dialer: dialer}, fake + return dialer } // TestResumeActor_AteletWireRequest is the characteristic test for the diff --git a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go index 2b347393bb..2476323ef9 100644 --- a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go @@ -32,6 +32,7 @@ import ( sdkmetric "go.opentelemetry.io/otel/sdk/metric" "google.golang.org/grpc/codes" "google.golang.org/protobuf/proto" + "k8s.io/client-go/tools/cache" ) // testWorkerUID derives a stable pod UID from a pod name, for Workers seeded @@ -369,13 +370,19 @@ func mustReadActor(t *testing.T, ctx context.Context, st store.Interface, actorR } // workerDeletedDuringReleaseStore runs the DeleteWorker workflow's release of -// an actor right after a finalize step frees the actor's worker. That workflow -// takes no actor lease, so its crash can land between the step's read of the -// actor and its write. +// an actor right after a finalize step frees the actor's worker, so its crash +// runs between the step's read of the actor and its write. In production the +// release takes the actor's lease, which pause and suspend hold through their +// finalize steps, so it waits for them instead; AcquireLease is bypassed here +// to keep the finalize step's own version guard against the crash covered. type workerDeletedDuringReleaseStore struct { store.Interface } +func (s workerDeletedDuringReleaseStore) AcquireLease(ctx context.Context, _ string) (*store.Lease, error) { + return store.NewLease(ctx, func() {}), nil +} + func (s workerDeletedDuringReleaseStore) ReleaseActorFromWorker(ctx context.Context, workerName, actorUID string) (*ateapipb.Worker, error) { worker, err := s.Interface.GetWorker(ctx, workerName) if err != nil { @@ -389,7 +396,8 @@ func (s workerDeletedDuringReleaseStore) ReleaseActorFromWorker(ctx context.Cont if err != nil { return nil, err } - if err := NewWorkerWorkflow(s.Interface).releaseBoundActor(ctx, worker, assignment); err != nil { + dialer := NewAteletDialer(cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{byNode: nil}), "", "", "") + if err := NewWorkerWorkflow(s, dialer).releaseBoundActor(ctx, worker, assignment); err != nil { return nil, err } return released, nil diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go index 7d12051f2c..9513e68bee 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go @@ -19,15 +19,49 @@ import ( "errors" "fmt" "log/slog" + "time" "go.ate.dev/substrate/cmd/ateapi/internal/store" "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/ateattr" "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/proto/ateletpb" "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) +// defaultWorkerReclaimTimeout bounds each Terminate a worker delete sends to +// reclaim a released actor's node state, and how long a node whose atelet did +// not answer is skipped by subsequent reclaims. +const defaultWorkerReclaimTimeout = 30 * time.Second + +// nodeGaveUp reports whether an earlier reclaim on nodeName timed out or could +// not reach atelet within w.reclaimTimeout, pruning any expired entries. +// Remembering the node across DeleteWorker calls keeps a hung atelet from +// stalling workersync on every worker pod deleted from that node. +func (w *WorkerWorkflow) nodeGaveUp(nodeName string) bool { + now := w.now() + w.mu.Lock() + defer w.mu.Unlock() + for node, until := range w.gaveUp { + if !now.Before(until) { + delete(w.gaveUp, node) + } + } + _, ok := w.gaveUp[nodeName] + return ok +} + +// giveUpOnNode records that nodeName's atelet could not be reached or did not +// answer, so subsequent reclaims on that node are skipped for w.reclaimTimeout. +func (w *WorkerWorkflow) giveUpOnNode(nodeName string) { + w.mu.Lock() + defer w.mu.Unlock() + w.gaveUp[nodeName] = w.now().Add(w.reclaimTimeout) +} + // DeleteWorker executes the workflow to deregister a Worker. The caller reaches // here because the Worker's pod is gone, so the Actor bound to it — if any — has // lost its sandbox and is released before the record is removed. @@ -99,21 +133,28 @@ func (w *WorkerWorkflow) ensureMarkedForDeletion(ctx context.Context, name strin // // A single failure stops the sweep, leaving the Worker record in place with the // Actors that have not been released still bound to it, so a retry picks up -// where this left off. +// where this left off. An Actor with an operation in progress is skipped +// instead, so the others are still released, and the call then fails as +// ABORTED for the caller to retry once that operation is done. func (w *WorkerWorkflow) ensureBoundActorsReleased(ctx context.Context, worker *ateapipb.Worker) (err error) { ctx, done := stepSpan(ctx, "ReleaseBoundActors") defer func() { err = done(err) }() // Every page: a Worker can hold thousands of Actors and a page holds at // most a thousand, so stopping at the first would leave the rest bound. - var released int + var released, busy int for token := ""; ; { page, err := w.store.ListWorkerAssignments(ctx, worker.GetMetadata().GetName(), store.ListOptions{PageToken: token}) if err != nil { return fmt.Errorf("while listing the assignments of worker %s: %w", worker.GetMetadata().GetName(), err) } for _, assignment := range page.Items { - if err := w.releaseBoundActor(ctx, worker, assignment); err != nil { + err := w.releaseBoundActor(ctx, worker, assignment) + if errors.Is(err, errActorBusy) { + busy++ + continue + } + if err != nil { return err } released++ @@ -123,6 +164,9 @@ func (w *WorkerWorkflow) ensureBoundActorsReleased(ctx context.Context, worker * } token = page.NextPageToken } + if busy > 0 { + return apierror.Aborted("%d actors on worker %s have an operation in progress; left for a retry", busy, worker.GetMetadata().GetName()) + } if released == 0 { markSkipped(ctx, "worker has no actors assigned") } @@ -139,8 +183,11 @@ func (w *WorkerWorkflow) ensureBoundActorsReleased(ctx context.Context, worker * // assignment and an Actor that has since moved elsewhere both leave no Actor // pointing at this Worker, which is the state this is driving towards. // -// A concurrent SuspendActor or ResumeActor wins the optimistic version check; -// this attempt fails as ABORTED so the caller retries against the newer state. +// It runs under the Actor's lease, as releaseEarlierAssignment does, so no +// operation on the Actor is midway: a suspend cannot finalize a snapshot while +// the reclaim below runs, nor lose a checkpoint it is still uploading. An Actor +// whose lease is held is left for a retry (errActorBusy). A write that still +// races this one fails the version check as ABORTED. func (w *WorkerWorkflow) releaseBoundActor(ctx context.Context, worker *ateapipb.Worker, assignment *ateapipb.ActorAssignment) error { if assignment.GetActor() == nil { markSkipped(ctx, "assignment names no actor") @@ -148,6 +195,16 @@ func (w *WorkerWorkflow) releaseBoundActor(ctx context.Context, worker *ateapipb } name := worker.GetMetadata().GetName() actorRef := resources.ActorRefFromObjectRef(assignment.GetActor()) + lease, err := w.store.AcquireLease(ctx, actorLeaseKey(actorRef)) + if errors.Is(err, store.ErrLeaseConflict) { + return errActorBusy + } + if err != nil { + return fmt.Errorf("while acquiring the lease of actor %s: %w", actorRef, err) + } + defer lease.Close() + ctx = lease.Context() + actor, err := w.store.GetActor(ctx, actorRef) if errors.Is(err, store.ErrNotFound) { markSkipped(ctx, "assigned actor no longer exists") @@ -171,6 +228,13 @@ func (w *WorkerWorkflow) releaseBoundActor(ctx context.Context, worker *ateapipb markSkipped(ctx, "actor suspended cleanly before the pod went away") return nil } + + // Reclaim the actor's runtime state on the node before crashing it. A + // crash keeps the actor's assigned node, so if this best-effort reclaim is + // skipped or fails, deleting or reverting the actor reclaims the state + // there later. + w.reclaimActorStateOnNode(ctx, worker, actor) + return w.crashBoundActor(ctx, worker, actorRef, actor, "Releasing actor from a worker whose pod is gone", crashMessageWorkerPodGone) } @@ -235,6 +299,75 @@ func (w *WorkerWorkflow) crashBoundActor(ctx context.Context, worker *ateapipb.W return nil } +// reclaimActorStateOnNode asks the atelet on the worker's node to terminate the +// actor, removing /var/lib/ate/actors/ and /var/lib/ate/snapshots/ +// (durable dir, checkpoint and restore images, and local snapshots) and +// unmounting its external volumes. +// +// The worker's pod and process are gone by the time this runs, while the +// node's atelet remains and reclaims the directories without a live worker. +// +// This call is best-effort so an unreachable node cannot wedge worker +// deregistration. If it fails, the actor's revert or delete reclaims the state +// on its assigned node. Each Terminate gets w.reclaimTimeout so a hung unmount, +// such as against a dead NFS server, cannot run out the delete's own deadline +// and fail the release after it. +func (w *WorkerWorkflow) reclaimActorStateOnNode(ctx context.Context, worker *ateapipb.Worker, actor *ateapipb.Actor) { + ctx, done := stepSpan(ctx, "ReclaimActorStateOnNode") + defer func() { _ = done(nil) }() + + nodeName := worker.GetNodeName() + if w.nodeGaveUp(nodeName) { + markSkipped(ctx, "an earlier reclaim on the node timed out or could not reach atelet") + return + } + + actorRef := resources.ActorRefFromActor(actor) + logAttrs := []any{ + slog.Any("actor", actorRef), + slog.String("actor_uid", actor.GetMetadata().GetUid()), + slog.String("worker", worker.GetMetadata().GetName()), + slog.String("node", nodeName), + } + + conn, err := w.dialer.DialForAteletOnNode(nodeName) + if err != nil { + // Includes ErrNoAteletOnNode: the atelet is restarting, or the node + // itself is gone. Nothing to reclaim against right now. + w.giveUpOnNode(nodeName) + slog.WarnContext(ctx, "Could not reach the atelet holding a released actor's state; leaving it, and other actors' state on the node, for their revert or delete", + append(logAttrs, slog.Any("err", err))...) + return + } + + slog.InfoContext(ctx, "Reclaiming the node state of an actor released from a worker whose pod is gone", logAttrs...) + // The template is not resolvable from this workflow, and a delete can + // outlive it. The fallback spec carries the external volumes to unmount. + callCtx, cancel := context.WithTimeout(ctx, w.reclaimTimeout) + defer cancel() + _, err = ateletpb.NewAteletClient(conn).Terminate(callCtx, &ateletpb.TerminateRequest{ + WorkerPodUid: worker.GetWorkerPodUid(), + Atespace: actor.GetMetadata().GetAtespace(), + ActorName: actor.GetMetadata().GetName(), + ActorUid: actor.GetMetadata().GetUid(), + ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), + ActorTemplateName: actor.GetActorTemplate().GetName(), + Spec: fallbackWorkloadSpec(actor), + }) + switch status.Code(err) { + case codes.OK: + case codes.NotFound: + slog.InfoContext(ctx, "Actor already terminated on its node", logAttrs...) + case codes.DeadlineExceeded, codes.Unavailable: + w.giveUpOnNode(nodeName) + slog.WarnContext(ctx, "The atelet holding a released actor's state did not answer; leaving it, and other actors' state on the node, for their revert or delete", + append(logAttrs, slog.Any("err", err))...) + default: + slog.WarnContext(ctx, "Failed to reclaim a released actor's node state; leaving it for the actor's revert or delete", + append(logAttrs, slog.Any("err", err))...) + } +} + // finalizeDeleted removes the worker from the store and returns the deleted // record. func (w *WorkerWorkflow) finalizeDeleted(ctx context.Context, worker *ateapipb.Worker) (_ *ateapipb.Worker, err error) { diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go index d408a328c9..0f29a746af 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go @@ -17,17 +17,22 @@ package controlapi import ( "context" "errors" + "slices" "strings" + "sync" "testing" + "time" "go.ate.dev/substrate/cmd/ateapi/internal/store" "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/proto/ateletpb" "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/pkg/proto/ateapipb" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "google.golang.org/grpc/codes" + "google.golang.org/protobuf/proto" ) // newWorkerDeleteWorkflow returns a workflow backed by a real store, which is @@ -37,7 +42,7 @@ func newWorkerDeleteWorkflow(t *testing.T) (*WorkerWorkflow, store.Interface) { t.Helper() persistence, cleanup := storetest.SetupTestStore(t) t.Cleanup(cleanup) - return NewWorkerWorkflow(persistence), persistence + return NewWorkerWorkflow(persistence, newNodeAteletDialer(t, &capturingTerminator{})), persistence } // apiActorRef names the Actor seedAPIActor stores. @@ -88,7 +93,7 @@ func TestDeleteWorkerWorkflow_DrainsBeforeSweeping(t *testing.T) { actor := seedAPIActor(t, ctx, persistence, ateapipb.ActorState_ACTOR_STATE_RUNNING) assignAPIWorker(t, ctx, persistence, apiWorkerName, actor.GetMetadata().GetUid()) - wf := NewWorkerWorkflow(failingUpdateActorStore{Interface: persistence, err: errors.New("release failed")}) + wf := NewWorkerWorkflow(failingUpdateActorStore{Interface: persistence, err: errors.New("release failed")}, newNodeAteletDialer(t, &capturingTerminator{})) if _, err := wf.DeleteWorker(ctx, apiWorkerName, store.DeletePreconditions{}); err == nil { t.Fatal("DeleteWorker() = nil error, want the release failure reported") } @@ -310,7 +315,7 @@ func TestDeleteWorkerWorkflow_FailedReleaseKeepsWorker(t *testing.T) { actor := seedAPIActor(t, ctx, persistence, ateapipb.ActorState_ACTOR_STATE_RUNNING) assignAPIWorker(t, ctx, persistence, apiWorkerName, actor.GetMetadata().GetUid()) - wf := NewWorkerWorkflow(failingUpdateActorStore{Interface: persistence, err: tc.updateErr}) + wf := NewWorkerWorkflow(failingUpdateActorStore{Interface: persistence, err: tc.updateErr}, newNodeAteletDialer(t, &capturingTerminator{})) _, err := wf.DeleteWorker(ctx, apiWorkerName, store.DeletePreconditions{}) if err == nil { t.Fatal("DeleteWorker() = nil error, want the release failure reported") @@ -357,7 +362,7 @@ func TestDeleteWorkerWorkflow_ActorDeletedDuringRelease(t *testing.T) { actor := seedAPIActor(t, ctx, persistence, ateapipb.ActorState_ACTOR_STATE_RUNNING) assignAPIWorker(t, ctx, persistence, apiWorkerName, actor.GetMetadata().GetUid()) - wf := NewWorkerWorkflow(failingUpdateActorStore{Interface: persistence, err: store.ErrNotFound}) + wf := NewWorkerWorkflow(failingUpdateActorStore{Interface: persistence, err: store.ErrNotFound}, newNodeAteletDialer(t, &capturingTerminator{})) if _, err := wf.DeleteWorker(ctx, apiWorkerName, store.DeletePreconditions{}); err != nil { t.Fatalf("DeleteWorker() failed: %v", err) } @@ -420,3 +425,334 @@ func TestDeleteWorkerWorkflow_DrainedWorkerIsStillAdmitted(t *testing.T) { t.Errorf("version = %d, want %d: admission is one write", got, want) } } + +// A Worker is deleted because its pod is gone. The pod took the worker process +// with it, leaving the actor's state directory on the node. This delete +// reclaims it ahead of the actor's revert or delete. +func TestDeleteWorkerWorkflow_ReclaimsActorNodeState(t *testing.T) { + tests := []struct { + name string + // state the actor is in when its pod disappears. + state ateapipb.ActorState + // seed further shapes the stored actor. + seed func(*ateapipb.Actor) + // terminateErr is what the atelet answers, if anything. + terminateErr error + wantTerminate bool + }{ + { + name: "running actor is reclaimed", + state: ateapipb.ActorState_ACTOR_STATE_RUNNING, + wantTerminate: true, + }, + { + // An actor resumed from a local snapshot keeps Locality on that + // snapshot while running. When its worker pod vanishes, it crashes + // and its node state is reclaimed along with any local snapshot. + name: "running actor with a local snapshot is reclaimed", + state: ateapipb.ActorState_ACTOR_STATE_RUNNING, + seed: withPausedShape(t, "node-1"), + wantTerminate: true, + }, + { + // A cleanly suspended actor already ran Checkpoint's teardown, + // volume unmount, and resetActorDirs, and stays resumable. + name: "cleanly suspended actor is left alone", + state: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + wantTerminate: false, + }, + { + // An unreachable or unhappy node must not wedge deregistration: + // the worker still goes, and the actor's revert or delete collects + // what this could not. + name: "a failing terminate does not fail the delete", + state: ateapipb.ActorState_ACTOR_STATE_RUNNING, + terminateErr: apierror.Internal("atelet is having a bad day"), + wantTerminate: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + persistence, cleanup := storetest.SetupTestStore(t) + t.Cleanup(cleanup) + + atelet := &capturingTerminator{err: tt.terminateErr} + wf := NewWorkerWorkflow(persistence, newNodeAteletDialer(t, atelet)) + + seedAPIWorker(t, ctx, persistence, validWorker(apiWorkerName)) + seeds := []func(*ateapipb.Actor){} + if tt.seed != nil { + seeds = append(seeds, tt.seed) + } + actor := seedAPIActor(t, ctx, persistence, tt.state, seeds...) + assignAPIWorker(t, ctx, persistence, apiWorkerName, actor.GetMetadata().GetUid()) + + if _, err := wf.DeleteWorker(ctx, apiWorkerName, store.DeletePreconditions{}); err != nil { + t.Fatalf("DeleteWorker() failed: %v", err) + } + + got := atelet.requests() + if !tt.wantTerminate { + if len(got) != 0 { + t.Fatalf("atelet received %d Terminate calls, want none: %v", len(got), got) + } + return + } + if len(got) != 1 { + t.Fatalf("atelet received %d Terminate calls, want exactly 1", len(got)) + } + req := got[0] + // The request names the actor and the worker pod UID it ran on, so + // atelet can check whether that worker is still present. + if req.GetActorUid() != actor.GetMetadata().GetUid() { + t.Errorf("Terminate actor_uid = %q, want %q", req.GetActorUid(), actor.GetMetadata().GetUid()) + } + if req.GetAtespace() != apiActorRef.Atespace || req.GetActorName() != apiActorRef.Name { + t.Errorf("Terminate named %s/%s, want %s", req.GetAtespace(), req.GetActorName(), apiActorRef) + } + if want := validWorker(apiWorkerName).GetWorkerPodUid(); req.GetWorkerPodUid() != want { + t.Errorf("Terminate worker_pod_uid = %q, want the worker's pod uid %q", req.GetWorkerPodUid(), want) + } + }) + } +} + +// A suspend holds the actor's lease from writing its snapshot to finalizing +// it. A release that reclaimed in between would race the checkpoint, so the +// release waits for the lease: while it is held, the actor and its worker are +// left alone and the delete fails ABORTED for a retry. Once it is free no +// suspend is running and the release reclaims and crashes the actor. +func TestDeleteWorkerWorkflow_ReleasesUnderTheActorLease(t *testing.T) { + ctx := context.Background() + persistence, cleanup := storetest.SetupTestStore(t) + t.Cleanup(cleanup) + + atelet := &capturingTerminator{} + wf := NewWorkerWorkflow(persistence, newNodeAteletDialer(t, atelet)) + + seedAPIWorker(t, ctx, persistence, validWorker(apiWorkerName)) + actor := seedAPIActor(t, ctx, persistence, ateapipb.ActorState_ACTOR_STATE_SUSPENDING, func(a *ateapipb.Actor) { + withPauseInProgress(t)(a) + a.Status.AssignedNode = "node-1" + }) + assignAPIWorker(t, ctx, persistence, apiWorkerName, actor.GetMetadata().GetUid()) + + lease, err := persistence.AcquireLease(ctx, actorLeaseKey(apiActorRef)) + if err != nil { + t.Fatalf("AcquireLease() failed: %v", err) + } + _, err = wf.DeleteWorker(ctx, apiWorkerName, store.DeletePreconditions{}) + if got := atelet.requests(); len(got) != 0 { + t.Errorf("atelet received %d Terminate calls while the actor's lease was held, want none", len(got)) + } + if got := apierror.Code(err); got != codes.Aborted { + t.Errorf("DeleteWorker() with the actor's lease held = %v, want ABORTED", err) + } + held := mustGetActor(t, ctx, persistence) + if got := held.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDING { + t.Errorf("actor state = %v, want SUSPENDING while its lease is held", got) + } + if got := held.GetStatus().GetWorkerAssignment().GetWorker().GetName(); got != apiWorkerName { + t.Errorf("actor points at worker %q, want %q while its lease is held", got, apiWorkerName) + } + if _, err := persistence.GetWorker(ctx, apiWorkerName); err != nil { + t.Errorf("GetWorker() = %v, want the worker kept for the retry", err) + } + + lease.Close() + if _, err := wf.DeleteWorker(ctx, apiWorkerName, store.DeletePreconditions{}); err != nil { + t.Fatalf("DeleteWorker() once the lease is free failed: %v", err) + } + if got := atelet.requests(); len(got) != 1 { + t.Errorf("atelet received %d Terminate calls once the lease was free, want 1", len(got)) + } + if got := mustGetActor(t, ctx, persistence).GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_CRASHED { + t.Errorf("actor state = %v, want CRASHED once the lease is free", got) + } +} + +// One actor with an operation in progress does not hold up the rest of the +// worker's actors: they are reclaimed and crashed in the same call, which then +// fails ABORTED so the delete is retried for the busy one. +func TestDeleteWorkerWorkflow_BusyActorDoesNotHoldUpOthers(t *testing.T) { + ctx := context.Background() + persistence, cleanup := storetest.SetupTestStore(t) + t.Cleanup(cleanup) + + atelet := &capturingTerminator{} + wf := NewWorkerWorkflow(persistence, newNodeAteletDialer(t, atelet)) + + seedAPIWorker(t, ctx, persistence, validWorker(apiWorkerName)) + var refs []resources.ActorRef + var uids []string + for _, name := range []string{"busy", "free"} { + actor := seedAPIActor(t, ctx, persistence, ateapipb.ActorState_ACTOR_STATE_RUNNING, func(a *ateapipb.Actor) { + a.Metadata.Name = name + }) + ref := resources.ActorRefFromActor(actor) + if err := persistence.BindActorToWorker(ctx, apiWorkerName, &ateapipb.ActorAssignment{ + Actor: &ateapipb.ObjectRef{Atespace: ref.Atespace, Name: ref.Name}, + ActorUid: actor.GetMetadata().GetUid(), + }, nil); err != nil { + t.Fatalf("assigning %s to worker %s: %v", ref, apiWorkerName, err) + } + refs = append(refs, ref) + uids = append(uids, actor.GetMetadata().GetUid()) + } + + lease, err := persistence.AcquireLease(ctx, actorLeaseKey(refs[0])) + if err != nil { + t.Fatalf("AcquireLease() failed: %v", err) + } + defer lease.Close() + _, err = wf.DeleteWorker(ctx, apiWorkerName, store.DeletePreconditions{}) + if got := apierror.Code(err); got != codes.Aborted { + t.Errorf("DeleteWorker() with one actor's lease held = %v, want ABORTED", err) + } + got := atelet.requests() + if len(got) != 1 || got[0].GetActorUid() != uids[1] { + t.Errorf("atelet received %v, want exactly one Terminate, for the free actor %s", got, uids[1]) + } + for i, want := range []ateapipb.ActorState{ateapipb.ActorState_ACTOR_STATE_RUNNING, ateapipb.ActorState_ACTOR_STATE_CRASHED} { + actor, err := persistence.GetActor(ctx, refs[i]) + if err != nil { + t.Fatalf("GetActor(%s) failed: %v", refs[i], err) + } + if actor.GetStatus().GetState() != want { + t.Errorf("actor %s state = %v, want %v", refs[i], actor.GetStatus().GetState(), want) + } + } +} + +// An unmount that hangs, such as one against a dead NFS server, or an +// unreachable atelet returning Unavailable, must not wedge deregistration. +// Each reclaim gets w.reclaimTimeout, and once one times out or returns +// Unavailable the worker's other actors, and subsequent worker deletes on that +// node within w.reclaimTimeout, leave theirs to their revert or delete. Once +// w.reclaimTimeout elapses, a later delete tries the node again. +func TestDeleteWorkerWorkflow_HungReclaimDoesNotWedgeDelete(t *testing.T) { + for _, tt := range []struct { + name string + hang bool + err error + }{ + {name: "timeout", hang: true}, + {name: "unavailable", err: apierror.Unavailable("atelet shutting down")}, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + persistence, cleanup := storetest.SetupTestStore(t) + t.Cleanup(cleanup) + + now := time.Now() + atelet := &capturingTerminator{hang: tt.hang, err: tt.err} + wf := NewWorkerWorkflow(persistence, newNodeAteletDialer(t, atelet)) + wf.reclaimTimeout = 50 * time.Millisecond + wf.now = func() time.Time { return now } + + bindRunningActor := func(workerName, actorName string) resources.ActorRef { + actor := seedAPIActor(t, ctx, persistence, ateapipb.ActorState_ACTOR_STATE_RUNNING, func(a *ateapipb.Actor) { + a.Metadata.Name = actorName + a.Status.WorkerAssignment.Worker = workerRef(workerName) + a.Status.WorkerAssignment.WorkerPodUid = workerName + }) + ref := resources.ActorRefFromActor(actor) + if err := persistence.BindActorToWorker(ctx, workerName, &ateapipb.ActorAssignment{ + Actor: &ateapipb.ObjectRef{Atespace: ref.Atespace, Name: ref.Name}, + ActorUid: actor.GetMetadata().GetUid(), + }, nil); err != nil { + t.Fatalf("assigning %s to worker %s: %v", ref, workerName, err) + } + return ref + } + + seedAPIWorker(t, ctx, persistence, validWorker(apiWorkerName)) + refs := []resources.ActorRef{ + bindRunningActor(apiWorkerName, "actor-1"), + bindRunningActor(apiWorkerName, "actor-2"), + } + + // A deadline, as the RPC has. A Terminate without a timeout of its own runs + // into it, and the release after it fails. + rpcCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if _, err := wf.DeleteWorker(rpcCtx, apiWorkerName, store.DeletePreconditions{}); err != nil { + t.Fatalf("DeleteWorker(first) failed: %v", err) + } + if got := len(atelet.requests()); got != 1 { + t.Errorf("atelet received %d Terminate calls on first worker delete, want 1: the first failure gives up on the node", got) + } + + // A second worker deleted from the same node within w.reclaimTimeout + // skips Terminate without waiting out another timeout. + seedAPIWorker(t, ctx, persistence, validWorker(apiOtherWorkerName)) + refs = append(refs, bindRunningActor(apiOtherWorkerName, "actor-3")) + if _, err := wf.DeleteWorker(rpcCtx, apiOtherWorkerName, store.DeletePreconditions{}); err != nil { + t.Fatalf("DeleteWorker(second) failed: %v", err) + } + if got := len(atelet.requests()); got != 1 { + t.Errorf("atelet received %d Terminate calls across two worker deletes on the same node, want 1", got) + } + + for _, ref := range refs { + got, err := persistence.GetActor(ctx, ref) + if err != nil { + t.Fatalf("GetActor(%s) failed: %v", ref, err) + } + if got.GetStatus().GetWorkerAssignment() != nil { + t.Errorf("actor %s still points at a worker: %v", ref, got.GetStatus().GetWorkerAssignment()) + } + } + + // Once w.reclaimTimeout elapses, a subsequent worker delete tries the node again. + now = now.Add(wf.reclaimTimeout) + atelet.mu.Lock() + atelet.hang = false + atelet.err = nil + atelet.mu.Unlock() + seedAPIWorker(t, ctx, persistence, validWorker(apiWorkerName)) + bindRunningActor(apiWorkerName, "actor-4") + if _, err := wf.DeleteWorker(rpcCtx, apiWorkerName, store.DeletePreconditions{}); err != nil { + t.Fatalf("DeleteWorker(after timeout) failed: %v", err) + } + if got := len(atelet.requests()); got != 2 { + t.Errorf("atelet received %d Terminate calls after w.reclaimTimeout elapsed, want 2", got) + } + }) + } +} + +// capturingTerminator is an atelet that records the Terminate calls it is sent +// and answers them with err, or, with hang set, blocks until the caller gives +// up, like an atelet stuck in an unmount. +type capturingTerminator struct { + ateletpb.UnimplementedAteletServer + + mu sync.Mutex + err error + hang bool + reqs []*ateletpb.TerminateRequest +} + +func (f *capturingTerminator) Terminate(ctx context.Context, req *ateletpb.TerminateRequest) (*ateletpb.TerminateResponse, error) { + f.mu.Lock() + f.reqs = append(f.reqs, proto.Clone(req).(*ateletpb.TerminateRequest)) + hang, err := f.hang, f.err + f.mu.Unlock() + if hang { + <-ctx.Done() + return nil, ctx.Err() + } + if err != nil { + return nil, err + } + return &ateletpb.TerminateResponse{}, nil +} + +func (f *capturingTerminator) requests() []*ateletpb.TerminateRequest { + f.mu.Lock() + defer f.mu.Unlock() + return slices.Clone(f.reqs) +} diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index 99f19ca59b..03966a7259 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -102,6 +102,8 @@ type fakeWorker struct { // onRestore, if non-nil, runs inside RestoreWorkload after it recorded // the request, before it fails with restoreErr or succeeds. onRestore func() + // terminations counts TerminateWorkload calls. + terminations atomic.Int32 } func (f *fakeWorker) recordFidelity(rpc string, fidelity ateworkerpb.SnapshotFidelity) { @@ -167,6 +169,7 @@ func (f *fakeWorker) RestoreWorkload(_ context.Context, req *ateworkerpb.Restore } func (f *fakeWorker) TerminateWorkload(_ context.Context, req *ateworkerpb.TerminateWorkloadRequest) (*ateworkerpb.TerminateWorkloadResponse, error) { + f.terminations.Add(1) f.recordActorDirs("TerminateWorkload", req.GetActorDirs()) if f.onTerminate != nil { if err := f.onTerminate(); err != nil { @@ -490,7 +493,7 @@ func TestRestoreUsesRequestSandboxRuntime(t *testing.T) { s.anonGCSClient = fakeObjectStorage{data: restoreRelease.tarball} advertiseRuntime(t, s, restoreRec) - if _, err := s.Restore(ctx, &ateletpb.RestoreRequest{ + restoreReq := &ateletpb.RestoreRequest{ Atespace: atespace, ActorName: actorName, ActorUid: actorUID, @@ -500,11 +503,32 @@ func TestRestoreUsesRequestSandboxRuntime(t *testing.T) { SandboxRuntime: runtimeFor(t, restoreRec), Spec: spec, Snapshot: lifecycleSnapshot(actorUID, snapshotName, 1, ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY), - }); err != nil { - t.Fatalf("Restore: %v", err) } + // Restore writes the sandbox record before calling RestoreWorkload, so a + // Restore that fails or times out after reaching the worker still leaves + // the record for a subsequent Terminate to tear the sandbox down. + if err := os.Remove(ateletpath.ActorSandboxAssetsFile(actorUID)); err != nil { + t.Fatalf("removing sandbox record: %v", err) + } + worker.restoreErr = status.Error(codes.DeadlineExceeded, "restore timed out") + if _, err := s.Restore(ctx, restoreReq); err == nil { + t.Fatal("Restore succeeded when RestoreWorkload failed") + } got, err := readSandboxRecord(actorUID) + if err != nil { + t.Fatalf("reading on-node sandbox record after failed RestoreWorkload: %v", err) + } + if sha, want := got.Assets[gvisorAssetName].SHA256, restoreRelease.entry.SHA256; sha != want { + t.Errorf("sandbox record gvisor sha256 after failed RestoreWorkload = %q, want %q", sha, want) + } + + worker.restoreErr = nil + if _, err := s.Restore(ctx, restoreReq); err != nil { + t.Fatalf("Restore: %v", err) + } + + got, err = readSandboxRecord(actorUID) if err != nil { t.Fatalf("reading on-node sandbox record: %v", err) } @@ -1132,6 +1156,247 @@ func TestTerminateWithoutWorkerPodUID(t *testing.T) { } } +// A deleted worker pod takes its worker with it and leaves the actor's state on +// the node. The worker delete reclaims it with a Terminate that still names +// that worker, so one that refuses to run without a live worker leaves the +// state until the actor is deleted or reverted. All three ways the sandbox can +// be absent are covered: the pod's socket gone with it, a socket that outlived +// its process (Unavailable from TerminateWorkload), and an actor with no +// sandbox record on disk. +func TestTerminateReclaimsStateWithoutALiveSandbox(t *testing.T) { + const ( + atespace = "ate-demo" + actorName = "counter" + actorUID = "01234567-89ab-cdef-0123-456789abcdef" + workerPodUID = "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f" + ) + + tests := []struct { + name string + // setup arranges the absence under test. + setup func(t *testing.T, s *Atelet) + }{ + { + name: "worker socket gone with its pod", + setup: func(t *testing.T, _ *Atelet) { + orig := workerSocketPath + missing := filepath.Join(t.TempDir(), "worker.sock") + workerSocketPath = func(string) string { return missing } + t.Cleanup(func() { workerSocketPath = orig }) + // A sandbox record whose asset is not in the cache would fail + // ensureSandboxAssets (anonGCSClient is nil) if Terminate read + // the record before checking whether the worker socket exists. + if err := writeSandboxRecord(actorUID, &sandboxAssetsRecord{ + SandboxClass: "gvisor", + Assets: map[string]assetEntry{gvisorAssetName: {URL: "gs://test-bucket/gvisor-release.tar.gz", SHA256: strings.Repeat("a", 64)}}, + }); err != nil { + t.Fatalf("writing sandbox record: %v", err) + } + }, + }, + { + // After an ungraceful exit the socket can outlive the pod, or the + // worker can shut down while TerminateWorkload is in flight; + // TerminateWorkload then fails with Unavailable. + name: "worker socket outlived its process", + setup: func(t *testing.T, s *Atelet) { + release := newGVisorTestRelease(t) + s.anonGCSClient = fakeObjectStorage{data: release.tarball} + rec := testSandboxRecord(release) + if err := writeSandboxRecord(actorUID, rec); err != nil { + t.Fatalf("writing sandbox record: %v", err) + } + serveFakeWorker(t, &fakeWorker{ + onTerminate: func() error { + return status.Error(codes.Unavailable, "worker shutting down") + }, + }) + }, + }, + { + // Run and Restore write the record before starting the workload, + // and an earlier Terminate removes it with the actor directory. + // Without one, no sandbox was started. + name: "worker reachable but the actor has no sandbox record", + setup: func(t *testing.T, _ *Atelet) { + serveFakeWorker(t, &fakeWorker{}) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + useTempNodeDirs(t) + ctx := t.Context() + + stateDirs := []string{ + ateletpath.DurableDirVolumeMountsDir(actorUID), + ateletpath.CheckpointStateDir(actorUID), + ateletpath.RestoreStateDir(actorUID), + } + for _, dir := range stateDirs { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("seeding %s: %v", dir, err) + } + if err := os.WriteFile(filepath.Join(dir, "payload"), []byte("actor state"), 0o600); err != nil { + t.Fatalf("seeding %s: %v", dir, err) + } + } + snapshotDir := ateletpath.LocalSnapshotDir(actorUID, "pause-snap-1") + if err := os.MkdirAll(snapshotDir, 0o755); err != nil { + t.Fatalf("seeding %s: %v", snapshotDir, err) + } + + s := &Atelet{ + workerDialer: newWorkerDialer(1), + sandboxAssetCache: newTestAssetCache(t), + systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), + snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), + } + tt.setup(t, s) + if _, err := s.Terminate(ctx, &ateletpb.TerminateRequest{ + Atespace: atespace, + ActorName: actorName, + ActorUid: actorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + WorkerPodUid: workerPodUID, + Spec: &ateletpb.WorkloadSpec{}, + }); err != nil { + t.Fatalf("Terminate: %v", err) + } + + // Terminate removes both the actor and snapshot directories + // outright rather than emptying the state dirs in place, so the + // seeded payloads go with them and both trees are the assertion. + assertDirGone(t, "actor dir", ateletpath.ActorPath(actorUID)) + assertDirGone(t, "actor snapshot dir", ateletpath.ActorSnapshotPath(actorUID)) + }) + } +} + +// TestTerminateCanBeRetried pins that a repeated Terminate succeeds, both when +// a single caller retries and when DeleteActor or RevertActor sends a second +// Terminate after DeleteWorker's best-effort Terminate. +func TestTerminateCanBeRetried(t *testing.T) { + for _, tt := range []struct { + name string + // failFirst fails the first Terminate after its worker teardown, the way + // a volume that has not finished unmounting does. + failFirst bool + // secondWithoutWorker sends the second Terminate with an empty + // WorkerPodUid, as DeleteActor and RevertActor do after DeleteWorker + // has already cleared the actor's worker assignment. + secondWithoutWorker bool + // wantTeardowns is how many TerminateWorkload calls worker sees. A retry + // tears down again only when it names a worker and the sandbox record + // survived the first call. + wantTeardowns int32 + }{ + {name: "after a terminate that succeeded", wantTeardowns: 1}, + {name: "after a terminate that failed past the teardown", failFirst: true, wantTeardowns: 2}, + {name: "delete or revert after a worker-delete terminate that succeeded", secondWithoutWorker: true, wantTeardowns: 1}, + {name: "delete or revert after a worker-delete terminate that failed partway", failFirst: true, secondWithoutWorker: true, wantTeardowns: 1}, + } { + t.Run(tt.name, func(t *testing.T) { + a := runPluginActor(t) + ctx := t.Context() + + snap := a.snapshot("pause-snap-1", 1) + if _, err := a.checkpoint(t, snap, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); err != nil { + t.Fatalf("Checkpoint: %v", err) + } + + // A populated volume directory, which resetActorDirs refuses to + // delete through. + straggler := filepath.Join(ateletpath.ExternalVolumeHostPath(pluginActorUID, "data"), "file") + if tt.failFirst { + if err := os.MkdirAll(filepath.Dir(straggler), 0o755); err != nil { + t.Fatalf("seeding %s: %v", straggler, err) + } + if err := os.WriteFile(straggler, []byte("still mounted"), 0o600); err != nil { + t.Fatalf("seeding %s: %v", straggler, err) + } + } + + firstReq := &ateletpb.TerminateRequest{ + Atespace: "ate-demo", + ActorName: "counter", + ActorUid: pluginActorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + WorkerPodUid: pluginWorkerPodUID, + Spec: a.spec, + } + _, err := a.s.Terminate(ctx, firstReq) + if tt.failFirst { + if err == nil { + t.Fatal("first Terminate succeeded through a populated volume directory") + } + // The unmount finishes before the second Terminate. + if err := os.Remove(straggler); err != nil { + t.Fatalf("removing %s: %v", straggler, err) + } + } else if err != nil { + t.Fatalf("first Terminate: %v", err) + } + + secondReq := proto.Clone(firstReq).(*ateletpb.TerminateRequest) + if tt.secondWithoutWorker { + secondReq.WorkerPodUid = "" + } + if _, err := a.s.Terminate(ctx, secondReq); err != nil { + t.Fatalf("second Terminate: %v", err) + } + if got := a.worker.terminations.Load(); got != tt.wantTeardowns { + t.Errorf("worker saw %d TerminateWorkload calls, want %d", got, tt.wantTeardowns) + } + assertDirGone(t, "actor dir", ateletpath.ActorPath(pluginActorUID)) + assertDirGone(t, "actor snapshot dir", ateletpath.ActorSnapshotPath(pluginActorUID)) + }) + } +} + +// TestTerminateWorkerFailurePreservesState checks that when a live worker +// returns Internal from TerminateWorkload, Terminate fails without resetting +// actor directories or pruning local snapshots. +func TestTerminateWorkerFailurePreservesState(t *testing.T) { + a := runPluginActor(t) + snap := a.snapshot("pause-snap-1", 1) + if _, err := a.checkpoint(t, snap, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); err != nil { + t.Fatalf("Checkpoint: %v", err) + } + + durableFile := filepath.Join(ateletpath.DurableDirVolumeMountPoint(pluginActorUID, "work"), "payload") + if err := os.MkdirAll(filepath.Dir(durableFile), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(durableFile, []byte("runtime-payload"), 0o600); err != nil { + t.Fatal(err) + } + + a.worker.onTerminate = func() error { return apierror.Internal("sandbox busy") } + _, err := a.s.Terminate(t.Context(), &ateletpb.TerminateRequest{ + Atespace: "ate-demo", + ActorName: "counter", + ActorUid: pluginActorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + WorkerPodUid: pluginWorkerPodUID, + Spec: a.spec, + }) + if got := apierror.Code(err); got != codes.Internal { + t.Fatalf("Terminate error = %v (code %v), want Internal", err, got) + } + if _, err := os.Stat(durableFile); err != nil { + t.Errorf("durable-dir payload removed after failed TerminateWorkload: %v", err) + } + snapshotFile := filepath.Join(ateletpath.LocalSnapshotDir(pluginActorUID, "pause-snap-1"), "checkpoint.img") + if _, err := os.Stat(snapshotFile); err != nil { + t.Errorf("local snapshot removed after failed TerminateWorkload: %v", err) + } +} + // pluginActorUID is the actor runPluginActor runs. const ( pluginActorUID = "01234567-89ab-cdef-0123-456789abcdef" diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 2dc22fbad6..f5fc795f7f 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -1613,9 +1613,9 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res if err := resetSandboxAssetsDir(actorUID); err != nil { return nil, err } - // The record below is written last, so until then it describes the - // previous activation; a failed restore must not leave this one's - // asset links behind for a later Terminate to take for that record's. + // The record below is written before RestoreWorkload, so until then it + // describes the previous activation; a failed restore must not leave this + // one's asset links behind for a later Terminate to take for that record's. // Nor its snapshot files: staged ones are links into the snapshot cache // or copies, and either would hold the cache's entries, or disk, until // the next lifecycle call resets the directory. @@ -1765,6 +1765,13 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res return nil, apierror.InvalidArgument("invalid workload spec: %v", err) } + // Record the sandbox binaries running the guest on-node before + // RestoreWorkload, as Run does before RunWorkload, so a Checkpoint or + // Terminate can resolve the runtime as soon as the sandbox is started. + if err := writeSandboxRecord(actorUID, runtimeRec); err != nil { + return nil, err + } + actorDirs := ateletpath.ActorDirs(actorUID) actorDirs.RestoreDir = checkpointDir @@ -1791,14 +1798,6 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res return nil, s.boundRetryableRestoreError(actorUID, req.GetSnapshot().GetSnapshotUuid(), workerRestoreError(ctx, err)) } - // Record the sandbox binaries actually running the guest on-node so a - // subsequent Checkpoint of this restored actor can re-pin the same version - // (Checkpoint overwrites the identity fields from its own request). - if err := writeSandboxRecord(actorUID, runtimeRec); err != nil { - // Note: crash the actor right away, if we cannot write the sandbox record now, we will not be able to checkpoint it later. - return nil, err - } - return &ateletpb.RestoreResponse{}, nil } @@ -1945,43 +1944,8 @@ func (s *Atelet) Terminate(ctx context.Context, req *ateletpb.TerminateRequest) endTerminate := s.snapshots.BeginTerminate(actorUID) defer endTerminate() - if req.GetWorkerPodUid() != "" { - var assetPaths map[string]string - sandboxRec, err := readSandboxRecord(actorUID) - if err != nil { - return nil, fmt.Errorf("failed to read sandbox record during terminate (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) - } - paths, err := s.ensureSandboxAssets(ctx, sandboxRec, ateletpath.SandboxAssetsDir(actorUID)) - if err != nil { - return nil, fmt.Errorf("failed to ensure sandbox assets during terminate (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) - } - assetPaths = paths - - client, err := s.dialWorker(ctx, req.GetWorkerPodUid()) - if err != nil { - return nil, fmt.Errorf("failed to dial worker for terminate (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) - } - - spec, err := buildWorkerWorkloadSpec(req.GetSpec(), nil) - if err != nil { - return nil, apierror.InvalidArgument("invalid workload spec: %v", err) - } - if _, err := client.TerminateWorkload(ctx, &ateworkerpb.TerminateWorkloadRequest{ - Atespace: req.GetAtespace(), - ActorName: req.GetActorName(), - ActorUid: req.GetActorUid(), - ActorTemplateAtespace: req.GetActorTemplateAtespace(), - ActorTemplateName: req.GetActorTemplateName(), - RunscPath: runscPathFor(assetPaths), - Spec: spec, - ActorDirs: ateletpath.ActorDirs(actorUID), - }); err != nil { - if status.Code(err) == codes.NotFound { - slog.InfoContext(ctx, "workload not found on worker during terminate", slog.Any("actor", actorRef), slog.String("actorUID", actorUID)) - } else { - return nil, fmt.Errorf("failed calling worker TerminateWorkload (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) - } - } + if err := s.terminateWorkloadOnWorker(ctx, req, actorRef, actorUID); err != nil { + return nil, err } // Deregister after teardown succeeds @@ -2015,6 +1979,98 @@ func (s *Atelet) Terminate(ctx context.Context, req *ateletpb.TerminateRequest) return &ateletpb.TerminateResponse{}, nil } +// terminateWorkloadOnWorker tears the workload down on the worker hosting it. +// +// A Terminate can legitimately arrive with no worker to tear down: none named, +// because the actor no longer holds a worker (the delete or revert of a +// suspended or crashed actor), or one named that is gone, its worker pod +// deleted, evicted, or its node drained. Either way the actor's state is still +// on the node, so the teardown is skipped rather than failing the RPC and +// leaving the actor state behind. +// +// A worker that is present but rejects the call still fails: that is a live +// sandbox this could not tear down, and reclaiming its directories underneath +// it would be worse than leaving them. +func (s *Atelet) terminateWorkloadOnWorker(ctx context.Context, req *ateletpb.TerminateRequest, actorRef resources.ActorRef, actorUID string) error { + workerPodUID := req.GetWorkerPodUid() + if workerPodUID == "" { + // No worker named: the actor has no worker pod left, only state on + // this node, such as a suspended or crashed actor's on delete or revert. + slog.InfoContext(ctx, "no worker named during terminate; reclaiming the actor's node state without a sandbox teardown", + slog.Any("actor", actorRef), slog.String("actorUID", actorUID)) + return nil + } + // The socket is the worker's liveness. The worker removes its socket on a + // graceful exit, and workerSandboxSyncer removes the pod's directory once + // the pod is deleted, so the common pod-is-gone path finds no socket and + // does no other work; in particular, it never reaches ensureSandboxAssets, + // which would fetch binaries for a sandbox that no longer exists. After an + // ungraceful exit the socket can outlive the pod until the syncer removes + // the directory: that path does reach ensureSandboxAssets, and since the + // dial is lazy, TerminateWorkload then fails with Unavailable, which is + // tolerated below. + if _, err := os.Stat(workerSocketPath(workerPodUID)); err != nil { + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("failed to stat the worker socket during terminate (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) + } + slog.InfoContext(ctx, "worker is gone during terminate; reclaiming the actor's node state without a sandbox teardown", + slog.Any("actor", actorRef), slog.String("actorUID", actorUID), slog.String("workerPodUID", workerPodUID)) + return nil + } + + sandboxRec, err := readSandboxRecord(actorUID) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + // Run and Restore write the record before starting the workload, + // and an earlier Terminate removes it with the actor directory. + // Without it, no sandbox was started. + slog.InfoContext(ctx, "no sandbox record during terminate; reclaiming the actor's node state without a sandbox teardown", + slog.Any("actor", actorRef), slog.String("actorUID", actorUID)) + return nil + } + return fmt.Errorf("failed to read sandbox record during terminate (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) + } + assetPaths, err := s.ensureSandboxAssets(ctx, sandboxRec, ateletpath.SandboxAssetsDir(actorUID)) + if err != nil { + return fmt.Errorf("failed to ensure sandbox assets during terminate (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) + } + + client, err := s.dialWorker(ctx, workerPodUID) + if err != nil { + return fmt.Errorf("failed to dial worker for terminate (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) + } + + spec, err := buildWorkerWorkloadSpec(req.GetSpec(), nil) + if err != nil { + return apierror.InvalidArgument("invalid workload spec: %v", err) + } + if _, err := client.TerminateWorkload(ctx, &ateworkerpb.TerminateWorkloadRequest{ + Atespace: req.GetAtespace(), + ActorName: req.GetActorName(), + ActorUid: req.GetActorUid(), + ActorTemplateAtespace: req.GetActorTemplateAtespace(), + ActorTemplateName: req.GetActorTemplateName(), + RunscPath: runscPathFor(assetPaths), + Spec: spec, + ActorDirs: ateletpath.ActorDirs(actorUID), + }); err != nil { + switch status.Code(err) { + case codes.NotFound: + slog.InfoContext(ctx, "workload not found on worker during terminate", slog.Any("actor", actorRef), slog.String("actorUID", actorUID)) + case codes.Unavailable: + // The socket outlived the process serving it (a worker shutting + // down while this RPC was in flight, or a stale socket file). Same + // situation as a missing socket: there is nothing left to tear + // down, and the state on disk still has to go. + slog.InfoContext(ctx, "worker unreachable during terminate; reclaiming the actor's node state without a sandbox teardown", + slog.Any("actor", actorRef), slog.String("actorUID", actorUID), slog.Any("err", err)) + default: + return fmt.Errorf("failed calling worker TerminateWorkload (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) + } + } + return nil +} + // checkLocalSnapshotFiles verifies each snapshot file exists in dir as a // regular file. Lstat via os.Root, so a symlink cannot point worker outside the // snapshot. From c3875751a6026ca620ee67641fb89f48b39f39fd Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Wed, 21 Oct 2026 23:08:39 +0000 Subject: [PATCH 04/12] atelet: invalidate node-local snapshots across node reboots (#3050) Not every file of a node-local snapshot is synced to disk, so a local copy written before an unclean node reboot may be left empty or truncated even when its manifest is intact. Stamp the node's boot ID into the manifest of each local snapshot copy (omitting it from uploaded manifests). At atelet startup, delete all pre-reboot local snapshots on disk, detaching and reporting any that are already durable. Restore and upload never read a local copy from an earlier boot, in case one is left behind. A local snapshot not yet uploaded is lost on reboot. An actor that needs durability across planned node maintenance should suspend with an immediate upload. AI use: AI-assisted, I reviewed and tested it. - [x] Tests pass - [x] Docs updated --- cmd/atelet/bootid.go | 57 ++++ cmd/atelet/bootid_linux_test.go | 28 ++ cmd/atelet/bootid_test.go | 91 ++++++ .../internal/snapshottracker/background.go | 67 ++++- .../snapshottracker/background_test.go | 258 ++++++++++++++++++ .../internal/snapshottracker/tracker.go | 15 + cmd/atelet/lifecycle_test.go | 167 ++++++++++++ cmd/atelet/main.go | 72 +++-- cmd/atelet/main_test.go | 48 ++++ cmd/atelet/sandbox_assets.go | 6 + internal/proto/ateletpb/atelet.pb.go | 7 +- internal/proto/ateletpb/atelet.proto | 11 +- internal/proto/ateletpb/atelet_grpc.pb.go | 8 + 13 files changed, 802 insertions(+), 33 deletions(-) create mode 100644 cmd/atelet/bootid.go create mode 100644 cmd/atelet/bootid_linux_test.go create mode 100644 cmd/atelet/bootid_test.go diff --git a/cmd/atelet/bootid.go b/cmd/atelet/bootid.go new file mode 100644 index 0000000000..2f75216fce --- /dev/null +++ b/cmd/atelet/bootid.go @@ -0,0 +1,57 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + "strings" + + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" +) + +// bootIDPath is where Linux exposes the boot ID: a random UUID generated once +// per boot, so it changes on every reboot, clean or not, and on nothing else. +// It is not namespaced, so atelet reads the node's. +const bootIDPath = "/proc/sys/kernel/random/boot_id" + +// readBootID returns the boot ID read from path. It is never empty, so a local +// snapshot whose manifest records none never matches it. +func readBootID(path string) (string, error) { + b, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("while reading the boot ID: %w", err) + } + id := strings.TrimSpace(string(b)) + if id == "" { + return "", fmt.Errorf("boot ID file %s is empty", path) + } + return id, nil +} + +// localSnapshotBootCheck returns the snapshot tracker's check of whether an +// actor's local snapshot was written on the boot bootID: whether the +// snapshot's manifest records bootID. A manifest that is missing or does not +// parse records none. +func localSnapshotBootCheck(bootID string) func(actorUID, snapshotUUID string) bool { + return func(actorUID, snapshotUUID string) bool { + manifest, err := readSnapshotManifest(ateletpath.LocalSnapshotDir(actorUID, snapshotUUID)) + if err != nil { + return false + } + rec, err := unmarshalSandboxRecord(manifest) + return err == nil && rec.BootID == bootID + } +} diff --git a/cmd/atelet/bootid_linux_test.go b/cmd/atelet/bootid_linux_test.go new file mode 100644 index 0000000000..0283181a3c --- /dev/null +++ b/cmd/atelet/bootid_linux_test.go @@ -0,0 +1,28 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package main + +import "testing" + +// TestReadBootIDFromHost reads the boot ID atelet reads at startup, which +// only Linux exposes. +func TestReadBootIDFromHost(t *testing.T) { + got, err := readBootID(bootIDPath) + if err != nil || got == "" { + t.Errorf("readBootID(%q) = (%q, %v), want a non-empty boot ID", bootIDPath, got, err) + } +} diff --git a/cmd/atelet/bootid_test.go b/cmd/atelet/bootid_test.go new file mode 100644 index 0000000000..52046be46b --- /dev/null +++ b/cmd/atelet/bootid_test.go @@ -0,0 +1,91 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "testing" + + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" +) + +func TestReadBootID(t *testing.T) { + t.Run("reads and trims boot ID", func(t *testing.T) { + p := filepath.Join(t.TempDir(), "boot_id") + const want = "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f" + if err := os.WriteFile(p, []byte(want+"\n"), 0o600); err != nil { + t.Fatal(err) + } + got, err := readBootID(p) + if err != nil || got != want { + t.Errorf("readBootID = (%q, %v), want (%q, nil)", got, err, want) + } + }) + + t.Run("rejects empty file", func(t *testing.T) { + p := filepath.Join(t.TempDir(), "boot_id") + if err := os.WriteFile(p, []byte(" \n\t"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := readBootID(p); err == nil { + t.Error("readBootID accepted an empty boot ID file") + } + }) + + t.Run("rejects missing file", func(t *testing.T) { + if _, err := readBootID(filepath.Join(t.TempDir(), "missing")); err == nil { + t.Error("readBootID accepted a missing file") + } + }) +} + +// localSnapshotBootCheck matches a local snapshot only when its manifest +// records the given boot ID. +func TestLocalSnapshotBootCheck(t *testing.T) { + const ( + actorUID = "01234567-89ab-cdef-0123-456789abcdef" + snapshotUUID = "pause-snap-1" + ) + for _, tc := range []struct { + name string + // manifest is the manifest written beside the snapshot, or empty + // for none. + manifest string + want bool + }{ + {name: "records the current boot", manifest: `{"bootId":"boot-2"}`, want: true}, + {name: "records an earlier boot", manifest: `{"bootId":"boot-1"}`}, + {name: "records no boot", manifest: `{}`}, + {name: "does not parse", manifest: `{"bootId":`}, + {name: "is missing"}, + } { + t.Run(tc.name, func(t *testing.T) { + useTempNodeDirs(t) + if tc.manifest != "" { + dir := ateletpath.LocalSnapshotDir(actorUID, snapshotUUID) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, sandboxManifestName), []byte(tc.manifest), 0o600); err != nil { + t.Fatal(err) + } + } + if got := localSnapshotBootCheck("boot-2")(actorUID, snapshotUUID); got != tc.want { + t.Errorf("localSnapshotBootCheck(%q) = %v, want %v", "boot-2", got, tc.want) + } + }) + } +} diff --git a/cmd/atelet/internal/snapshottracker/background.go b/cmd/atelet/internal/snapshottracker/background.go index d836af2f07..f00dcd783b 100644 --- a/cmd/atelet/internal/snapshottracker/background.go +++ b/cmd/atelet/internal/snapshottracker/background.go @@ -139,13 +139,14 @@ type actorOp struct { err error } -// Start reads each actor's snapshot state file on this node, queues the actor -// for when its next background upload or detach is due and for any report due, -// so its background work resumes after an atelet restart, and starts each -// queue's workers and the report workers. With instruments, it also adds each -// actor's undurable local snapshot to their tally before any worker runs. +// Start reads each actor's snapshot state file on this node, deletes any local +// snapshot from before a node reboot, queues the actor for when its next +// background upload or detach is due and for any report due, so its background +// work resumes after an atelet restart, and starts each queue's workers and +// the report workers. With instruments, it also adds each actor's undurable +// local snapshot to their tally before any worker runs. func (t *Tracker) Start() { - if t.store != nil || t.reporting.reporter != nil || t.undurableActorsTally != nil { + if t.store != nil || t.reporting.reporter != nil || t.undurableActorsTally != nil || t.fromCurrentBoot != nil { actorUIDs, err := savedActorUIDs() if err != nil { // Best-effort: each actor's next commit queues it again. @@ -156,6 +157,7 @@ func (t *Tracker) Start() { break } if a := t.findLocked(actorUID); a != nil { + t.prunePreRebootLocalLocked(a) t.undurableActorsTally.observe(a) t.enqueueSavedReportLocked(a) t.requeueLocked(a) @@ -175,6 +177,59 @@ func (t *Tracker) Start() { } } +// prunePreRebootLocalLocked deletes the actor's local snapshots and stops +// tracking them when one it tracks was written before the node rebooted, since +// its files may be torn. A LatestLocal written on this boot stays. A +// LatestLocal already durable is reported as detached, or detached again later +// if deleting its files fails. One not yet uploaded is lost. +func (t *Tracker) prunePreRebootLocalLocked(a *actorSnapshots) { + if t.fromCurrentBoot == nil { + return + } + localPreReboot := a.LatestLocal != nil && !t.fromCurrentBoot(a.actorUID, a.LatestLocal.uuid()) + uploadingPreReboot := a.Uploading != nil && !t.fromCurrentBoot(a.actorUID, a.Uploading.uuid()) + if !localPreReboot && !uploadingPreReboot { + return + } + t.addLocked(a) + // Only LatestLocal can need keeping. Uploading is never newer than + // LatestLocal, so it is from an earlier boot whenever LatestLocal is, + // except after a detach that atelet stopped partway through deleting the + // actor's copies, since a deleted copy counts as from an earlier boot. + // That detach made LatestLocal durable first, so deleting Uploading too + // finishes it. + var keep []string + if a.LatestLocal != nil && !localPreReboot { + keep = []string{a.LatestLocal.uuid()} + } + pruneErr := PruneLocalCheckpoints(t.ctx, a.actorUID, keep...) + if pruneErr != nil { + // Restore and uploads refuse what is left, which the actor's next + // checkpoint or terminate deletes. + slog.WarnContext(t.ctx, "Failed to delete local snapshots left by an earlier boot or an interrupted detach", + slog.String("actorUID", a.actorUID), slog.Any("err", pruneErr)) + } + if uploadingPreReboot { + a.Uploading = nil + } + if localPreReboot { + switch { + case a.uploadNeeded(*a.LatestLocal): + slog.WarnContext(t.ctx, "Lost a local snapshot that was not uploaded before the node rebooted", + slog.String("actorUID", a.actorUID), slog.String("snapshot", a.LatestLocal.uuid())) + t.clearLatestLocalLocked(a) + case pruneErr != nil: + a.DetachAt = later(a.DetachAt, t.backOffLocked(a)) + default: + slog.InfoContext(t.ctx, "Deleted an uploaded local snapshot left by an earlier boot or an interrupted detach", + slog.String("actorUID", a.actorUID), slog.String("snapshot", a.LatestLocal.uuid())) + t.clearLatestLocalLocked(a) + t.markReportDueLocked(a) + } + } + t.saveLocked(a) +} + // Wait blocks until the workers and any detach a Detach call started have // stopped, which they do once the tracker's context ends. It returns ctx's // error if ctx ends first. Detach starts nothing after Wait begins. diff --git a/cmd/atelet/internal/snapshottracker/background_test.go b/cmd/atelet/internal/snapshottracker/background_test.go index 982cadca07..9eb2c356c4 100644 --- a/cmd/atelet/internal/snapshottracker/background_test.go +++ b/cmd/atelet/internal/snapshottracker/background_test.go @@ -91,6 +91,13 @@ func inMemory(tr *Tracker) bool { return ok } +// withCurrentBoot makes the tracker treat the local snapshots named in uuids as +// written since the node last booted, and every other one as written before it +// rebooted. +func withCurrentBoot(uuids ...string) Option { + return WithBootCheck(func(_, uuid string) bool { return slices.Contains(uuids, uuid) }) +} + // Start runs the workers until the tracker's context ends. Wait then returns // once they have stopped, even when a worker has taken an actor whose lock an // RPC still holds. @@ -212,6 +219,257 @@ func TestStartQueuesActorsWithSnapshotState(t *testing.T) { } } +// With WithBootCheck, Start deletes any local snapshot written before the node +// rebooted and stops tracking it: it reports an already-durable one as +// detached, and queues no background upload or detach for an undurable one. +// Local snapshots from the current boot stay. +func TestStartPrunesLocalSnapshotsFromEarlierBoot(t *testing.T) { + useTempNodeDirs(t) + writeLocalSnapshot := func(actorUID, snapUUID string) string { + dir := ateletpath.LocalSnapshotDir(actorUID, snapUUID) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "checkpoint.img"), []byte("memory"), 0o600); err != nil { + t.Fatal(err) + } + return dir + } + + for _, actorUID := range []string{"actor-suspended-old", "actor-running-old", "actor-same-boot", "actor-undurable-old"} { + if err := os.MkdirAll(ateletpath.ActorPath(actorUID), 0o700); err != nil { + t.Fatal(err) + } + } + + t0 := time.Unix(1_700_000_000, 0).Add(-testDetachDelay) + before := New(t.Context(), newFakeSnapshotStore(), clocktesting.NewFakeClock(t0), WithUploadDelay(testUploadDelay), WithDetachDelay(testDetachDelay)) + + s1 := testRef("s1", 1, t0) + before.CommitDurable("actor-suspended-old", s1) + before.CommitLocal("actor-suspended-old", s1) + suspendedOldDir := writeLocalSnapshot("actor-suspended-old", "s1") + + s2 := testRef("s2", 1, t0) + before.CommitDurable("actor-running-old", s2) + before.CommitLocal("actor-running-old", s2) + endStart, err := before.BeginStart(t.Context(), "actor-running-old") + if err != nil { + t.Fatalf("BeginStart: %v", err) + } + endStart() + runningOldDir := writeLocalSnapshot("actor-running-old", "s2") + + s3 := testRef("s3", 1, t0) + before.CommitDurable("actor-same-boot", s3) + before.CommitLocal("actor-same-boot", s3) + sameBootDir := writeLocalSnapshot("actor-same-boot", "s3") + + s4 := testRef("s4", 1, t0) + before.CommitLocal("actor-undurable-old", s4) + undurableOldDir := writeLocalSnapshot("actor-undurable-old", "s4") + + reporter := newFakeReporter() + clk := clocktesting.NewFakeClock(time.Unix(1_700_000_000, 0)) + after := New(t.Context(), newFakeSnapshotStore(), clk, + withCurrentBoot("s3"), + WithUploadDelay(testUploadDelay), + WithDetachDelay(testDetachDelay)) + setWorkers(after, 0, 0) + startReporting(after, reporter) + + for _, tc := range []struct { + actorUID string + snapUUID string + dir string + }{ + {"actor-suspended-old", "s1", suspendedOldDir}, + {"actor-running-old", "s2", runningOldDir}, + } { + if _, err := os.Stat(tc.dir); !errors.Is(err, os.ErrNotExist) { + t.Errorf("%s local snapshot dir stat = %v, want not exist", tc.actorUID, err) + } + st := after.State(tc.actorUID) + if got := uuidOf(st.GetLatestLocal()); got != "" { + t.Errorf("%s latest_local = %q, want empty", tc.actorUID, got) + } + if got := uuidOf(st.GetLatestDurable()); got != tc.snapUUID { + t.Errorf("%s latest_durable = %q, want %q", tc.actorUID, got, tc.snapUUID) + } + } + + if _, err := os.Stat(sameBootDir); err != nil { + t.Errorf("actor-same-boot local snapshot dir stat = %v, want kept", err) + } + if got := uuidOf(after.State("actor-same-boot").GetLatestLocal()); got != "s3" { + t.Errorf("actor-same-boot latest_local = %q, want s3", got) + } + + if _, err := os.Stat(undurableOldDir); !errors.Is(err, os.ErrNotExist) { + t.Errorf("actor-undurable-old local snapshot dir stat = %v, want not exist", err) + } + if got := after.State("actor-undurable-old").GetLatestLocal(); got != nil { + t.Errorf("actor-undurable-old latest_local = %v, want nil", got) + } + + var gotQueued [numOpKinds][]string + for k, q := range after.queues { + for q.Len() > 0 { + uid, _ := q.Get() + gotQueued[k] = append(gotQueued[k], uid) + q.Done(uid) + } + slices.Sort(gotQueued[k]) + } + if len(gotQueued[opUpload]) != 0 { + t.Errorf("upload queue = %v, want empty", gotQueued[opUpload]) + } + if want := []string{"actor-same-boot"}; !slices.Equal(gotQueued[opDetach], want) { + t.Errorf("detach queue = %v, want %v", gotQueued[opDetach], want) + } + + var reportedActors []string + for range 2 { + processNextReport(t, after) + r := waitReported(t, reporter) + if r.state.GetLatestLocal() != nil { + t.Errorf("report for %s carries latest_local %v, want nil", r.actorUID, r.state.GetLatestLocal()) + } + reportedActors = append(reportedActors, r.actorUID) + } + slices.Sort(reportedActors) + if want := []string{"actor-running-old", "actor-suspended-old"}; !slices.Equal(reportedActors, want) { + t.Errorf("reported detached actors = %v, want %v", reportedActors, want) + } + if n := after.reporting.queue.Len(); n != 0 { + t.Errorf("reports still queued = %d, want 0", n) + } + + t.Run("keeps unsaved detach in memory when state save fails", func(t *testing.T) { + useActorDir(t) + before := newIdleTestTracker(t, newFakeSnapshotStore()) + s1 := testRef("s1", 1, before.clock.Now()) + before.CommitDurable(trackedActorUID, s1) + before.CommitLocal(trackedActorUID, s1) + // With no local-checkpoint directory on disk, PruneLocalCheckpoints is a + // no-op and succeeds even when the actor directory is read-only, while + // saving the updated snapshot state file fails. + actorDir := ateletpath.ActorPath(trackedActorUID) + if err := os.Chmod(actorDir, 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(actorDir, 0o700) }) + + after := newStoreTracker(t, newFakeSnapshotStore(), withCurrentBoot()) + setWorkers(after, 0, 0) + after.Start() + + if got := after.State(trackedActorUID).GetLatestLocal(); got != nil { + t.Errorf("latest_local = %v, want nil kept in memory after failed save", got) + } + }) + + t.Run("schedules detach retry when pruning fails", func(t *testing.T) { + useActorDir(t) + before := newIdleTestTracker(t, newFakeSnapshotStore()) + s1 := testRef("s1", 1, before.clock.Now()) + before.CommitDurable(trackedActorUID, s1) + before.CommitLocal(trackedActorUID, s1) + endStart, err := before.BeginStart(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("BeginStart: %v", err) + } + endStart() + + dir := writeLocalSnapshot(trackedActorUID, "s1") + nested := filepath.Join(dir, "sub") + if err := os.MkdirAll(nested, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(nested, "keep"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chmod(nested, 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(nested, 0o700) }) + + after := newStoreTracker(t, newFakeSnapshotStore(), withCurrentBoot()) + setWorkers(after, 0, 0) + after.Start() + + if got := uuidOf(after.State(trackedActorUID).GetLatestLocal()); got != "s1" { + t.Errorf("latest_local = %q, want s1 still recorded when prune fails", got) + } + if got := detachDueOf(after, trackedActorUID); !got.Equal(after.clock.Now().Add(testRetryDelay)) { + t.Errorf("detachDue = %v, want %v", got, after.clock.Now().Add(testRetryDelay)) + } + }) + + t.Run("keeps current-boot LatestLocal when only Uploading is from earlier boot", func(t *testing.T) { + before := newIdleTestTracker(t, newFakeSnapshotStore()) + t0 := before.clock.Now() + before.lock(trackedActorUID) + before.saveLocked(&actorSnapshots{actorUID: trackedActorUID, actorSnapshotsRecord: actorSnapshotsRecord{ + Uploading: testRef("s1", 1, t0).clone(), + LatestLocal: testRef("s2", 2, t0.Add(time.Second)).clone(), + }}) + before.unlock(trackedActorUID) + upDir := writeLocalSnapshot(trackedActorUID, "s1") + localDir := writeLocalSnapshot(trackedActorUID, "s2") + + after := newStoreTracker(t, newFakeSnapshotStore(), withCurrentBoot("s2")) + setWorkers(after, 0, 0) + after.Start() + + if _, err := os.Stat(upDir); !errors.Is(err, os.ErrNotExist) { + t.Errorf("earlier-boot Uploading dir stat = %v, want not exist", err) + } + if _, err := os.Stat(localDir); err != nil { + t.Errorf("current-boot LatestLocal dir stat = %v, want kept", err) + } + st := after.State(trackedActorUID) + if st.GetUploading() != nil { + t.Errorf("uploading = %v, want nil", st.GetUploading()) + } + if got := uuidOf(st.GetLatestLocal()); got != "s2" { + t.Errorf("latest_local = %q, want s2", got) + } + }) + + t.Run("stops tracking an undurable snapshot when deleting it fails", func(t *testing.T) { + before := newIdleTestTracker(t, newFakeSnapshotStore()) + // Older than the upload delay, so a snapshot still tracked would count + // as overdue. + before.CommitLocal(trackedActorUID, testRef("s1", 1, before.clock.Now().Add(-2*testUploadDelay))) + dir := writeLocalSnapshot(trackedActorUID, "s1") + nested := filepath.Join(dir, "sub") + if err := os.MkdirAll(nested, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(nested, "keep"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chmod(nested, 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(nested, 0o700) }) + + inst, _ := newTestInstruments(t) + after := newStoreTracker(t, newFakeSnapshotStore(), withCurrentBoot(), WithInstruments(inst)) + setWorkers(after, 0, 0) + after.Start() + + st := after.State(trackedActorUID) + if st.GetLatestLocal() != nil || st.GetUploadDueTime() != nil { + t.Errorf("state = %v, want no latest_local and no upload due", st) + } + if counts := after.undurableActorsTally.counts(); counts != (undurableActorsCounts{}) { + t.Errorf("undurable tally = %+v, want empty", counts) + } + }) +} + // Forget cancels the actor's background operation in progress and waits for it // to end, releasing the actor's lock meanwhile, before it drops the actor's // state in memory and on disk. diff --git a/cmd/atelet/internal/snapshottracker/tracker.go b/cmd/atelet/internal/snapshottracker/tracker.go index a52c704166..66970b595e 100644 --- a/cmd/atelet/internal/snapshottracker/tracker.go +++ b/cmd/atelet/internal/snapshottracker/tracker.go @@ -151,6 +151,11 @@ type Tracker struct { store Store // clock is what due times are measured on, and the queues wait on. clock clock.WithTicker + // fromCurrentBoot reports whether an actor's local snapshot was written + // since the node last booted, or is nil without WithBootCheck. When set, + // atelet deletes any local snapshot written before the node rebooted (see + // prunePreRebootLocalLocked). + fromCurrentBoot func(actorUID, snapshotUUID string) bool // uploadDelay is how long after UndurableSince the actor's latest local // snapshot becomes due for background upload: DefaultUploadDelay unless // WithUploadDelay sets it. @@ -223,6 +228,16 @@ type Tracker struct { // Option configures a Tracker. type Option func(*Tracker) +// WithBootCheck sets how atelet tells whether an actor's local snapshot was +// written since the node last booted. A snapshot whose local copy is gone must +// count as written before. With it, atelet deletes any local snapshot written +// before the node rebooted, whose files may be torn. +func WithBootCheck(fromCurrentBoot func(actorUID, snapshotUUID string) bool) Option { + return func(t *Tracker) { + t.fromCurrentBoot = fromCurrentBoot + } +} + // WithUploadDelay sets how long after the oldest snapshot of an actor not yet // durable was taken atelet uploads the actor's latest local snapshot. func WithUploadDelay(d time.Duration) Option { diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index 03966a7259..2b5c55e8ed 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -3476,3 +3476,170 @@ func TestRestoreStopsRetryingAfterConsecutiveUnavailableFailures(t *testing.T) { t.Fatalf("Restore after a canceled attempt: code = %v, want %v", got, codes.Internal) } } + +// TestCheckpointManifestBootID checks that Checkpoint stamps the node's boot ID +// into the local manifest only, omitting it from the uploaded manifest. +func TestCheckpointManifestBootID(t *testing.T) { + a := runPluginActor(t) + a.s.bootID = "boot-1" + snap := a.snapshot("suspend-snap-1", 1) + + if _, err := a.checkpoint(t, snap, ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE); err != nil { + t.Fatalf("Checkpoint: %v", err) + } + + localBytes, err := os.ReadFile(filepath.Join(ateletpath.LocalSnapshotDir(pluginActorUID, "suspend-snap-1"), sandboxManifestName)) + if err != nil { + t.Fatalf("reading local manifest: %v", err) + } + localRec, err := unmarshalSandboxRecord(localBytes) + if err != nil { + t.Fatalf("unmarshalling local manifest: %v", err) + } + if localRec.BootID != "boot-1" { + t.Errorf("local manifest bootId = %q, want %q", localRec.BootID, "boot-1") + } + + remoteBytes := a.store.objects[manifestKey("suspend-snap-1")] + if len(remoteBytes) == 0 { + t.Fatal("no remote manifest uploaded") + } + if strings.Contains(string(remoteBytes), `"bootId"`) { + t.Errorf("remote manifest %s carries bootId, want none", remoteBytes) + } + + localDir := ateletpath.LocalSnapshotDir(pluginActorUID, "suspend-snap-1") + noWorkers := []snapshottracker.Option{ + snapshottracker.WithUploadWorkers(0), + snapshottracker.WithDetachWorkers(0), + snapshottracker.WithReportWorkers(0), + } + sameBoot := snapshottracker.New(t.Context(), nil, clock.RealClock{}, append(noWorkers, snapshottracker.WithBootCheck(localSnapshotBootCheck("boot-1")))...) + sameBoot.Start() + if _, err := os.Stat(localDir); err != nil { + t.Fatalf("local snapshot after restart on same boot: stat = %v, want kept", err) + } + if got := sameBoot.State(pluginActorUID).GetLatestLocal().GetSnapshot().GetSnapshotUuid(); got != "suspend-snap-1" { + t.Errorf("latest_local after restart on same boot = %q, want %q", got, "suspend-snap-1") + } + + newBoot := snapshottracker.New(t.Context(), nil, clock.RealClock{}, append(noWorkers, snapshottracker.WithBootCheck(localSnapshotBootCheck("boot-2")))...) + newBoot.Start() + if _, err := os.Stat(localDir); !errors.Is(err, os.ErrNotExist) { + t.Errorf("local snapshot after restart on new boot: stat = %v, want not exist", err) + } + if got := newBoot.State(pluginActorUID).GetLatestLocal(); got != nil { + t.Errorf("latest_local after restart on new boot = %v, want nil", got) + } + if got := newBoot.State(pluginActorUID).GetLatestDurable().GetSnapshot().GetSnapshotUuid(); got != "suspend-snap-1" { + t.Errorf("latest_durable after restart on new boot = %q, want %q", got, "suspend-snap-1") + } +} + +// TestRestoreLocalSnapshotFromEarlierBoot checks that Restore never reads a +// node-local snapshot whose manifest carries a boot ID from before the node +// last rebooted: it restores from object storage when an uploaded copy +// exists, and fails with NotFound wrapping errUnusableLocalSnapshot otherwise. +func TestRestoreLocalSnapshotFromEarlierBoot(t *testing.T) { + const ( + atespace = "ate-demo" + actorName = "counter" + actorUID = "01234567-89ab-cdef-0123-456789abcdef" + workerPodUID = "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f" + snapshotName = "pause-snap-1" + ) + + worker := &fakeWorker{snapshotFiles: map[string]string{"checkpoint.img": ""}} + serveFakeWorker(t, worker) + + host := imageVolumeTestRegistry(t) + pinnedImage := pushTestImage(t, host+"/actor:v1", singleFileLayer(t, "bin/app", "app")) + + release := newGVisorTestRelease(t) + sandboxRec := testSandboxRecord(release) + spec := &ateletpb.WorkloadSpec{ + Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, + } + baseManifest := func() sandboxAssetsRecord { + return sandboxAssetsRecord{ + SandboxClass: "gvisor", + Assets: map[string]assetEntry{gvisorAssetName: release.entry}, + SnapshotFiles: []string{"checkpoint.img"}, + BootID: "boot-1", + } + } + + newHerder := func(t *testing.T, objStore *recordingObjectStorage) *Atelet { + t.Helper() + s := newPluginHerder(t, objStore) + s.bootID = "boot-2" + s.workerDialer = newWorkerDialer(1) + s.imageCache = newImageVolumeStore(t) + s.anonGCSClient = fakeObjectStorage{data: release.tarball} + s.systemInfoVolumes = newSystemInfoVolumeRefresher(nil, nil) + advertiseRuntime(t, s, sandboxRec) + return s + } + + restoreReq := &ateletpb.RestoreRequest{ + Atespace: atespace, + ActorName: actorName, + ActorUid: actorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + WorkerPodUid: workerPodUID, + SandboxRuntime: runtimeFor(t, sandboxRec), + Spec: spec, + Snapshot: &ateletpb.Snapshot{ + SnapshotUuid: snapshotName, + Generation: 1, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateletpb.ObjectStorage{SnapshotUri: pausedSnapshotURI}, + }, + } + + t.Run("uploaded copy exists restores from object storage", func(t *testing.T) { + useTempNodeDirs(t) + objStore := &recordingObjectStorage{} + s := newHerder(t, objStore) + + uri, err := resources.ParseSnapshotURI(pausedSnapshotURI) + if err != nil { + t.Fatal(err) + } + remoteRec := baseManifest() + remoteSrcDir := t.TempDir() + writeLocalSnapshot(t, remoteSrcDir, remoteRec, map[string]string{"checkpoint.img": "object-storage-memory"}) + if err := s.uploadSnapshot(t.Context(), testActor, uri, remoteSrcDir, &remoteRec); err != nil { + t.Fatalf("uploadSnapshot: %v", err) + } + + localDir := ateletpath.LocalSnapshotDir(actorUID, snapshotName) + writeLocalSnapshot(t, localDir, baseManifest(), map[string]string{"checkpoint.img": "local-memory"}) + + if _, err := s.Restore(t.Context(), restoreReq); err != nil { + t.Fatalf("Restore: %v", err) + } + if got := worker.restored["checkpoint.img"]; got != "object-storage-memory" { + t.Errorf("restored checkpoint.img = %q, want %q", got, "object-storage-memory") + } + if worker.preserveRestoreDir { + t.Error("preserve_restore_dir = true, want false when restoring from object storage") + } + }) + + t.Run("no uploaded copy fails with NotFound", func(t *testing.T) { + useTempNodeDirs(t) + s := newHerder(t, &recordingObjectStorage{}) + localDir := ateletpath.LocalSnapshotDir(actorUID, snapshotName) + writeLocalSnapshot(t, localDir, baseManifest(), map[string]string{"checkpoint.img": "local-memory"}) + + _, err := s.Restore(t.Context(), restoreReq) + if code := apierror.Code(err); code != codes.NotFound { + t.Fatalf("Restore error = %v (code %v), want NotFound", err, code) + } + if !errors.Is(err, errUnusableLocalSnapshot) { + t.Errorf("Restore error = %v, want it to wrap %v", err, errUnusableLocalSnapshot) + } + }) +} diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index f5fc795f7f..f638e938b7 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -380,12 +380,17 @@ func main() { defer ateapiConn.Close() workerService := ateapipb.NewWorkerServiceClient(ateapiConn) + bootID, err := readBootID(bootIDPath) + if err != nil { + serverboot.Fatal(ctx, "Failed to read the node's boot ID", err) + } // The snapshot tracker's workers get a context of their own, cancelled only // once the gRPC server has drained, so background snapshot work keeps // running while in-flight RPCs finish. trackerCtx, stopTracker := context.WithCancel(ctx) store := &trackerStore{} snapshots := snapshottracker.New(trackerCtx, store, clock.RealClock{}, + snapshottracker.WithBootCheck(localSnapshotBootCheck(bootID)), snapshottracker.WithUploadDelay(*snapshotUploadDelay), snapshottracker.WithDetachDelay(*snapshotDetachDelay), snapshottracker.WithUploadTimeout(*snapshotUploadTimeout), @@ -421,6 +426,7 @@ func main() { systemInfoVolumes, snapshots, workerService, + bootID, ) store.s = wmService snapshots.Start() @@ -724,6 +730,10 @@ type Atelet struct { // failures, so a restore that can never succeed stops being retried (see // boundRetryableRestoreError). restoreFailures restoreFailureCounter + // bootID is the node's boot ID (see readBootID), recorded in the manifest + // of each local snapshot. A local snapshot recording another, or none, + // was written before the node last rebooted. + bootID string } var _ ateletpb.AteletServer = (*Atelet)(nil) @@ -755,6 +765,7 @@ func NewService( systemInfoVolumes *systemInfoVolumeRefresher, snapshots *snapshottracker.Tracker, workers ateapipb.WorkerServiceClient, + bootID string, ) *Atelet { wms := &Atelet{ workerDialer: workerDialer, @@ -773,6 +784,7 @@ func NewService( systemInfoVolumes: systemInfoVolumes, snapshots: snapshots, workerActorJWTs: newWorkerActorJWTs(workers), + bootID: bootID, } return wms } @@ -1275,8 +1287,11 @@ func (s *Atelet) moveLocalCheckpoint(ctx context.Context, req *ateletpb.Checkpoi } } - // Write the self-describing snapshot manifest beside the images. - manifest, err := json.Marshal(rec) + // Write the self-describing snapshot manifest beside the images, stamped + // with this boot so a reader can tell whether the node has rebooted since. + local := *rec + local.BootID = s.bootID + manifest, err := json.Marshal(&local) if err != nil { return fmt.Errorf("while marshaling snapshot manifest: %w", err) } @@ -1333,7 +1348,10 @@ func (s *Atelet) uploadSnapshot(ctx context.Context, actor actorRef, uri resourc return fmt.Errorf("while uploading snapshot files: %w", err) } - manifest, err := json.Marshal(rec) + // The boot ID describes this node's local copy, not the snapshot. + remote := *rec + remote.BootID = "" + manifest, err := json.Marshal(&remote) if err != nil { return fmt.Errorf("while marshaling snapshot manifest: %w", err) } @@ -1422,17 +1440,29 @@ func (s *Atelet) promoteLocalSnapshot(ctx context.Context, req *ateletpb.Promote // uploadLocalCheckpointDir uploads the local checkpoint in localDir to uri. It // returns the sandbox class recorded in the snapshot manifest (empty when the -// manifest was not read), and whether it uploaded: it does not when the local -// copy is gone but already uploaded. Parameterized by localDir for tests. +// local copy is gone), and whether it uploaded: it does not when the local +// copy is gone but already uploaded. A local copy written before the node last +// rebooted counts as gone, since its files may be torn. Parameterized by +// localDir for tests. func (s *Atelet) uploadLocalCheckpointDir(ctx context.Context, req *ateletpb.PromoteSnapshotRequest, localDir string, uri resources.SnapshotURI) (sandboxClass string, uploaded bool, err error) { actor := actorRef{atespace: req.GetAtespace(), name: req.GetActorName(), uid: req.GetActorUid()} localSnapshotName := req.GetSnapshot().GetSnapshotUuid() + var rec *sandboxAssetsRecord manifest, err := readSnapshotManifest(localDir) + if err == nil { + if rec, err = unmarshalSandboxRecord(manifest); err != nil { + return "", false, err + } + if rec.BootID != s.bootID { + err = fmt.Errorf("local snapshot %q was written before the node last rebooted: %w", localSnapshotName, os.ErrNotExist) + } + } if errors.Is(err, os.ErrNotExist) { - // The local snapshot is gone. A detach may have uploaded and pruned - // it: the remote manifest is uploaded last, so its presence means the - // whole snapshot is committed. Absent on both sides, the local - // snapshot is unrecoverable. + // The local snapshot is gone, or was written before the node last + // rebooted and may be torn. A detach, or an upload before the reboot, + // may have uploaded it: the remote manifest is uploaded last, so its + // presence means the whole snapshot is committed. Absent on both + // sides, the local snapshot is unrecoverable. _, fetchErr := s.fetchManifest(ctx, actor, uri.String()) if fetchErr == nil { slog.InfoContext(ctx, "Local snapshot already uploaded and pruned; nothing to do", slog.String("snapshot_uri", uri.String())) @@ -1448,11 +1478,6 @@ func (s *Atelet) uploadLocalCheckpointDir(ctx context.Context, req *ateletpb.Pro return "", false, wrapFileSystemErr("while reading local snapshot manifest", err) } - rec, err := unmarshalSandboxRecord(manifest) - if err != nil { - return "", false, err - } - capturedFidelity := rec.Fidelity if capturedFidelity == "" { return rec.SandboxClass, false, apierror.FailedPrecondition("local snapshot %q has no fidelity recorded in its manifest; resume and suspend the actor again", localSnapshotName) @@ -1489,15 +1514,17 @@ func (st *trackerStore) UploadLocalSnapshot(ctx context.Context, actorUID string } // errUnusableLocalSnapshot marks a node-local snapshot that is present on disk -// (not simply absent) but cannot be read or validated: its manifest cannot be -// read or parsed, or lists a file that is missing or not a regular file, as an -// interrupted prune can leave. File contents and sizes are not checked. +// (not simply absent) but cannot be used: its manifest cannot be read or +// parsed, was written before the node last rebooted (its files may be torn), +// or lists a file that is missing or not a regular file, as an interrupted +// prune can leave. File contents and sizes are not checked. var errUnusableLocalSnapshot = errors.New("unusable local snapshot") // readLocalSnapshot returns the manifest of the node-local snapshot in -// localDir once every file it lists is present. A missing manifest reports -// os.ErrNotExist, and any other failure wraps errUnusableLocalSnapshot. -func readLocalSnapshot(localDir string) (*sandboxAssetsRecord, error) { +// localDir once it records bootID and every file it lists is present. A +// missing manifest reports os.ErrNotExist, and any other failure wraps +// errUnusableLocalSnapshot. +func readLocalSnapshot(localDir, bootID string) (*sandboxAssetsRecord, error) { manifest, err := readSnapshotManifest(localDir) if errors.Is(err, os.ErrNotExist) { return nil, err @@ -1509,6 +1536,9 @@ func readLocalSnapshot(localDir string) (*sandboxAssetsRecord, error) { if err != nil { return nil, fmt.Errorf("%w: %w", errUnusableLocalSnapshot, err) } + if rec.BootID != bootID { + return nil, fmt.Errorf("%w: written before the node last rebooted", errUnusableLocalSnapshot) + } if err := checkLocalSnapshotFiles(localDir, rec.SnapshotFiles); err != nil { return nil, fmt.Errorf("%w: %w", errUnusableLocalSnapshot, err) } @@ -1650,7 +1680,7 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res } }() var sandboxRec *sandboxAssetsRecord - localRec, localErr := readLocalSnapshot(localDir) + localRec, localErr := readLocalSnapshot(localDir, s.bootID) durable := req.GetSnapshot().GetObject() if localErr == nil { directLocal = true diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index d44079b368..c028be60b3 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -1515,6 +1515,54 @@ func TestUploadLocalCheckpointDir(t *testing.T) { t.Fatal("uploadLocalCheckpointDir succeeded, want error") } }) + + t.Run("earlier boot already uploaded skips upload", func(t *testing.T) { + store := &recordingObjectStorage{ + objects: map[string][]byte{ + pausedSnapshotPath + "/manifest.json": []byte(`{"sandboxClass":"microvm"}`), + }, + putErr: errors.New("must not upload a local copy from before the node rebooted"), + } + s := newPluginHerder(t, store) + s.bootID = "boot-2" + dir := filepath.Join(t.TempDir(), "pause-snap-1") + rec := fullRec("microvm") + rec.BootID = "boot-1" + writeLocalSnapshot(t, dir, rec, map[string]string{ + "config.json": "cfg", "memory-ranges": "", "data.tar": "data", + }) + + _, uploaded, err := s.uploadLocalCheckpointDir(ctx, validPromoteSnapshotRequest(), dir, uri) + if err != nil { + t.Fatalf("uploadLocalCheckpointDir: %v", err) + } + if uploaded { + t.Error("uploaded = true, want false when the remote manifest already exists") + } + }) + + t.Run("earlier boot not uploaded is NotFound", func(t *testing.T) { + store := &recordingObjectStorage{} + s := newPluginHerder(t, store) + s.bootID = "boot-2" + dir := filepath.Join(t.TempDir(), "pause-snap-1") + rec := fullRec("microvm") + rec.BootID = "boot-1" + writeLocalSnapshot(t, dir, rec, map[string]string{ + "config.json": "cfg", "memory-ranges": "mem", "data.tar": "data", + }) + + _, uploaded, err := s.uploadLocalCheckpointDir(ctx, validPromoteSnapshotRequest(), dir, uri) + if uploaded { + t.Error("uploaded = true, want false") + } + if got := apierror.Code(err); got != codes.NotFound { + t.Fatalf("code = %v (err %v), want NotFound", got, err) + } + if len(store.keys()) != 0 { + t.Errorf("objects uploaded for a local copy from before the node rebooted: %v", store.keys()) + } + }) } func TestShouldHaveSnapshots(t *testing.T) { diff --git a/cmd/atelet/sandbox_assets.go b/cmd/atelet/sandbox_assets.go index bea59b8a62..bcacecab65 100644 --- a/cmd/atelet/sandbox_assets.go +++ b/cmd/atelet/sandbox_assets.go @@ -106,6 +106,12 @@ type sandboxAssetsRecord struct { // is knowable from the manifest alone. Empty in the on-node record written at // Run/Restore. Fidelity string `json:"fidelity,omitempty"` + // BootID is the node's boot ID when atelet wrote this local copy of the + // snapshot. Not every file of a local copy is synced to disk, so one + // written before the node last rebooted may be torn and is deleted on + // reboot. Set only in the manifest of a local copy, never in an uploaded + // one. + BootID string `json:"bootId,omitempty"` } // ensureSandboxAssets stages every asset in the record under dstDir, named diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index b9fd6030ec..a2dbbbe46b 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -3696,9 +3696,10 @@ type RestoreRequest struct { Spec *WorkloadSpec `protobuf:"bytes,7,opt,name=spec,proto3" json:"spec,omitempty"` // Snapshot to restore from. atelet reads its local copy if it holds a // usable one, and `snapshot.object` otherwise. A local copy is unusable if - // its manifest does not parse or lists a missing file. While atelet is - // detaching the actor, it waits for the detach to finish and then reads - // `snapshot.object`. Fails if neither copy exists: that generation is lost. + // its manifest does not parse, lists a missing file, or was written before + // the node last rebooted. While atelet is detaching the actor, it waits for + // the detach to finish and then reads `snapshot.object`. Fails if neither + // copy exists: that generation is lost. // // `snapshot.object` is required even while the control plane has not // recorded the snapshot's upload complete: atelet may have uploaded the diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index 28918fe34b..7d1680cec4 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -410,6 +410,10 @@ service Atelet { // snapshot from the node. Waits for an in-flight detach, and fails for an // actor running on this node. // + // atelet never uploads a local copy written before the node last rebooted, + // since its files may be torn: it answers from the manifest at + // `snapshot.object` instead, and fails with NOT_FOUND if there is none. + // // Idempotent: once the snapshot is durable, atelet answers from the // manifest at `snapshot.object`, and fails if there is none. // @@ -1112,9 +1116,10 @@ message RestoreRequest { // Snapshot to restore from. atelet reads its local copy if it holds a // usable one, and `snapshot.object` otherwise. A local copy is unusable if - // its manifest does not parse or lists a missing file. While atelet is - // detaching the actor, it waits for the detach to finish and then reads - // `snapshot.object`. Fails if neither copy exists: that generation is lost. + // its manifest does not parse, lists a missing file, or was written before + // the node last rebooted. While atelet is detaching the actor, it waits for + // the detach to finish and then reads `snapshot.object`. Fails if neither + // copy exists: that generation is lost. // // `snapshot.object` is required even while the control plane has not // recorded the snapshot's upload complete: atelet may have uploaded the diff --git a/internal/proto/ateletpb/atelet_grpc.pb.go b/internal/proto/ateletpb/atelet_grpc.pb.go index 1257667a81..87d4d8915a 100644 --- a/internal/proto/ateletpb/atelet_grpc.pb.go +++ b/internal/proto/ateletpb/atelet_grpc.pb.go @@ -385,6 +385,10 @@ type AteletClient interface { // snapshot from the node. Waits for an in-flight detach, and fails for an // actor running on this node. // + // atelet never uploads a local copy written before the node last rebooted, + // since its files may be torn: it answers from the manifest at + // `snapshot.object` instead, and fails with NOT_FOUND if there is none. + // // Idempotent: once the snapshot is durable, atelet answers from the // manifest at `snapshot.object`, and fails if there is none. // @@ -512,6 +516,10 @@ type AteletServer interface { // snapshot from the node. Waits for an in-flight detach, and fails for an // actor running on this node. // + // atelet never uploads a local copy written before the node last rebooted, + // since its files may be torn: it answers from the manifest at + // `snapshot.object` instead, and fails with NOT_FOUND if there is none. + // // Idempotent: once the snapshot is durable, atelet answers from the // manifest at `snapshot.object`, and fails if there is none. // From e7cfad2ed77ea16de6bb72b7b966703de447bf3a Mon Sep 17 00:00:00 2001 From: ILLUM1N0X Date: Wed, 21 Oct 2026 23:57:47 +0000 Subject: [PATCH 05/12] On GCS, snapshot transfers run as a dedicated service account and their tokens no longer run short near the end of each hour (#3093) Fixes #2722 The credential exchange downscoped ate-api-server's own token, so GCS audit logs named ate-api-server for every node's snapshot transfer, and the metadata server's cached token left downscoped tokens short for the last twenty minutes of each hour. The exchange now impersonates a dedicated Google service account, `ate-snapshot-transfer` by default, and fetches a new token once less than 40 minutes are left. setup-gcp creates the account and its grants, and ate-setup passes it to the exchange. #### Breaking change GKE installs must run `go run ./tools/setup-gcp create iam` and set `SNAPSHOT_TRANSFER_GSA_NAME` before deploying. ate-setup refuses to deploy ate-api-server on GKE without it. With Cloud SQL, also rerun `create cloudsql`, so its account may create the transfer account's tokens. Tested on a GKE cluster: the full e2e suite passed on gVisor workers, a micro-VM actor suspended and resumed, and GCS audit logs named the transfer account for every node snapshot transfer. AI use: AI-assisted, I reviewed and tested it. - [x] Tests pass - [x] Docs updated --- cmd/ate-setup/internal/config/config.go | 32 +++ cmd/ate-setup/internal/config/config_test.go | 62 ++++++ cmd/ate-setup/internal/config/load.go | 12 +- cmd/ate-setup/internal/config/setting.go | 4 + cmd/ate-setup/internal/steps/apienv.go | 6 + cmd/ate-setup/internal/steps/apienv_test.go | 89 +++++++- cmd/ate-setup/internal/steps/deploy.go | 9 +- cmd/objectstore-plugin/exchange.go | 16 +- cmd/objectstore-plugin/exchange_test.go | 3 +- cmd/objectstore-plugin/main.go | 1 + docs/operator-install.md | 17 +- hack/ate-dev-env.sh.example | 3 + hack/teardown.sh | 28 ++- .../snapshotcredentials_test.go | 10 + internal/installdefaults/installdefaults.go | 4 + .../exchangeclient/exchangeclient.go | 3 +- .../objectstoreplugin/gcsissuer/issuer.go | 66 +++--- .../gcsissuer/issuer_test.go | 6 +- .../gcsissuer/transferaccount.go | 108 ++++++++++ .../gcsissuer/transferaccount_test.go | 193 ++++++++++++++++++ manifests/ate-install/ate-api-server.yaml | 9 +- tools/setup-gcp/README.md | 44 +++- tools/setup-gcp/cloud-sql.md | 1 + tools/setup-gcp/cmd/api.go | 2 + tools/setup-gcp/cmd/bootstrap.go | 22 +- tools/setup-gcp/cmd/bucket.go | 70 ++++--- tools/setup-gcp/cmd/bucket_test.go | 38 +++- tools/setup-gcp/cmd/cloudsql.go | 24 +-- tools/setup-gcp/cmd/common.go | 11 + tools/setup-gcp/cmd/iam.go | 12 +- tools/setup-gcp/cmd/transferaccount.go | 132 ++++++++++++ tools/setup-gcp/cmd/transferaccount_test.go | 63 ++++++ 32 files changed, 973 insertions(+), 127 deletions(-) create mode 100644 internal/objectstoreplugin/gcsissuer/transferaccount.go create mode 100644 internal/objectstoreplugin/gcsissuer/transferaccount_test.go create mode 100644 tools/setup-gcp/cmd/transferaccount.go create mode 100644 tools/setup-gcp/cmd/transferaccount_test.go diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index 50df2171ec..ab39897cf8 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -139,6 +139,11 @@ type Config struct { // template's snapshotConfig.storageLocation override it. SnapshotLocation string + // TransferGSAName is the name of the Google service account snapshot + // transfers run as on GCS, in ProjectID (SNAPSHOT_TRANSFER_GSA_NAME). + // Only the in-tree GCS backend uses it. See TransferServiceAccount. + TransferGSAName string + // KODockerRepo is where ko pushes images (KO_DOCKER_REPO). KODockerRepo string // KODefaultPlatforms constrains ko's build platforms. @@ -283,6 +288,33 @@ func (c *Config) RequireIngressAllowedSPIFFEIDs() error { return c.Resolved().Require("ingressAllowedSPIFFEIDs") } +// TransferServiceAccount is the email of the snapshot transfer service +// account, built from TransferGSAName and ProjectID as tools/setup-gcp builds +// it. It is empty when either is. +func (c *Config) TransferServiceAccount() string { + if c.TransferGSAName == "" || c.ProjectID == "" { + return "" + } + return c.TransferGSAName + "@" + c.ProjectID + ".iam.gserviceaccount.com" +} + +// RequireTransferServiceAccount fails when the install would deploy +// ate-api-server on GCS without a transfer service account. Its credential +// exchange impersonates the account and refuses to start without one. +// ate-setup installs GCS everywhere but kind, which uses S3. +func (c *Config) RequireTransferServiceAccount() error { + if c.Kind { + return nil + } + if c.TransferGSAName == "" { + return c.Resolved().Require("storage.transferGSAName") + } + if c.ProjectID == "" { + return c.Resolved().Require("gcp.projectID") + } + return nil +} + // SetResolved attaches resolved settings to a Config built directly. Load // does this itself; the callers here are tests and the demo test harness. func (c *Config) SetResolved(r *Resolved) { c.resolved = r } diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index d39d53ab7c..99ae080bbd 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -87,6 +87,7 @@ func loadEnv(t *testing.T) { "MEMORYSTORE_INSTANCE", "PROJECT_ID", "SNAPSHOT_LOCATION", + "SNAPSHOT_TRANSFER_GSA_NAME", "ATE_API_POSTGRES_CLOUDSQL_INSTANCE", } { // Unset, not blanked. An exported empty variable is a value a channel @@ -314,6 +315,67 @@ func TestRequireIngressAllowedSPIFFEIDs(t *testing.T) { } } +// The credential exchange impersonates the transfer account only on GCS, so +// only a GKE deploy needs one, and its email needs both the name and the +// project. +func TestRequireTransferServiceAccount(t *testing.T) { + for _, tc := range []struct { + name string + cfg Config + wantKey string + }{ + {name: "GKE without a name", cfg: Config{ProjectID: "proj"}, wantKey: "storage.transferGSAName"}, + {name: "GKE without a project", cfg: Config{TransferGSAName: "xfer"}, wantKey: "gcp.projectID"}, + {name: "GKE with both", cfg: Config{TransferGSAName: "xfer", ProjectID: "proj"}}, + {name: "kind without either", cfg: Config{Kind: true}}, + } { + t.Run(tc.name, func(t *testing.T) { + err := tc.cfg.RequireTransferServiceAccount() + if tc.wantKey == "" { + if err != nil { + t.Fatalf("RequireTransferServiceAccount() = %v, want nil", err) + } + return + } + var required *RequiredError + if !errors.As(err, &required) || required.Setting.Key != tc.wantKey { + t.Fatalf("RequireTransferServiceAccount() = %v, want a RequiredError for %s", err, tc.wantKey) + } + }) + } +} + +func TestLoadTransferServiceAccount(t *testing.T) { + for _, tc := range []struct { + name string + opts Options + env map[string]string + wantName string + want string + }{ + {name: "built from the name and the project", env: map[string]string{"SNAPSHOT_TRANSFER_GSA_NAME": "xfer", "PROJECT_ID": "proj"}, wantName: "xfer", want: "xfer@proj.iam.gserviceaccount.com"}, + {name: "no default name", env: map[string]string{"PROJECT_ID": "proj"}}, + {name: "kind ignores the environment", opts: Options{Kind: true}, env: map[string]string{"SNAPSHOT_TRANSFER_GSA_NAME": "xfer", "PROJECT_ID": "proj"}}, + } { + t.Run(tc.name, func(t *testing.T) { + loadEnv(t) + for k, v := range tc.env { + t.Setenv(k, v) + } + cfg, err := Load(tc.opts) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if cfg.TransferGSAName != tc.wantName { + t.Errorf("TransferGSAName = %q, want %q", cfg.TransferGSAName, tc.wantName) + } + if got := cfg.TransferServiceAccount(); got != tc.want { + t.Errorf("TransferServiceAccount() = %q, want %q", got, tc.want) + } + }) + } +} + func TestLoadDockerBuildFlags(t *testing.T) { loadEnv(t) t.Setenv("DOCKER_BUILD_FLAGS", " --cache-from type=gha --cache-to type=gha,mode=max ") diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index 1b31c4fac1..a85f7d64d5 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -234,6 +234,7 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro BucketName: r.String("storage.bucketName"), SnapshotLocation: r.String("storage.snapshotLocation"), + TransferGSAName: r.String("storage.transferGSAName"), KODockerRepo: r.String("ko.dockerRepo"), KODefaultPlatforms: r.String("ko.defaultPlatforms"), ActorJWTAlgorithm: r.String("actorJWT.algorithm"), @@ -333,11 +334,12 @@ func splitKubeconfig(r *Resolved) (explicitPath, scriptValue string) { // kindCluster.enabled -- which replays to whatever the defaults happen to be // on the next machine and the next release, not to the install it recorded. func applyKindDefaults(r *Resolved) { - // A Kind cluster has no GCP project, so the coordinates are cleared even - // when the developer environment named them. Only when something did name - // them: clearing an already-empty setting would report a value as set by - // the profile when the profile changed nothing. - for _, key := range []string{"gcp.projectID", "gcp.clusterLocation"} { + // A Kind cluster has no GCP project, so the coordinates and the transfer + // service account are cleared even when the developer environment named + // them. Only when something did name them: clearing an already-empty + // setting would report a value as set by the profile when the profile + // changed nothing. + for _, key := range []string{"gcp.projectID", "gcp.clusterLocation", "storage.transferGSAName"} { if r.String(key) != "" { r.set(key, "", OriginKindProfile) } diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index f3e0117c7a..389da4d55c 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -153,6 +153,10 @@ var Registry = []Setting{ Key: "storage.snapshotLocation", Env: "SNAPSHOT_LOCATION", Flag: "snapshot-location", Kind: KindString, Usage: "Cluster-wide object-storage URI ate-api-server stores snapshots under, for example gs://BUCKET/PREFIX/ (required if the PerTemplateSnapshotLocation preview gate is not enabled)", }, + { + Key: "storage.transferGSAName", Env: "SNAPSHOT_TRANSFER_GSA_NAME", Flag: "snapshot-transfer-gsa-name", Kind: KindString, + Usage: "Name of the Google service account snapshot transfers run as on GCS, in gcp.projectID. ate-api-server's credential exchange impersonates it (required to deploy ate-api-server on GKE, unused on kind)", + }, { Key: "atenet.dataplane", Env: "ATE_ATENET_DATAPLANE", Flag: "atenet-dataplane", Kind: KindString, diff --git a/cmd/ate-setup/internal/steps/apienv.go b/cmd/ate-setup/internal/steps/apienv.go index c9fc76e08d..481f72de1e 100644 --- a/cmd/ate-setup/internal/steps/apienv.go +++ b/cmd/ate-setup/internal/steps/apienv.go @@ -58,6 +58,9 @@ func (e *Env) CreateAPIServerEnvVars(ctx context.Context) error { if err != nil { return err } + if err := e.Cfg.RequireTransferServiceAccount(); err != nil { + return err + } if err := e.Kube.EnsureNamespace(ctx, e.Namespace()); err != nil { return err } @@ -135,6 +138,9 @@ func (e *Env) CreateAPIServerEnvVars(ctx context.Context) error { configVars["ATE_API_POSTGRES_POOL_MAX_CONNS"] = poolMaxConns } configVars["ATE_API_ACTOR_JWT_ISSUER"] = e.Cfg.ActorJWTIssuer + // Written empty on kind, so the sidecar's argument is never the literal + // $(ATE_API_SNAPSHOT_TRANSFER_SERVICE_ACCOUNT). + configVars["ATE_API_SNAPSHOT_TRANSFER_SERVICE_ACCOUNT"] = e.Cfg.TransferServiceAccount() if snapshotLocation != "" { configVars["ATE_API_SNAPSHOT_LOCATION"] = snapshotLocation } diff --git a/cmd/ate-setup/internal/steps/apienv_test.go b/cmd/ate-setup/internal/steps/apienv_test.go index 81e22f22bd..3c97dea3dc 100644 --- a/cmd/ate-setup/internal/steps/apienv_test.go +++ b/cmd/ate-setup/internal/steps/apienv_test.go @@ -63,7 +63,7 @@ func TestCreateAPIServerEnvVarsPostgresIdentities(t *testing.T) { }{ { name: "bundled identities", - cfg: config.Config{PostgresReadWriteRole: config.DefaultPostgresReadWriteRole, PostgresOwnerRole: config.DefaultPostgresOwnerRole}, + cfg: config.Config{TransferGSAName: testTransferGSAName, ProjectID: testProjectID, PostgresReadWriteRole: config.DefaultPostgresReadWriteRole, PostgresOwnerRole: config.DefaultPostgresOwnerRole}, readWriteDSN: bundledPostgresDSN(postgressetup.ReadWriteUser), ownerDSN: bundledPostgresDSN(postgressetup.OwnerUser), readWriteRole: config.DefaultPostgresReadWriteRole, ownerRole: config.DefaultPostgresOwnerRole, @@ -71,6 +71,7 @@ func TestCreateAPIServerEnvVarsPostgresIdentities(t *testing.T) { { name: "size10 bundled identities", cfg: config.Config{ + TransferGSAName: testTransferGSAName, ProjectID: testProjectID, ClusterSize: config.ClusterSizeSize10, PostgresReadWriteRole: config.DefaultPostgresReadWriteRole, PostgresOwnerRole: config.DefaultPostgresOwnerRole, }, readWriteDSN: bundledPostgresDSN(postgressetup.ReadWriteUser) + config.Size10PostgresPoolParams, @@ -80,6 +81,7 @@ func TestCreateAPIServerEnvVarsPostgresIdentities(t *testing.T) { { name: "external one login", cfg: config.Config{ + TransferGSAName: testTransferGSAName, ProjectID: testProjectID, PostgresReadWriteConnectionString: "postgres://operator@database/atepg", PostgresReadWriteRole: "tenant_readwrite", PostgresOwnerRole: "tenant_owner", }, @@ -90,6 +92,7 @@ func TestCreateAPIServerEnvVarsPostgresIdentities(t *testing.T) { { name: "external separate logins", cfg: config.Config{ + TransferGSAName: testTransferGSAName, ProjectID: testProjectID, PostgresReadWriteConnectionString: "postgres://runtime@database/atepg", PostgresOwnerConnectionString: "postgres://owner@database/atepg", PostgresReadWriteRole: "tenant_readwrite", PostgresOwnerRole: "tenant_owner", @@ -101,6 +104,7 @@ func TestCreateAPIServerEnvVarsPostgresIdentities(t *testing.T) { { name: "Cloud SQL one login", cfg: config.Config{ + TransferGSAName: testTransferGSAName, ProjectID: testProjectID, PostgresReadWriteRole: "tenant_readwrite", PostgresOwnerRole: "tenant_owner", CloudSQL: config.CloudSQLConfig{Instance: "p:r:i", InstanceSet: true, GSA: "svc@p.iam.gserviceaccount.com"}, }, @@ -142,7 +146,7 @@ func TestCreateAPIServerEnvVarsPostgresIdentities(t *testing.T) { } func TestCreateAPIServerEnvVarsRejectsCustomBundledIdentity(t *testing.T) { - cfg := config.Config{PostgresReadWriteRole: "tenant_readwrite", PostgresOwnerRole: "tenant_owner"} + cfg := config.Config{TransferGSAName: testTransferGSAName, ProjectID: testProjectID, PostgresReadWriteRole: "tenant_readwrite", PostgresOwnerRole: "tenant_owner"} cfg.SnapshotLocation = "gs://ate-snapshots/" e := &Env{Cfg: &cfg, Kube: fakeKube(t, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: NamespaceAteSystem}}, @@ -176,6 +180,7 @@ func TestCreateAPIServerEnvVarsKeepsExternalPostgres(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { cfg := config.Config{ + TransferGSAName: testTransferGSAName, ProjectID: testProjectID, PostgresReadWriteRole: config.DefaultPostgresReadWriteRole, PostgresOwnerRole: config.DefaultPostgresOwnerRole, CloudSQL: tc.cloudSQL, SnapshotLocation: "gs://ate-snapshots/", @@ -230,7 +235,7 @@ func TestCreateAPIServerEnvVarsKeepsOwnerLogin(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - cfg := config.Config{PostgresReadWriteConnectionString: readWrite, PostgresOwnerConnectionString: tc.owner, SnapshotLocation: "gs://ate-snapshots/"} + cfg := config.Config{TransferGSAName: testTransferGSAName, ProjectID: testProjectID, PostgresReadWriteConnectionString: readWrite, PostgresOwnerConnectionString: tc.owner, SnapshotLocation: "gs://ate-snapshots/"} e := &Env{Cfg: &cfg, Kube: fakeKube(t, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: NamespaceAteSystem}}, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: ConfigMapAPIEnvVars, Namespace: NamespaceAteSystem}}, @@ -265,6 +270,7 @@ func TestCreateAPIServerEnvVarsKeepsOwnerLogin(t *testing.T) { func TestCreateAPIServerEnvVarsPoolSize(t *testing.T) { cfg := config.Config{ + TransferGSAName: testTransferGSAName, ProjectID: testProjectID, PostgresReadWriteConnectionString: "postgres://runtime@postgres/atepg", PostgresPoolMaxConns: "20", } @@ -294,6 +300,7 @@ func TestCreateAPIServerEnvVarsPoolSize(t *testing.T) { func TestCreateAPIServerEnvVarsActorJWTIssuer(t *testing.T) { cfg := config.Config{ + TransferGSAName: testTransferGSAName, ProjectID: testProjectID, PostgresReadWriteConnectionString: "postgres://runtime@postgres/atepg", ActorJWTIssuer: "https://idp.example.com/prod", } @@ -330,6 +337,7 @@ func TestCreateAPIServerEnvVarsAdoptsPostgresIdentity(t *testing.T) { { name: "preserve recorded identity", cfg: config.Config{ + TransferGSAName: testTransferGSAName, ProjectID: testProjectID, PostgresReadWriteRole: config.DefaultPostgresReadWriteRole, PostgresOwnerRole: config.DefaultPostgresOwnerRole, }, @@ -342,6 +350,7 @@ func TestCreateAPIServerEnvVarsAdoptsPostgresIdentity(t *testing.T) { { name: "explicit overrides win", cfg: config.Config{ + TransferGSAName: testTransferGSAName, ProjectID: testProjectID, PostgresReadWriteRole: config.DefaultPostgresReadWriteRole, PostgresOwnerRole: config.DefaultPostgresOwnerRole, PostgresReadWriteRoleSet: true, @@ -423,7 +432,7 @@ func TestRecordedConnectionStrings(t *testing.T) { {"separate DSNs", map[string][]byte{"ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING": []byte("readwrite"), "ATE_API_POSTGRES_OWNER_CONNECTION_STRING": []byte("owner")}, "owner"}, } { t.Run(tc.name, func(t *testing.T) { - e := &Env{Cfg: &config.Config{}, Kube: fakeKube(t, &corev1.Secret{ + e := &Env{Cfg: &config.Config{TransferGSAName: testTransferGSAName, ProjectID: testProjectID}, Kube: fakeKube(t, &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Name: SecretAPIEnvVars, Namespace: NamespaceAteSystem}, Data: tc.data, })} @@ -482,9 +491,9 @@ func TestEnsureEnvVarsSafeStandalone(t *testing.T) { t.Run(tc.name, func(t *testing.T) { var e *Env if tc.dep == nil { - e = &Env{Cfg: &config.Config{}, Kube: fakeKube(t)} + e = &Env{Cfg: &config.Config{TransferGSAName: testTransferGSAName, ProjectID: testProjectID}, Kube: fakeKube(t)} } else { - e = &Env{Cfg: &config.Config{}, Kube: fakeKube(t, tc.dep)} + e = &Env{Cfg: &config.Config{TransferGSAName: testTransferGSAName, ProjectID: testProjectID}, Kube: fakeKube(t, tc.dep)} } err := e.EnsureEnvVarsSafeStandalone(t.Context()) if tc.wantErr { @@ -502,7 +511,7 @@ func TestEnsureEnvVarsSafeStandalone(t *testing.T) { func TestAnnotateAPIServerEnvHash(t *testing.T) { t.Run("fresh install is a no-op", func(t *testing.T) { - e := &Env{Cfg: &config.Config{}, Kube: fakeKube(t)} + e := &Env{Cfg: &config.Config{TransferGSAName: testTransferGSAName, ProjectID: testProjectID}, Kube: fakeKube(t)} if err := e.annotateAPIServerEnvHash(t.Context()); err != nil { t.Errorf("annotateAPIServerEnvHash() error = %v, want nil", err) } @@ -513,7 +522,7 @@ func TestAnnotateAPIServerEnvHash(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Namespace: NamespaceAteSystem, Name: SecretAPIEnvVars}, Data: map[string][]byte{"ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING": []byte("postgresql://h/atepg")}, } - e := &Env{Cfg: &config.Config{}, Kube: fakeKube(t, apiServerDeployment(SecretAPIEnvVars), secret)} + e := &Env{Cfg: &config.Config{TransferGSAName: testTransferGSAName, ProjectID: testProjectID}, Kube: fakeKube(t, apiServerDeployment(SecretAPIEnvVars), secret)} if err := e.annotateAPIServerEnvHash(t.Context()); err != nil { t.Fatalf("annotateAPIServerEnvHash() error = %v", err) @@ -536,7 +545,7 @@ func TestAnnotateAPIServerEnvHash(t *testing.T) { // Deployment, patches the running one. func TestOnlyAStandaloneEnvUpdateRollsTheAPIServer(t *testing.T) { newEnv := func(t *testing.T) *Env { - return &Env{Cfg: &config.Config{PostgresReadWriteConnectionString: "postgres://runtime@postgres/atepg", SnapshotLocation: "gs://ate-snapshots/"}, + return &Env{Cfg: &config.Config{TransferGSAName: testTransferGSAName, ProjectID: testProjectID, PostgresReadWriteConnectionString: "postgres://runtime@postgres/atepg", SnapshotLocation: "gs://ate-snapshots/"}, Kube: fakeKube(t, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: NamespaceAteSystem}}, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: ConfigMapAPIEnvVars, Namespace: NamespaceAteSystem}}, @@ -688,6 +697,7 @@ spec: func TestCreateAPIServerEnvVarsOmitsAuthz(t *testing.T) { e := &Env{Cfg: &config.Config{ + TransferGSAName: testTransferGSAName, ProjectID: testProjectID, PostgresReadWriteRole: config.DefaultPostgresReadWriteRole, PostgresOwnerRole: config.DefaultPostgresOwnerRole, APIAuthzBootstrapOwners: "system:serviceaccount:ate-system:ate-client", @@ -774,6 +784,7 @@ func TestCreateAPIServerEnvVarsSnapshotLocation(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { cfg := config.Config{ + TransferGSAName: testTransferGSAName, ProjectID: testProjectID, PostgresReadWriteConnectionString: "postgres://runtime@postgres/atepg", SnapshotLocation: tc.location, PreviewGates: tc.gates, @@ -806,3 +817,63 @@ func TestCreateAPIServerEnvVarsSnapshotLocation(t *testing.T) { }) } } + +// The transfer account the test configs name, and the email ate-setup builds +// from them. +const ( + testTransferGSAName = "ate-snapshot-transfer" + testProjectID = "proj" + testTransferAccount = "ate-snapshot-transfer@proj.iam.gserviceaccount.com" +) + +// The sidecar reads the transfer account from the ConfigMap. kind writes the +// key empty, since its S3 backend needs no account. A GKE install without one +// fails before writing anything. +func TestCreateAPIServerEnvVarsTransferServiceAccount(t *testing.T) { + for _, tc := range []struct { + name string + cfg config.Config + want string + wantErr bool + }{ + {name: "GKE", cfg: config.Config{TransferGSAName: testTransferGSAName, ProjectID: testProjectID}, want: testTransferAccount}, + {name: "kind", cfg: config.Config{Kind: true}}, + {name: "GKE without an account", cfg: config.Config{}, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := tc.cfg + cfg.PostgresReadWriteConnectionString = "postgres://runtime@postgres/atepg" + cfg.SnapshotLocation = "gs://ate-snapshots/" + e := &Env{Cfg: &cfg, Kube: fakeKube(t, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: NamespaceAteSystem}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: ConfigMapAPIEnvVars, Namespace: NamespaceAteSystem}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: SecretAPIEnvVars, Namespace: NamespaceAteSystem}}, + )} + err := e.CreateAPIServerEnvVars(t.Context()) + if tc.wantErr { + if err == nil || !strings.Contains(err.Error(), "SNAPSHOT_TRANSFER_GSA_NAME") { + t.Fatalf("CreateAPIServerEnvVars() = %v, want the missing account named", err) + } + cm, err := e.Kube.GetConfigMap(t.Context(), NamespaceAteSystem, ConfigMapAPIEnvVars) + if err != nil { + t.Fatal(err) + } + if len(cm.Data) != 0 { + t.Fatalf("the ConfigMap was written before the missing account was reported: %v", cm.Data) + } + return + } + if err != nil { + t.Fatal(err) + } + cm, err := e.Kube.GetConfigMap(t.Context(), NamespaceAteSystem, ConfigMapAPIEnvVars) + if err != nil { + t.Fatal(err) + } + got, ok := cm.Data["ATE_API_SNAPSHOT_TRANSFER_SERVICE_ACCOUNT"] + if !ok || got != tc.want { + t.Errorf("ATE_API_SNAPSHOT_TRANSFER_SERVICE_ACCOUNT = %q (present %v), want %q", got, ok, tc.want) + } + }) + } +} diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index 174821109c..e235e96b0a 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -68,10 +68,14 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { if err != nil { return err } - // Likewise a router that would refuse to start without an allowlist. + // Likewise a router that would refuse to start without an allowlist, and + // a credential exchange without a transfer service account. if err := e.Cfg.RequireIngressAllowedSPIFFEIDs(); err != nil { return err } + if err := e.Cfg.RequireTransferServiceAccount(); err != nil { + return err + } // Likewise the CSI request, even though it is only acted on partway // through: a driver this cluster cannot run is worth knowing before the // control plane goes up, not after. @@ -297,6 +301,9 @@ func (e *Env) DeployAteAPIServer(ctx context.Context) error { if _, err := clusterSnapshotLocation(e.Cfg.SnapshotLocation, e.Cfg.PreviewGates); err != nil { return err } + if err := e.Cfg.RequireTransferServiceAccount(); err != nil { + return err + } if err := e.EnsureCRDs(ctx); err != nil { return err } diff --git a/cmd/objectstore-plugin/exchange.go b/cmd/objectstore-plugin/exchange.go index 26d69e8863..b0c6739bca 100644 --- a/cmd/objectstore-plugin/exchange.go +++ b/cmd/objectstore-plugin/exchange.go @@ -40,11 +40,12 @@ import ( // exchangeConfig is what control mode needs to serve the credential exchange. type exchangeConfig struct { - listen string - credBundle string - issuer string - jwksFile string - ateapi actorlookup.Config + listen string + credBundle string + issuer string + jwksFile string + transferAccount string + ateapi actorlookup.Config } // newExchangeService builds the exchange handler for the configured backend. @@ -64,6 +65,9 @@ func newExchangeService(ctx context.Context, cfg exchangeConfig) (exchangepb.Cre if cfg.jwksFile == "" { return nil, nil, errors.New("--actor-jwt-jwks-file is required to serve the exchange on GCS") } + if cfg.transferAccount == "" { + return nil, nil, errors.New("--transfer-service-account is required to serve the exchange on GCS") + } // Read the key set once now, so a wrong path fails the start rather // than the first exchange. keys := oidcjwt.FileKeySource(cfg.jwksFile) @@ -75,7 +79,7 @@ func newExchangeService(ctx context.Context, cfg exchangeConfig) (exchangepb.Cre if err != nil { return nil, nil, err } - issuer, err := gcsissuer.New(ctx) + issuer, err := gcsissuer.New(ctx, cfg.transferAccount) if err != nil { lookup.Close() return nil, nil, fmt.Errorf("while setting up the GCS credential issuer: %w", err) diff --git a/cmd/objectstore-plugin/exchange_test.go b/cmd/objectstore-plugin/exchange_test.go index e2b637c12d..82afd2106a 100644 --- a/cmd/objectstore-plugin/exchange_test.go +++ b/cmd/objectstore-plugin/exchange_test.go @@ -41,7 +41,8 @@ func TestNewExchangeService(t *testing.T) { }{ {"GCS needs the issuer", exchangeConfig{jwksFile: "jwks.json"}, "--actor-jwt-issuer"}, {"GCS needs the JWKS file", exchangeConfig{issuer: "https://idp.ate-system.svc"}, "--actor-jwt-jwks-file"}, - {"GCS needs a readable JWKS file", exchangeConfig{issuer: "https://idp.ate-system.svc", jwksFile: filepath.Join(t.TempDir(), "absent.json")}, "key set"}, + {"GCS needs the transfer account", exchangeConfig{issuer: "https://idp.ate-system.svc", jwksFile: "jwks.json"}, "--transfer-service-account"}, + {"GCS needs a readable JWKS file", exchangeConfig{issuer: "https://idp.ate-system.svc", jwksFile: filepath.Join(t.TempDir(), "absent.json"), transferAccount: "xfer@proj.iam.gserviceaccount.com"}, "key set"}, } { t.Run(tc.name, func(t *testing.T) { t.Setenv("ATE_STORAGE_BACKEND", "gcs") diff --git a/cmd/objectstore-plugin/main.go b/cmd/objectstore-plugin/main.go index 9e42ba69f7..87585bf334 100644 --- a/cmd/objectstore-plugin/main.go +++ b/cmd/objectstore-plugin/main.go @@ -77,6 +77,7 @@ func main() { flags.StringVar(&xcfg.credBundle, "exchange-cred-bundle", "/run/servicedns.podcert.ate.dev/credential-bundle.pem", "control mode: PEM credential bundle the exchange serves with, the pod's service DNS certificate") flags.StringVar(&xcfg.issuer, "actor-jwt-issuer", "", "control mode: issuer URL of the actor JWTs the exchange accepts, the value ate-api-server runs with") flags.StringVar(&xcfg.jwksFile, "actor-jwt-jwks-file", "/run/actor-id-jwt-jwks/jwks.json", "control mode: JWK set file holding the public keys actor JWTs are verified with") + flags.StringVar(&xcfg.transferAccount, "transfer-service-account", "", "control mode on GCS: email of the Google service account snapshot transfers run as. The exchange impersonates it and downscopes its token per actor") flags.StringVar(&xcfg.ateapi.Address, "ateapi-address", "127.0.0.1:443", "control mode: ate-api-server gRPC address, the loopback in the shared pod") flags.StringVar(&xcfg.ateapi.CAFile, "ateapi-ca-file", "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "control mode: CA bundle used to verify ate-api-server") flags.StringVar(&xcfg.ateapi.ServerName, "ateapi-server-name", installdefaults.APIServiceHost(installdefaults.NamespaceFromPodEnv()), "control mode: DNS name expected on the ate-api-server certificate, the API Service in the pod's namespace") diff --git a/docs/operator-install.md b/docs/operator-install.md index c92ae0aa18..bbf0678424 100644 --- a/docs/operator-install.md +++ b/docs/operator-install.md @@ -319,6 +319,7 @@ demos, the [benchmarks](../benchmarking/README.md#settings), and the | `rolloutTimeout` | `ATE_INSTALL_ROLLOUT_TIMEOUT` | `--rollout-timeout` | `1m0s` | (global) | | `storage.bucketName` | `BUCKET_NAME` | `--storage-bucket-name` | — | (global) | | `storage.snapshotLocation` | `SNAPSHOT_LOCATION` | `--snapshot-location` | — | (global) | +| `storage.transferGSAName` | `SNAPSHOT_TRANSFER_GSA_NAME` | `--snapshot-transfer-gsa-name` | — | (global) | ## Snapshot storage @@ -330,6 +331,18 @@ no default, and ate-setup fails before deploying without it unless the some other way, pass the flag yourself; without the gate, the server refuses to start without it. +On GCS, ate-api-server's credential exchange runs snapshot transfers as a +dedicated Google service account, so GCS audit logs name that account. Set +its name with `SNAPSHOT_TRANSFER_GSA_NAME` (or ate-setup's +`--snapshot-transfer-gsa-name`), and ate-setup builds its email in +`gcp.projectID`. ate-setup refuses to deploy ate-api-server on GKE without +one. The exchange impersonates the account, so +ate-api-server's identity needs `roles/iam.serviceAccountTokenCreator` on it, +and the account needs object admin and bucket viewer on every snapshot bucket. +`go run ./tools/setup-gcp create iam` creates it under that name, +`ate-snapshot-transfer` by default, with both grants. +kind does not use it. + With `--kind`, ate-setup defaults `storage.snapshotLocation` to `gs:///` (`gs://ate-snapshots/`, the bucket the in-cluster rustfs creates) and ignores an ambient `SNAPSHOT_LOCATION`. On GKE, ate-setup @@ -350,8 +363,8 @@ shows, which also covers recreating a cluster with micro-VM workers. With the `PerTemplateSnapshotLocation` preview gate, an ActorTemplate may set `snapshotConfig.storageLocation` to store its snapshots elsewhere; templates that leave it empty use `--snapshot-location`. Grant ate-api-server's identity -object admin and bucket viewer on any other bucket a template names, as -setup-gcp does on the snapshot bucket. Without the gate, +and the snapshot transfer service account object admin and bucket viewer on +any other bucket a template names, as setup-gcp does on the snapshot bucket. Without the gate, ate-api-server ignores `snapshotConfig.storageLocation`. See [Snapshot Storage Layout](api-guide.md#snapshot-storage-layout). diff --git a/hack/ate-dev-env.sh.example b/hack/ate-dev-env.sh.example index da90304479..0e9a8bc72b 100644 --- a/hack/ate-dev-env.sh.example +++ b/hack/ate-dev-env.sh.example @@ -47,6 +47,9 @@ export KUBECTL_CONTEXT= export BUCKET_NAME=snapshot-substrate-test-${PROJECT_ID} # Cluster-wide location ate-api-server stores snapshots under. export SNAPSHOT_LOCATION="gs://${BUCKET_NAME}/" +# Google service account snapshot transfers run as on GCS. setup-gcp creates +# it, and ate-setup and hack/teardown.sh read the same name. +export SNAPSHOT_TRANSFER_GSA_NAME=ate-snapshot-transfer export KO_DOCKER_REPO="gcr.io/${PROJECT_ID}/ate-images" diff --git a/hack/teardown.sh b/hack/teardown.sh index 7c24d85193..68360cfc15 100755 --- a/hack/teardown.sh +++ b/hack/teardown.sh @@ -41,12 +41,19 @@ require() { } # --- Helper Functions --- + +# transfer_service_account prints the snapshot transfer service account's +# email, from SNAPSHOT_TRANSFER_GSA_NAME as setup-gcp and ate-setup build it. +transfer_service_account() { + echo "${SNAPSHOT_TRANSFER_GSA_NAME:-ate-snapshot-transfer}@${PROJECT_ID}.iam.gserviceaccount.com" +} function usage() { echo "Usage: $0 [options]" echo "Options:" echo " --revoke-gke-node-permissions Revoke GKE nodes permission to pull images" echo " --revoke-atelet-permissions Revoke atelet's project-level IAM bindings" - echo " --delete-iam-policy-bindings Delete bucket IAM policy bindings for atelet and ate-api-server" + echo " --delete-iam-policy-bindings Delete bucket IAM policy bindings for atelet, ate-api-server and the transfer account" + echo " --delete-transfer-service-account Delete the snapshot transfer service account" echo " --delete-snapshot-bucket Delete snapshot bucket" echo " --delete-gvisor-node-pool Delete gVisor node pool" echo " --delete-cluster Delete GKE cluster" @@ -127,12 +134,29 @@ delete_iam_policy_bindings() { --quiet || true done done + local transfer + transfer="serviceAccount:$(transfer_service_account)" + for role in roles/storage.objectAdmin roles/storage.bucketViewer; do + gcloud storage buckets remove-iam-policy-binding "gs://${BUCKET_NAME}" \ + --member="${transfer}" \ + --role="${role}" \ + --quiet || true + done gcloud storage buckets remove-iam-policy-binding "gs://${asset_bucket}" \ --member="${wi}/atelet" \ --role=roles/storage.objectViewer \ --quiet || true } +# Delete the snapshot transfer service account. Deleting it drops the token +# creator grants on it. +delete_transfer_service_account() { + require PROJECT_ID + echo "Deleting the snapshot transfer service account..." + gcloud iam service-accounts delete "$(transfer_service_account)" \ + --project="${PROJECT_ID}" --quiet || true +} + # Delete Snapshot Bucket (Reverse of create_snapshot_bucket) delete_snapshot_bucket() { require PROJECT_ID BUCKET_NAME @@ -175,6 +199,7 @@ while [[ "$#" -gt 0 ]]; do --revoke-gke-node-permissions) revoke_gke_node_permissions ;; --revoke-atelet-permissions) revoke_atelet_permissions ;; --delete-iam-policy-bindings) delete_iam_policy_bindings ;; + --delete-transfer-service-account) delete_transfer_service_account ;; --delete-snapshot-bucket) delete_snapshot_bucket ;; --delete-gvisor-node-pool) delete_gvisor_node_pool ;; --delete-cluster) delete_cluster ;; @@ -182,6 +207,7 @@ while [[ "$#" -gt 0 ]]; do --all) delete_dashboards delete_iam_policy_bindings + delete_transfer_service_account revoke_atelet_permissions revoke_gke_node_permissions delete_snapshot_bucket diff --git a/internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go b/internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go index bf343c5ff9..cc71b2ecd3 100644 --- a/internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go +++ b/internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go @@ -307,6 +307,16 @@ func TestExchangeScopesTheTokenToTheActor(t *testing.T) { if left := time.Until(resp.GetExpireTime().AsTime()); token == "" || left <= 0 { t.Fatalf("exchange returned token %q expiring in %v", token, left) } + // A forced exchange mints from the current source token, which is + // replaced once less than 40 minutes are left. A minute is allowed for + // clock skew between this machine and Google. + fresh, err := xchg.ExchangeActorJwt(ctx, &exchangepb.ExchangeActorJwtRequest{ActorJwt: sign.jwt(t, forActor(alpha)), Force: true}) + if err != nil { + t.Fatalf("forced exchange for a running actor: %v", err) + } + if left := time.Until(fresh.GetExpireTime().AsTime()); left < 39*time.Minute { + t.Errorf("a freshly minted token expires in %v, want at least 39m", left) + } // The golden is readable and not writable. The actor's own prefix, // which holds nothing yet, is writable. expectGCS(t, ctx, "golden manifest read", token, http.MethodGet, goldenURI+"/manifest.json", "", http.StatusOK) diff --git a/internal/installdefaults/installdefaults.go b/internal/installdefaults/installdefaults.go index ed0aad3190..2402e7986a 100644 --- a/internal/installdefaults/installdefaults.go +++ b/internal/installdefaults/installdefaults.go @@ -52,6 +52,10 @@ const ( ControllerServiceAccount = "ate-controller" APIServerServiceAccount = "ate-api-server" + // TransferServiceAccountName is the default name of the Google service + // account snapshot transfers run as on GCS, which tools/setup-gcp creates. + TransferServiceAccountName = "ate-snapshot-transfer" + // WorkerServiceAccount is what worker pods run as when their WorkerPool // names none; atecontroller creates it in the pool's namespace. WorkerServiceAccount = "ateworker" diff --git a/internal/objectstoreplugin/exchangeclient/exchangeclient.go b/internal/objectstoreplugin/exchangeclient/exchangeclient.go index 40496993e7..d486a1f800 100644 --- a/internal/objectstoreplugin/exchangeclient/exchangeclient.go +++ b/internal/objectstoreplugin/exchangeclient/exchangeclient.go @@ -151,8 +151,7 @@ func (s *Source) Client(ctx context.Context, actorJWT string, force bool) (objec if err != nil { return nil, status.Errorf(codes.Internal, "while building the actor's storage client: %v", err) } - // A token minted inside the source credential's end-of-hour window is - // shorter than the floor. It serves this transfer and is not kept. + // A token shorter than the floor serves this transfer and is not kept. if tok.Expiry.Sub(s.now()) >= credential.MinServeLifetime { s.cache.Add(actorJWT, &entry{token: tok, client: client}) } diff --git a/internal/objectstoreplugin/gcsissuer/issuer.go b/internal/objectstoreplugin/gcsissuer/issuer.go index 1cdb9dfb9e..e7e1da230e 100644 --- a/internal/objectstoreplugin/gcsissuer/issuer.go +++ b/internal/objectstoreplugin/gcsissuer/issuer.go @@ -16,12 +16,13 @@ // GCS half of the control plugin's credential exchange: actorscope decides what // an actor may touch, this package turns that into a credential GCS enforces. // -// The source credential comes from the default credential chain, which on GKE -// with Workload Identity is the federated token. The downscope package -// exchanges it at STS with a Credential Access Boundary and returns an access -// token GCS limits to the boundary. STS reports no lifetime for that token, so -// the package falls back to the source token's expiry, and the two expire -// together. +// The source credential is a token for the snapshot transfer service account, +// which the pod's own identity impersonates through the IAM Credentials API. +// GCS audit logs then name that account rather than ate-api-server. The +// downscope package exchanges the token at STS with a Credential Access +// Boundary and returns an access token GCS limits to the boundary. STS reports +// no lifetime for that token, so the package falls back to the source token's +// expiry, and the two expire together. package gcsissuer import ( @@ -33,8 +34,8 @@ import ( "time" "golang.org/x/oauth2" - "golang.org/x/oauth2/google" "golang.org/x/oauth2/google/downscope" + "google.golang.org/api/option" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/utils/lru" @@ -60,13 +61,14 @@ const ( stsTimeout = 30 * time.Second ) -// Issuer exchanges the pod's credential for per-actor downscoped tokens and -// caches them by actor UID. +// Issuer exchanges the transfer account's token for per-actor downscoped +// tokens and caches them by actor UID. type Issuer struct { - source oauth2.TokenSource - client *http.Client - now func() time.Time - cache *lru.Cache + source oauth2.TokenSource + iamOptions []option.ClientOption + client *http.Client + now func() time.Time + cache *lru.Cache } // entry is a cached token with the grants it was minted for, so a request can @@ -79,7 +81,12 @@ type entry struct { // Option configures an Issuer. type Option func(*Issuer) -// WithTokenSource overrides the source credential, for tests. +// WithIAMCredentialsOptions configures the IAM Credentials client, for tests. +func WithIAMCredentialsOptions(opts ...option.ClientOption) Option { + return func(i *Issuer) { i.iamOptions = opts } +} + +// WithTokenSource replaces the transfer account's token, for tests. func WithTokenSource(ts oauth2.TokenSource) Option { return func(i *Issuer) { i.source = ts } } // WithHTTPClient overrides the client used for STS, for tests. @@ -94,10 +101,11 @@ func WithCacheSize(n int) Option { return func(i *Issuer) { i.cache = lru.New(max(n, 1)) } } -// New returns an Issuer whose source credential is the default credential -// chain, unless an option replaces it. The credential keeps refreshing for the -// Issuer's whole life, so ctx is detached from cancellation. -func New(ctx context.Context, opts ...Option) (*Issuer, error) { +// New returns an Issuer whose source credential is a token for the transfer +// service account with email account. Tests that replace the source with +// WithTokenSource pass an empty account. The credential keeps refreshing for +// the Issuer's whole life, so ctx is detached from cancellation. +func New(ctx context.Context, account string, opts ...Option) (*Issuer, error) { i := &Issuer{ client: &http.Client{Timeout: stsTimeout}, now: time.Now, @@ -107,13 +115,11 @@ func New(ctx context.Context, opts ...Option) (*Issuer, error) { o(i) } if i.source == nil { - creds, err := google.FindDefaultCredentialsWithParams(context.WithoutCancel(ctx), google.CredentialsParams{ - Scopes: []string{cloudPlatformScope}, - }) + src, err := newTransferAccountSource(context.WithoutCancel(ctx), account, i.iamOptions...) if err != nil { - return nil, fmt.Errorf("while loading the default credential: %w", err) + return nil, err } - i.source = creds.TokenSource + i.source = src } return i, nil } @@ -121,14 +127,12 @@ func New(ctx context.Context, opts ...Option) (*Issuer, error) { // Issue returns a token downscoped to grants for the actor. A cached token is // returned while it covers the grants and has at least minServeLifetime left, // unless force is set, which mints regardless and is what the node plugin -// sends after GCS refused a token. A minted token with less than -// minServeLifetime left is handed out anyway, since a short token beats a -// failed transfer, but it is not cached. That happens for a stretch at the -// end of every hour of the source credential's life: the downscoped token -// expires with its source, and the GKE metadata server serves the same cached -// source token until shortly before it expires, so no early refresh helps. In -// that stretch every exchange reaches STS and the node plugin re-exchanges -// once if a transfer outlives its token. +// sends after GCS refused a token. A minted token expires with the source +// token, which is fetched again once less than sourceRefreshMargin is left, so +// it starts with at least that long. If that fetch fails, the previous source +// token is used until it expires. A minted token with less than +// minServeLifetime left, which only that leaves behind, is handed out anyway, +// since a short token beats a failed transfer, but it is not cached. // // The cache is keyed by actor UID and each entry remembers its grants. Grants // narrow when the tag an actor was created from drops out after its first diff --git a/internal/objectstoreplugin/gcsissuer/issuer_test.go b/internal/objectstoreplugin/gcsissuer/issuer_test.go index 1bfd687ebe..6a1059daee 100644 --- a/internal/objectstoreplugin/gcsissuer/issuer_test.go +++ b/internal/objectstoreplugin/gcsissuer/issuer_test.go @@ -98,7 +98,7 @@ func newTestIssuer(t *testing.T, sts *fakeSTS, src oauth2.TokenSource, now func( t.Cleanup(srv.Close) target, _ := url.Parse(srv.URL) client := &http.Client{Transport: redirect{sts: sts, target: target}} - i, err := New(t.Context(), WithTokenSource(src), WithHTTPClient(client), WithClock(now), WithCacheSize(2)) + i, err := New(t.Context(), "", WithTokenSource(src), WithHTTPClient(client), WithClock(now), WithCacheSize(2)) if err != nil { t.Fatalf("New: %v", err) } @@ -275,7 +275,7 @@ func TestIssue_StopsWhenTheCallerGivesUp(t *testing.T) { t.Cleanup(srv.Close) target, _ := url.Parse(srv.URL) client := &http.Client{Transport: redirect{sts: &fakeSTS{t: t}, target: target}, Timeout: time.Minute} - i, err := New(t.Context(), WithTokenSource(staticSource(now.Add(time.Hour))), WithHTTPClient(client), WithClock(func() time.Time { return now })) + i, err := New(t.Context(), "", WithTokenSource(staticSource(now.Add(time.Hour))), WithHTTPClient(client), WithClock(func() time.Time { return now })) if err != nil { t.Fatal(err) } @@ -296,7 +296,7 @@ func TestWithCacheSize_ZeroIsBounded(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(sts.handler)) t.Cleanup(srv.Close) target, _ := url.Parse(srv.URL) - i, err := New(t.Context(), WithTokenSource(staticSource(now.Add(time.Hour))), WithHTTPClient(&http.Client{Transport: redirect{sts: sts, target: target}}), WithClock(func() time.Time { return now }), WithCacheSize(0)) + i, err := New(t.Context(), "", WithTokenSource(staticSource(now.Add(time.Hour))), WithHTTPClient(&http.Client{Transport: redirect{sts: sts, target: target}}), WithClock(func() time.Time { return now }), WithCacheSize(0)) if err != nil { t.Fatal(err) } diff --git a/internal/objectstoreplugin/gcsissuer/transferaccount.go b/internal/objectstoreplugin/gcsissuer/transferaccount.go new file mode 100644 index 0000000000..f139a5d1dc --- /dev/null +++ b/internal/objectstoreplugin/gcsissuer/transferaccount.go @@ -0,0 +1,108 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsissuer + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "time" + + "golang.org/x/oauth2" + "google.golang.org/api/iamcredentials/v1" + "google.golang.org/api/option" +) + +const ( + // transferTokenLifetime is what the source token is requested with, the + // longest the IAM Credentials API grants without an org policy change. + transferTokenLifetime = time.Hour + // sourceRefreshMargin is how much life a source token has left when the + // next one is fetched. Downscoped tokens expire with their source, so each + // starts with at least this long. + sourceRefreshMargin = 40 * time.Minute + // iamTimeout bounds one IAM Credentials call, as long as the metadata + // server client waits for a token. + iamTimeout = 5 * time.Second +) + +// transferAccountSource returns a token for the transfer service account on +// every call, from the IAM Credentials API. The caller is the pod's own +// identity, which needs roles/iam.serviceAccountTokenCreator on the account. +type transferAccountSource struct { + ctx context.Context + svc *iamcredentials.Service + name string +} + +func (s transferAccountSource) Token() (*oauth2.Token, error) { + ctx, cancel := context.WithTimeout(s.ctx, iamTimeout) + defer cancel() + resp, err := s.svc.Projects.ServiceAccounts.GenerateAccessToken(s.name, &iamcredentials.GenerateAccessTokenRequest{ + Scope: []string{cloudPlatformScope}, + Lifetime: fmt.Sprintf("%.0fs", transferTokenLifetime.Seconds()), + }).Context(ctx).Do() + if err != nil { + return nil, fmt.Errorf("while getting a token for %s: %w", s.name, err) + } + if resp.AccessToken == "" { + return nil, fmt.Errorf("the IAM Credentials API returned no token for %s", s.name) + } + expiry, err := time.Parse(time.RFC3339, resp.ExpireTime) + if err != nil { + return nil, fmt.Errorf("the IAM Credentials API returned expiry %q for %s: %w", resp.ExpireTime, s.name, err) + } + return &oauth2.Token{AccessToken: resp.AccessToken, TokenType: "Bearer", Expiry: expiry}, nil +} + +// fallbackSource returns the last token src gave while src fails and that +// token has not expired, so a brief IAM Credentials outage does not fail +// exchanges. +type fallbackSource struct { + src oauth2.TokenSource + last atomic.Pointer[oauth2.Token] +} + +func (f *fallbackSource) Token() (*oauth2.Token, error) { + tok, err := f.src.Token() + if err != nil { + // Valid would apply the 40 minute margin the reuse source sets on its + // tokens, so check the expiry itself. The token must outlast the STS + // call it is downscoped with. + if last := f.last.Load(); last != nil && time.Until(last.Expiry) > stsTimeout { + return last, nil + } + return nil, err + } + f.last.Store(tok) + return tok, nil +} + +// newTransferAccountSource returns the source credential: tokens for account, +// fetched again once less than sourceRefreshMargin is left. It calls the IAM +// Credentials API itself rather than through a library that caches until +// shortly before expiry, since that cache would hand back the same token. +func newTransferAccountSource(ctx context.Context, account string, opts ...option.ClientOption) (oauth2.TokenSource, error) { + if account == "" { + return nil, errors.New("no transfer service account to impersonate") + } + svc, err := iamcredentials.NewService(ctx, opts...) + if err != nil { + return nil, fmt.Errorf("while creating the IAM Credentials client: %w", err) + } + src := transferAccountSource{ctx: ctx, svc: svc, name: "projects/-/serviceAccounts/" + account} + return &fallbackSource{src: oauth2.ReuseTokenSourceWithExpiry(nil, src, sourceRefreshMargin)}, nil +} diff --git a/internal/objectstoreplugin/gcsissuer/transferaccount_test.go b/internal/objectstoreplugin/gcsissuer/transferaccount_test.go new file mode 100644 index 0000000000..14bd83ade3 --- /dev/null +++ b/internal/objectstoreplugin/gcsissuer/transferaccount_test.go @@ -0,0 +1,193 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsissuer + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "testing" + "time" + + "google.golang.org/api/option" +) + +const testAccount = "ate-snapshot-transfer@proj.iam.gserviceaccount.com" + +// fakeIAMCredentials answers generateAccessToken with a token that has left +// of its life remaining, counting calls. +type fakeIAMCredentials struct { + t *testing.T + mu sync.Mutex + calls int + left time.Duration + status int +} + +func (f *fakeIAMCredentials) handler(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls++ + if want := "/v1/projects/-/serviceAccounts/" + testAccount + ":generateAccessToken"; r.URL.Path != want { + f.t.Errorf("path = %q, want %q", r.URL.Path, want) + } + var req struct { + Scope []string `json:"scope"` + Lifetime string `json:"lifetime"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + f.t.Errorf("decode request: %v", err) + } + if len(req.Scope) != 1 || req.Scope[0] != cloudPlatformScope || req.Lifetime != "3600s" { + f.t.Errorf("request = %+v, want the cloud-platform scope for 3600s", req) + } + if f.status != 0 { + w.WriteHeader(f.status) + _, _ = w.Write([]byte(`{"error":{"code":403,"message":"denied"}}`)) + return + } + _ = json.NewEncoder(w).Encode(map[string]string{ + "accessToken": "transfer-token", + "expireTime": time.Now().Add(f.left).UTC().Format(time.RFC3339), + }) +} + +func (f *fakeIAMCredentials) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.calls +} + +func (f *fakeIAMCredentials) options(t *testing.T) []option.ClientOption { + srv := httptest.NewServer(http.HandlerFunc(f.handler)) + t.Cleanup(srv.Close) + return []option.ClientOption{option.WithEndpoint(srv.URL), option.WithoutAuthentication()} +} + +func TestTransferAccountSource_FetchesAgainWithLessThanFortyMinutesLeft(t *testing.T) { + for _, tc := range []struct { + name string + left time.Duration + wantCalls int + }{ + {"45 minutes left is reused", 45 * time.Minute, 1}, + {"39 minutes left is fetched again", 39 * time.Minute, 2}, + } { + t.Run(tc.name, func(t *testing.T) { + iam := &fakeIAMCredentials{t: t, left: tc.left} + src, err := newTransferAccountSource(t.Context(), testAccount, iam.options(t)...) + if err != nil { + t.Fatal(err) + } + for range 2 { + tok, err := src.Token() + if err != nil { + t.Fatalf("Token: %v", err) + } + if tok.AccessToken != "transfer-token" { + t.Fatalf("token = %q, want the transfer account's", tok.AccessToken) + } + } + if got := iam.callCount(); got != tc.wantCalls { + t.Errorf("IAM Credentials calls = %d, want %d", got, tc.wantCalls) + } + }) + } +} + +func TestTransferAccountSource_NamesTheAccountWhenRefused(t *testing.T) { + iam := &fakeIAMCredentials{t: t, status: http.StatusForbidden} + src, err := newTransferAccountSource(t.Context(), testAccount, iam.options(t)...) + if err != nil { + t.Fatal(err) + } + if _, err := src.Token(); err == nil || !strings.Contains(err.Error(), testAccount) { + t.Fatalf("Token error = %v, want one naming %s", err, testAccount) + } +} + +func TestTransferAccountSource_KeepsTheLastTokenWhenAFetchFails(t *testing.T) { + for _, tc := range []struct { + name string + left time.Duration + wantErr bool + }{ + {"an unexpired token is returned", 39 * time.Minute, false}, + {"a token that would expire during the STS call is not", 20 * time.Second, true}, + {"an expired token is not", -time.Minute, true}, + } { + t.Run(tc.name, func(t *testing.T) { + iam := &fakeIAMCredentials{t: t, left: tc.left} + src, err := newTransferAccountSource(t.Context(), testAccount, iam.options(t)...) + if err != nil { + t.Fatal(err) + } + if _, err := src.Token(); err != nil { + t.Fatalf("first Token: %v", err) + } + iam.mu.Lock() + iam.status = http.StatusServiceUnavailable + iam.mu.Unlock() + tok, err := src.Token() + if gotErr := err != nil; gotErr != tc.wantErr { + t.Fatalf("second Token error = %v, want error %v", err, tc.wantErr) + } + if !tc.wantErr && tok.AccessToken != "transfer-token" { + t.Errorf("token = %q, want the previous one", tok.AccessToken) + } + if got := iam.callCount(); got != 2 { + t.Errorf("IAM Credentials calls = %d, want 2", got) + } + }) + } +} + +func TestNew_NeedsATransferAccount(t *testing.T) { + if _, err := New(t.Context(), ""); err == nil || !strings.Contains(err.Error(), "transfer service account") { + t.Fatalf("New without a transfer account = %v, want an error naming it", err) + } +} + +// TestIssue_DownscopesTheTransferAccountsToken checks the issuer's source is +// the impersonated account: STS reports no lifetime, so the downscoped token +// expires with the token the IAM Credentials API returned. +func TestIssue_DownscopesTheTransferAccountsToken(t *testing.T) { + iam := &fakeIAMCredentials{t: t, left: time.Hour} + sts := &fakeSTS{t: t} + srv := httptest.NewServer(http.HandlerFunc(sts.handler)) + t.Cleanup(srv.Close) + target, _ := url.Parse(srv.URL) + i, err := New(t.Context(), testAccount, WithIAMCredentialsOptions(iam.options(t)...), + WithHTTPClient(&http.Client{Transport: redirect{sts: sts, target: target}})) + if err != nil { + t.Fatalf("New: %v", err) + } + tok, err := i.Issue(t.Context(), "9f1e", testGrants, false) + if err != nil { + t.Fatalf("Issue: %v", err) + } + if sts.last["subject_token"] != "transfer-token" { + t.Errorf("STS subject_token = %q, want the transfer account's token", sts.last["subject_token"]) + } + if left := time.Until(tok.Expiry); left < sourceRefreshMargin { + t.Errorf("downscoped token has %v left, want at least %v", left, sourceRefreshMargin) + } + if iam.callCount() != 1 { + t.Errorf("IAM Credentials calls = %d, want 1", iam.callCount()) + } +} diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index 92ff9cff30..07b1f6ff3c 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -103,9 +103,11 @@ spec: # JWTs with the public keys mounted below, and reaches ate-api-server # over the loopback with the pod identity certificate. The issuer must # be the one ate-api-server mints with, so both read it from the same - # ConfigMap key. + # ConfigMap key. On GCS it downscopes a token for the snapshot + # transfer service account, which it impersonates. - --exchange-listen=:8445 - --actor-jwt-issuer=$(ATE_API_ACTOR_JWT_ISSUER) + - --transfer-service-account=$(ATE_API_SNAPSHOT_TRANSFER_SERVICE_ACCOUNT) ports: - name: exchange containerPort: 8445 @@ -138,6 +140,11 @@ spec: configMapKeyRef: name: ate-api-server-envvars key: ATE_API_ACTOR_JWT_ISSUER + - name: ATE_API_SNAPSHOT_TRANSFER_SERVICE_ACCOUNT + valueFrom: + configMapKeyRef: + name: ate-api-server-envvars + key: ATE_API_SNAPSHOT_TRANSFER_SERVICE_ACCOUNT # The exchange verifies ate-api-server's certificate against the API # Service name in this namespace. - name: POD_NAMESPACE diff --git a/tools/setup-gcp/README.md b/tools/setup-gcp/README.md index 603bf59080..3a6b41883e 100644 --- a/tools/setup-gcp/README.md +++ b/tools/setup-gcp/README.md @@ -218,16 +218,18 @@ go run ./tools/setup-gcp create iam [flags] | `--bucket` | GCS bucket name. | `BUCKET_NAME` | None (Required for bucket bindings*) | | `--gke-nodes` | Grant GKE nodes permission to pull images. | - | `true` | | `--atelet` | Grant atelet project-level permissions. | - | `true` | -| `--bucket-bindings` | Grant ate-api-server access to the snapshot bucket, and atelet read access to the asset bucket. | - | `true` | +| `--transfer-service-account` | Create the snapshot transfer service account and let ate-api-server create tokens for it. | - | `true` | +| `--transfer-gsa-name` | Name of the snapshot transfer service account to create. ate-setup and `hack/teardown.sh` read the same variable. | `SNAPSHOT_TRANSFER_GSA_NAME` | `ate-snapshot-transfer` | +| `--bucket-bindings` | Grant ate-api-server and the snapshot transfer service account access to the snapshot bucket, and atelet read access to the asset bucket. | - | `true` | *\*Note: Required for bucket bindings unless the `BUCKET_NAME` environment variable is set.* #### What `create iam` actually grants On a fresh GCP project these bindings do not exist, and nothing else creates -them. Without them atelet cannot fetch micro-VM assets, ate-api cannot copy, -collect, or issue per-actor tokens for snapshots (`403` from GCS on the first -suspend/resume), and nodes cannot pull images. `bootstrap` and `create iam` +them. Without them atelet cannot fetch micro-VM assets, ate-api cannot copy or +collect snapshots or issue per-actor tokens for them (`403` from GCS on the +first suspend/resume), and nodes cannot pull images. `bootstrap` and `create iam` apply them for you. The table below is the reference for auditing them, or for applying them by hand in projects where you cannot run the tool with project-level IAM permissions. @@ -236,16 +238,22 @@ atelet and ate-api authenticate via [GKE Workload Identity Federation](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity): their Kubernetes ServiceAccounts (`ate-system/atelet` and `ate-system/ate-api-server`, created by `install-ate.sh`) are addressed -directly as IAM principals — no Google service account is created or -impersonated. This only works on a cluster with Workload Identity enabled -(`create cluster` enables the `PROJECT_ID.svc.id.goog` pool; a pre-existing -cluster must have it enabled too, or their GCS calls fail with `401`). +directly as IAM principals. The one Google service account `create iam` +creates is the snapshot transfer account⁴, which ate-api-server's credential +exchange impersonates and downscopes per actor for every snapshot transfer. +This only works on a cluster with Workload Identity enabled. `create cluster` +enables the `PROJECT_ID.svc.id.goog` pool, and a pre-existing cluster must +have it enabled too, or their GCS calls fail with `401`. | Member | Role | Resource | Why | | :--- | :--- | :--- | :--- | | atelet WI principal¹ | `roles/artifactregistry.reader` | project² | Pull sandbox runtime assets | | ate-api-server WI principal¹ | `roles/storage.objectAdmin` | snapshot bucket | Copy external snapshots for tags, delete the ones nothing refers to | | ate-api-server WI principal¹ | `roles/storage.bucketViewer` | snapshot bucket | List/stat the snapshot bucket | +| snapshot transfer SA⁴ | `roles/storage.objectAdmin` | snapshot bucket | Snapshot transfers, on tokens the credential exchange downscopes per actor | +| snapshot transfer SA⁴ | `roles/storage.bucketViewer` | snapshot bucket | List/stat the snapshot bucket | +| ate-api-server WI principal¹ | `roles/iam.serviceAccountTokenCreator` | snapshot transfer SA⁴ | The credential exchange impersonates it | +| Cloud SQL GSA⁵ | `roles/iam.serviceAccountTokenCreator` | snapshot transfer SA⁴ | Granted by `create cloudsql`: with Cloud SQL, ate-api-server's pod runs as this account | | atelet WI principal¹ | `roles/storage.objectViewer` | asset bucket (`BUCKET_NAME-assets`) | Read micro-VM sandbox assets | | default compute SA³ | `roles/artifactregistry.reader` | project² | Nodes pull images from Artifact Registry, which also serves gcr.io | @@ -253,6 +261,8 @@ cluster must have it enabled too, or their GCS calls fail with `401`). ² Project-level today; scoping these down is tracked in the TODOs in `cmd/iam.go`. ³ `PROJECT_NUMBER-compute@developer.gserviceaccount.com`. It gets no storage role, since root on a node can use it. Installs set up before this also granted it `roles/storage.objectViewer` on the project. Remove that with `gcloud projects remove-iam-policy-binding "${PROJECT_ID}" --member="serviceAccount:${PROJECT_NUMBER}-compute@developer.gserviceaccount.com" --role=roles/storage.objectViewer --condition=None`. +⁴ `NAME@PROJECT_ID.iam.gserviceaccount.com`, with `NAME` from `SNAPSHOT_TRANSFER_GSA_NAME`. setup-gcp and `hack/teardown.sh` default it to `ate-snapshot-transfer`. ate-setup has no default and requires it on GKE. +⁵ `ate-api-server@PROJECT_ID.iam.gserviceaccount.com` by default, see [cloud-sql.md](cloud-sql.md). Manual equivalent: @@ -260,13 +270,23 @@ Manual equivalent: WI="principal://iam.googleapis.com/projects/${PROJECT_NUMBER}/locations/global/workloadIdentityPools/${PROJECT_ID}.svc.id.goog/subject/ns/ate-system/sa" ATELET="${WI}/atelet" ATE_API="${WI}/ate-api-server" +TRANSFER_NAME="${SNAPSHOT_TRANSFER_GSA_NAME:-ate-snapshot-transfer}" +TRANSFER="${TRANSFER_NAME}@${PROJECT_ID}.iam.gserviceaccount.com" gcloud projects add-iam-policy-binding "${PROJECT_ID}" \ --member="${ATELET}" --role=roles/artifactregistry.reader +gcloud iam service-accounts create "${TRANSFER_NAME}" --project="${PROJECT_ID}" \ + --display-name="Substrate snapshot transfers" +gcloud iam service-accounts add-iam-policy-binding "${TRANSFER}" --project="${PROJECT_ID}" \ + --member="${ATE_API}" --role=roles/iam.serviceAccountTokenCreator gcloud storage buckets add-iam-policy-binding "gs://${BUCKET_NAME}" \ --member="${ATE_API}" --role=roles/storage.objectAdmin gcloud storage buckets add-iam-policy-binding "gs://${BUCKET_NAME}" \ --member="${ATE_API}" --role=roles/storage.bucketViewer +gcloud storage buckets add-iam-policy-binding "gs://${BUCKET_NAME}" \ + --member="serviceAccount:${TRANSFER}" --role=roles/storage.objectAdmin +gcloud storage buckets add-iam-policy-binding "gs://${BUCKET_NAME}" \ + --member="serviceAccount:${TRANSFER}" --role=roles/storage.bucketViewer gcloud storage buckets add-iam-policy-binding "gs://${BUCKET_NAME}-assets" \ --member="${ATELET}" --role=roles/storage.objectViewer ``` @@ -279,6 +299,13 @@ the digests of its assets, which do not change when the assets move to the asset bucket, so a node keeps fetching them from the snapshot bucket. The commands below still apply when the new cluster reuses the GCP project. +Run `create iam` before deploying a Substrate release whose credential +exchange impersonates the snapshot transfer account. It creates the account +and its grants, and ate-setup refuses to deploy ate-api-server on GKE until +`SNAPSHOT_TRANSFER_GSA_NAME` names it. With Cloud SQL, rerun +`create cloudsql` too, so its account may create the transfer account's +tokens. + setup-gcp only adds bindings. An install set up before the asset bucket existed still grants atelet object admin on the project and on the snapshot bucket. Upgrade Substrate first, to a release whose node plugin gets snapshot @@ -342,6 +369,7 @@ go run ./tools/setup-gcp bootstrap [flags] | `--boot-disk-type` | Boot disk type for the node pool (empty = GKE default). | `BOOT_DISK_TYPE` | None | | `--bucket-name` | Name of the GCS bucket for snapshots. | `BUCKET_NAME` | None (Required*) | | `--dashboard-dir` | Directory containing dashboard JSON files. | `DASHBOARD_DIR` | `tools/setup-gcp/dashboards` | +| `--transfer-gsa-name` | Name of the snapshot transfer service account to create. ate-setup and `hack/teardown.sh` read the same variable. | `SNAPSHOT_TRANSFER_GSA_NAME` | `ate-snapshot-transfer` | *\*Note: Required unless the `BUCKET_NAME` environment variable is set.* diff --git a/tools/setup-gcp/cloud-sql.md b/tools/setup-gcp/cloud-sql.md index 561e7093e9..e23db311cf 100644 --- a/tools/setup-gcp/cloud-sql.md +++ b/tools/setup-gcp/cloud-sql.md @@ -46,6 +46,7 @@ This idempotently creates: | Google service account | `ate-api-server@.iam.gserviceaccount.com` | | Project IAM roles | `roles/cloudsql.client`, `roles/cloudsql.instanceUser` on the GSA | | Workload Identity binding | `roles/iam.workloadIdentityUser` for `.svc.id.goog[ate-system/ate-api-server]` on the GSA | +| Token creator binding | `roles/iam.serviceAccountTokenCreator` for the GSA on the snapshot transfer service account, named by `SNAPSHOT_TRANSFER_GSA_NAME` and `ate-snapshot-transfer` by default, created if missing. ate-api-server's pod runs as the GSA, and its credential exchange impersonates the transfer account | | IAM database user | the GSA, type `CLOUD_IAM_SERVICE_ACCOUNT` | > **Note:** The repo otherwise uses WIF-direct diff --git a/tools/setup-gcp/cmd/api.go b/tools/setup-gcp/cmd/api.go index 586eca8f31..210a53f645 100644 --- a/tools/setup-gcp/cmd/api.go +++ b/tools/setup-gcp/cmd/api.go @@ -35,6 +35,8 @@ func enableRequiredAPIs(ctx context.Context, cfg *Config) error { "artifactregistry.googleapis.com", "cloudresourcemanager.googleapis.com", "container.googleapis.com", + "iam.googleapis.com", + "iamcredentials.googleapis.com", "networkconnectivity.googleapis.com", "serviceusage.googleapis.com", "storage.googleapis.com", diff --git a/tools/setup-gcp/cmd/bootstrap.go b/tools/setup-gcp/cmd/bootstrap.go index 08c3137479..6e3af7fe64 100644 --- a/tools/setup-gcp/cmd/bootstrap.go +++ b/tools/setup-gcp/cmd/bootstrap.go @@ -24,7 +24,7 @@ import ( var bootstrapCmd = &cobra.Command{ Use: "bootstrap", Short: "Fully bootstrap the GCP environment", - Long: `Runs all setup steps in order: enable APIs, create cluster, create bucket, grant IAM permissions, and create dashboards.`, + Long: `Runs all setup steps in order: enable APIs, create cluster, create buckets, create the snapshot transfer service account, grant IAM permissions, and create dashboards.`, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() warnDeprecatedMachineTypeEnv(cmd) @@ -40,37 +40,42 @@ var bootstrapCmd = &cobra.Command{ slog.Info("Starting full bootstrap...") - slog.Info("Step 1/7: Enabling required APIs...") + slog.Info("Step 1/8: Enabling required APIs...") if err := enableRequiredAPIs(ctx, &cfg); err != nil { return err } - slog.Info("Step 2/7: Creating GKE Cluster...") + slog.Info("Step 2/8: Creating GKE Cluster...") if err := createClusterIdempotent(ctx, &cfg); err != nil { return err } - slog.Info("Step 3/7: Creating GCS buckets for snapshots and micro-VM assets...") + slog.Info("Step 3/8: Creating GCS buckets for snapshots and micro-VM assets...") if err := createBuckets(ctx, &cfg); err != nil { return err } - slog.Info("Step 4/7: Granting GKE Node permissions...") + slog.Info("Step 4/8: Granting GKE Node permissions...") if err := grantGkeNodePermissions(ctx, &cfg); err != nil { return err } - slog.Info("Step 5/7: Granting Atelet permissions...") + slog.Info("Step 5/8: Granting Atelet permissions...") if err := grantAteletPermissions(ctx, &cfg); err != nil { return err } - slog.Info("Step 6/7: Creating IAM policy bindings for the buckets...") + slog.Info("Step 6/8: Creating the snapshot transfer service account...") + if err := setupTransferServiceAccount(ctx, &cfg, workloadIdentityPrincipal(&cfg, "ate-api-server")); err != nil { + return err + } + + slog.Info("Step 7/8: Creating IAM policy bindings for the buckets...") if err := createIamPolicyBindings(ctx, &cfg); err != nil { return err } - slog.Info("Step 7/7: Creating Monitoring Dashboards...") + slog.Info("Step 8/8: Creating Monitoring Dashboards...") if err := createMonitoringDashboards(ctx, &cfg); err != nil { return err } @@ -95,5 +100,6 @@ func init() { bootstrapCmd.Flags().Int32Var(&cfg.BootDiskSizeGB, "boot-disk-size", getEnv("BOOT_DISK_SIZE_GB", int32(0)), "Boot disk size in GB for the node pool; 0 = GKE default (100 GB) [env: BOOT_DISK_SIZE_GB]") bootstrapCmd.Flags().StringVar(&cfg.BootDiskType, "boot-disk-type", getEnv("BOOT_DISK_TYPE", ""), "Boot disk type for the node pool; empty = GKE default [env: BOOT_DISK_TYPE]") bootstrapCmd.Flags().StringVar(&cfg.BucketName, "bucket-name", getEnv("BUCKET_NAME", ""), "Name of the GCS bucket for snapshots [env: BUCKET_NAME]") + addTransferGSANameFlag(bootstrapCmd) bootstrapCmd.Flags().StringVar(&cfg.DashboardDir, "dashboard-dir", getEnv("DASHBOARD_DIR", "tools/setup-gcp/dashboards"), "Directory containing dashboard JSON files [env: DASHBOARD_DIR]") } diff --git a/tools/setup-gcp/cmd/bucket.go b/tools/setup-gcp/cmd/bucket.go index aca29daef1..2cb8fde59d 100644 --- a/tools/setup-gcp/cmd/bucket.go +++ b/tools/setup-gcp/cmd/bucket.go @@ -23,6 +23,7 @@ import ( "slices" "strconv" "strings" + "time" "cloud.google.com/go/iam" "cloud.google.com/go/storage" @@ -102,25 +103,28 @@ func createBucket(ctx context.Context, cfg *Config, name string) error { return nil } -// bucketGrant is a role on a bucket for an ate-system Kubernetes service -// account. +// bucketGrant is a role on a bucket for an IAM member. type bucketGrant struct { - bucket string - subject string - role string + bucket string + member string + role string } // bucketGrants are the roles setup-gcp grants on the two buckets. // ate-api-server copies snapshots for tags and deletes the ones nothing refers -// to, so it gets objectAdmin, plus bucketViewer to resolve the bucket. atelet -// reads only micro-VM sandbox assets with its own identity: on GCS every -// snapshot transfer runs on a token scoped to its actor, issued by the -// credential exchange. +// to, so it gets objectAdmin, plus bucketViewer to resolve the bucket. The +// transfer service account gets the same: the credential exchange downscopes +// its token to each actor's prefixes for every snapshot transfer. atelet reads +// only micro-VM sandbox assets with its own identity. func bucketGrants(cfg *Config) []bucketGrant { + apiServer := workloadIdentityPrincipal(cfg, "ate-api-server") + transfer := transferServiceAccountMember(cfg) return []bucketGrant{ - {bucket: cfg.BucketName, subject: "ate-api-server", role: "roles/storage.objectAdmin"}, - {bucket: cfg.BucketName, subject: "ate-api-server", role: "roles/storage.bucketViewer"}, - {bucket: cfg.assetBucketName(), subject: "atelet", role: "roles/storage.objectViewer"}, + {bucket: cfg.BucketName, member: apiServer, role: "roles/storage.objectAdmin"}, + {bucket: cfg.BucketName, member: apiServer, role: "roles/storage.bucketViewer"}, + {bucket: cfg.BucketName, member: transfer, role: "roles/storage.objectAdmin"}, + {bucket: cfg.BucketName, member: transfer, role: "roles/storage.bucketViewer"}, + {bucket: cfg.assetBucketName(), member: workloadIdentityPrincipal(cfg, "atelet"), role: "roles/storage.objectViewer"}, } } @@ -148,13 +152,34 @@ func createIamPolicyBindings(ctx context.Context, cfg *Config) error { grants := bucketGrants(cfg) for i, name := range names { - if !addGrants(policies[i], cfg, name, grants) { - slog.Info("IAM policy is already correct", slog.String("bucket", name)) - continue - } - slog.Info("Setting IAM policy for bucket", slog.String("bucket", name)) - if err := client.Bucket(name).IAM().SetPolicy(ctx, policies[i]); err != nil { - return fmt.Errorf("set bucket %s iam policy: %w", name, err) + handle := client.Bucket(name).IAM() + policy := policies[i] + // A service account created moments earlier can take a while to + // propagate, and granting to it too soon fails with "does not + // exist". Re-read the policy each attempt so etag conflicts also + // resolve, as grantCloudSQLProjectRoles does. + for attempt := 1; ; attempt++ { + if !addGrants(policy, cfg, name, grants) { + slog.Info("IAM policy is already correct", slog.String("bucket", name)) + break + } + slog.Info("Setting IAM policy for bucket", slog.String("bucket", name)) + err := handle.SetPolicy(ctx, policy) + if err == nil { + break + } + if attempt >= 6 { + return fmt.Errorf("set bucket %s iam policy: %w", name, err) + } + slog.Warn("Setting bucket IAM policy failed, retrying...", slog.String("bucket", name), slog.Int("attempt", attempt), slog.Any("err", err)) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(10 * time.Second): + } + if policy, err = handle.Policy(ctx); err != nil { + return fmt.Errorf("get bucket %s iam policy: %w", name, err) + } } } return nil @@ -168,12 +193,11 @@ func addGrants(policy *iam.Policy, cfg *Config, name string, grants []bucketGran if g.bucket != name { continue } - member := fmt.Sprintf("principal://iam.googleapis.com/projects/%s/locations/global/workloadIdentityPools/%s.svc.id.goog/subject/ns/ate-system/sa/%s", cfg.ProjectNumber, cfg.ProjectID, g.subject) - if hasUnconditionalBinding(policy, g.role, member) { + if hasUnconditionalBinding(policy, g.role, g.member) { continue } - slog.Info("Adding role to member", slog.String("bucket", name), slog.String("role", g.role), slog.String("member", member)) - policy.Add(member, iam.RoleName(g.role)) + slog.Info("Adding role to member", slog.String("bucket", name), slog.String("role", g.role), slog.String("member", g.member)) + policy.Add(g.member, iam.RoleName(g.role)) changed = true } return changed diff --git a/tools/setup-gcp/cmd/bucket_test.go b/tools/setup-gcp/cmd/bucket_test.go index 56da95968a..3e3237c2b7 100644 --- a/tools/setup-gcp/cmd/bucket_test.go +++ b/tools/setup-gcp/cmd/bucket_test.go @@ -101,19 +101,35 @@ func TestValidateBucketFlags(t *testing.T) { } } -// TestAteletStorageAccess pins atelet's storage access to read on the asset -// bucket. -func TestAteletStorageAccess(t *testing.T) { - cfg := &Config{BucketName: "snaps"} - var got []bucketGrant - for _, g := range bucketGrants(cfg) { - if g.subject == "atelet" { - got = append(got, g) +// TestBucketGrantsPerMember pins what each member may reach: atelet reads +// the asset bucket only, and the transfer account, like ate-api-server, may +// read and write the snapshot bucket only. +func TestBucketGrantsPerMember(t *testing.T) { + cfg := &Config{ProjectID: "proj", ProjectNumber: "123", BucketName: "snaps", TransferGSAName: "ate-snapshot-transfer"} + grantsOf := func(member string) []bucketGrant { + var got []bucketGrant + for _, g := range bucketGrants(cfg) { + if g.member == member { + got = append(got, g) + } } + return got } - want := []bucketGrant{{bucket: "snaps-assets", subject: "atelet", role: "roles/storage.objectViewer"}} - if !slices.Equal(got, want) { - t.Errorf("atelet bucket grants = %v, want %v", got, want) + atelet := workloadIdentityPrincipal(cfg, "atelet") + transfer := "serviceAccount:ate-snapshot-transfer@proj.iam.gserviceaccount.com" + for _, tc := range []struct { + member string + want []bucketGrant + }{ + {atelet, []bucketGrant{{bucket: "snaps-assets", member: atelet, role: "roles/storage.objectViewer"}}}, + {transfer, []bucketGrant{ + {bucket: "snaps", member: transfer, role: "roles/storage.objectAdmin"}, + {bucket: "snaps", member: transfer, role: "roles/storage.bucketViewer"}, + }}, + } { + if got := grantsOf(tc.member); !slices.Equal(got, tc.want) { + t.Errorf("grants for %s = %v, want %v", tc.member, got, tc.want) + } } } diff --git a/tools/setup-gcp/cmd/cloudsql.go b/tools/setup-gcp/cmd/cloudsql.go index 8bd5ab282a..7c0ca565a1 100644 --- a/tools/setup-gcp/cmd/cloudsql.go +++ b/tools/setup-gcp/cmd/cloudsql.go @@ -294,25 +294,15 @@ func createCloudSQLGSA(ctx context.Context, cfg *Config) error { if err != nil { return fmt.Errorf("get service account iam policy: %w", err) } - const wiRole = "roles/iam.workloadIdentityUser" - for _, b := range policy.Bindings { - if b.Role == wiRole { - for _, m := range b.Members { - if m == member { - slog.Info("Workload Identity binding exists. Skipping.") - return nil - } - } - b.Members = append(b.Members, member) - return setGSAPolicy(ctx, svc, resource, policy) - } + if !addServiceAccountBinding(policy, "roles/iam.workloadIdentityUser", member) { + slog.Info("Workload Identity binding exists. Skipping.") + return nil } - policy.Bindings = append(policy.Bindings, &iam.Binding{Role: wiRole, Members: []string{member}}) + slog.Info("Binding Workload Identity user to service account...") return setGSAPolicy(ctx, svc, resource, policy) } func setGSAPolicy(ctx context.Context, svc *iam.Service, resource string, policy *iam.Policy) error { - slog.Info("Binding Workload Identity user to service account...") _, err := svc.Projects.ServiceAccounts.SetIamPolicy(resource, &iam.SetIamPolicyRequest{Policy: policy}).Context(ctx).Do() if err != nil { return fmt.Errorf("set service account iam policy: %w", err) @@ -440,6 +430,11 @@ var cloudsqlCmd = &cobra.Command{ if err := createCloudSQLGSA(ctx, &cfg); err != nil { return err } + // ate-api-server's pod runs as this account once Cloud SQL is on, so + // its credential exchange creates transfer account tokens with it. + if err := setupTransferServiceAccount(ctx, &cfg, "serviceAccount:"+cloudSQLGSAEmail(&cfg)); err != nil { + return err + } if err := grantCloudSQLProjectRoles(ctx, &cfg); err != nil { return err } @@ -458,5 +453,6 @@ func init() { cloudsqlCmd.Flags().StringVar(&cfg.CloudSQLEdition, "edition", getEnv("CLOUDSQL_EDITION", "enterprise"), "Instance edition: enterprise | enterprise-plus (enables the local-SSD data cache) [env: CLOUDSQL_EDITION]") cloudsqlCmd.Flags().Int64Var(&cfg.CloudSQLStorageGB, "storage-size", getEnv("CLOUDSQL_STORAGE_GB", int64(0)), "Data disk size in GB; 0 = Cloud SQL default (10 GB, auto-resizing). PD IOPS scale with size [env: CLOUDSQL_STORAGE_GB]") cloudsqlCmd.Flags().StringVar(&cfg.CloudSQLGSAName, "gsa-name", getEnv("CLOUDSQL_GSA_NAME", "ate-api-server"), "Name of the Google service account to create for Workload Identity + IAM database auth [env: CLOUDSQL_GSA_NAME]") + addTransferGSANameFlag(cloudsqlCmd) cloudsqlCmd.Flags().StringVar(&cfg.Network, "network", getEnv("NETWORK", "default"), "VPC network name (must match the cluster's) [env: NETWORK]") } diff --git a/tools/setup-gcp/cmd/common.go b/tools/setup-gcp/cmd/common.go index 53782ef2c8..9eb30093f0 100644 --- a/tools/setup-gcp/cmd/common.go +++ b/tools/setup-gcp/cmd/common.go @@ -15,6 +15,7 @@ package cmd import ( + "fmt" "log/slog" "os" "strconv" @@ -45,6 +46,10 @@ type Config struct { BucketName string + // TransferGSAName is the name of the snapshot transfer service account + // (SNAPSHOT_TRANSFER_GSA_NAME), the variable ate-setup also reads. + TransferGSAName string + CloudSQLInstance string CloudSQLTier string CloudSQLEdition string @@ -122,3 +127,9 @@ func getEnv[T getEnvType](key string, fallback T) T { func (c *Config) assetBucketName() string { return c.BucketName + "-assets" } + +// workloadIdentityPrincipal is the IAM member for the ate-system Kubernetes +// service account ksa under GKE Workload Identity Federation. +func workloadIdentityPrincipal(cfg *Config, ksa string) string { + return fmt.Sprintf("principal://iam.googleapis.com/projects/%s/locations/global/workloadIdentityPools/%s.svc.id.goog/subject/ns/ate-system/sa/%s", cfg.ProjectNumber, cfg.ProjectID, ksa) +} diff --git a/tools/setup-gcp/cmd/iam.go b/tools/setup-gcp/cmd/iam.go index 8ae3de8ac1..ccae453f67 100644 --- a/tools/setup-gcp/cmd/iam.go +++ b/tools/setup-gcp/cmd/iam.go @@ -152,6 +152,7 @@ var iamCmd = &cobra.Command{ RunE: func(cmd *cobra.Command, args []string) error { gkeNodes, _ := cmd.Flags().GetBool("gke-nodes") atelet, _ := cmd.Flags().GetBool("atelet") + transferAccount, _ := cmd.Flags().GetBool("transfer-service-account") bucketBindings, _ := cmd.Flags().GetBool("bucket-bindings") if err := validateIamFlags(cmd.Context(), &cfg, bucketBindings); err != nil { @@ -168,6 +169,13 @@ var iamCmd = &cobra.Command{ return err } } + // Before the bucket bindings, which grant the account: IAM refuses a + // binding for an account that does not exist. + if transferAccount { + if err := setupTransferServiceAccount(cmd.Context(), &cfg, workloadIdentityPrincipal(&cfg, "ate-api-server")); err != nil { + return err + } + } if bucketBindings { if err := createIamPolicyBindings(cmd.Context(), &cfg); err != nil { return err @@ -182,5 +190,7 @@ func init() { iamCmd.Flags().StringVar(&cfg.BucketName, "bucket", getEnv("BUCKET_NAME", ""), "GCS bucket name (required for bucket bindings) [env: BUCKET_NAME]") iamCmd.Flags().Bool("gke-nodes", true, "Grant GKE nodes permission to pull images") iamCmd.Flags().Bool("atelet", true, "Grant atelet project-level permissions") - iamCmd.Flags().Bool("bucket-bindings", true, "Grant ate-api-server access to the snapshot bucket, and atelet read access to the asset bucket") + addTransferGSANameFlag(iamCmd) + iamCmd.Flags().Bool("transfer-service-account", true, "Create the snapshot transfer service account and let ate-api-server create tokens for it") + iamCmd.Flags().Bool("bucket-bindings", true, "Grant ate-api-server and the snapshot transfer service account access to the snapshot bucket, and atelet read access to the asset bucket") } diff --git a/tools/setup-gcp/cmd/transferaccount.go b/tools/setup-gcp/cmd/transferaccount.go new file mode 100644 index 0000000000..2c1128c29e --- /dev/null +++ b/tools/setup-gcp/cmd/transferaccount.go @@ -0,0 +1,132 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "fmt" + "log/slog" + "slices" + "time" + + "github.com/spf13/cobra" + iam "google.golang.org/api/iam/v1" + + "go.ate.dev/substrate/internal/installdefaults" +) + +// tokenCreatorRole lets a member create tokens for a service account, which +// ate-api-server's credential exchange does for the transfer account. +const tokenCreatorRole = "roles/iam.serviceAccountTokenCreator" + +// addTransferGSANameFlag registers the transfer account's name on cmd, read +// from the variable ate-setup also reads. +func addTransferGSANameFlag(cmd *cobra.Command) { + cmd.Flags().StringVar(&cfg.TransferGSAName, "transfer-gsa-name", getEnv("SNAPSHOT_TRANSFER_GSA_NAME", installdefaults.TransferServiceAccountName), + "Name of the snapshot transfer service account to create [env: SNAPSHOT_TRANSFER_GSA_NAME]") +} + +func transferGSAEmail(cfg *Config) string { + return fmt.Sprintf("%s@%s.iam.gserviceaccount.com", cfg.TransferGSAName, cfg.ProjectID) +} + +// transferServiceAccountMember is the transfer account as an IAM member. +func transferServiceAccountMember(cfg *Config) string { + return "serviceAccount:" + transferGSAEmail(cfg) +} + +// setupTransferServiceAccount creates the snapshot transfer service account +// unless it exists, and lets each of members create tokens for it. +func setupTransferServiceAccount(ctx context.Context, cfg *Config, members ...string) error { + svc, err := iam.NewService(ctx) + if err != nil { + return fmt.Errorf("create iam client: %w", err) + } + email := transferGSAEmail(cfg) + resource := fmt.Sprintf("projects/%s/serviceAccounts/%s", cfg.ProjectID, email) + _, err = svc.Projects.ServiceAccounts.Get(resource).Context(ctx).Do() + switch { + case err == nil: + slog.Info("Service account exists. Skipping create.", slog.String("gsa", email)) + case isNotFound(err): + slog.Info("Creating service account...", slog.String("gsa", email)) + _, err = svc.Projects.ServiceAccounts.Create("projects/"+cfg.ProjectID, &iam.CreateServiceAccountRequest{ + AccountId: cfg.TransferGSAName, + ServiceAccount: &iam.ServiceAccount{DisplayName: "Substrate snapshot transfers"}, + }).Context(ctx).Do() + if err != nil { + return fmt.Errorf("create service account %s: %w", email, err) + } + default: + return fmt.Errorf("get service account %s: %w", email, err) + } + + // A service account created moments earlier, this one or a member, can + // take a while to propagate, and granting too soon fails with "does not + // exist". Re-read the policy each attempt so etag conflicts also resolve, + // as grantCloudSQLProjectRoles does. + for attempt := 1; ; attempt++ { + err := grantTokenCreator(ctx, svc, resource, email, members) + if err == nil { + return nil + } + if attempt >= 6 { + return err + } + slog.Warn("Granting token creator failed, retrying...", slog.String("gsa", email), slog.Int("attempt", attempt), slog.Any("err", err)) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(10 * time.Second): + } + } +} + +// grantTokenCreator grants members token creator on the service account. +func grantTokenCreator(ctx context.Context, svc *iam.Service, resource, email string, members []string) error { + policy, err := svc.Projects.ServiceAccounts.GetIamPolicy(resource).Context(ctx).Do() + if err != nil { + return fmt.Errorf("get service account %s iam policy: %w", email, err) + } + changed := false + for _, member := range members { + if addServiceAccountBinding(policy, tokenCreatorRole, member) { + slog.Info("Granting token creator on service account", slog.String("gsa", email), slog.String("member", member)) + changed = true + } + } + if !changed { + slog.Info("Service account IAM policy is already correct", slog.String("gsa", email)) + return nil + } + return setGSAPolicy(ctx, svc, resource, policy) +} + +// addServiceAccountBinding adds member to role in policy, and reports whether +// it added anything. +func addServiceAccountBinding(policy *iam.Policy, role, member string) bool { + for _, b := range policy.Bindings { + if b.Role != role || b.Condition != nil { + continue + } + if slices.Contains(b.Members, member) { + return false + } + b.Members = append(b.Members, member) + return true + } + policy.Bindings = append(policy.Bindings, &iam.Binding{Role: role, Members: []string{member}}) + return true +} diff --git a/tools/setup-gcp/cmd/transferaccount_test.go b/tools/setup-gcp/cmd/transferaccount_test.go new file mode 100644 index 0000000000..3f72d468fe --- /dev/null +++ b/tools/setup-gcp/cmd/transferaccount_test.go @@ -0,0 +1,63 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "slices" + "testing" + + iam "google.golang.org/api/iam/v1" +) + +func TestAddServiceAccountBinding(t *testing.T) { + const member = "principal://iam.googleapis.com/projects/123/locations/global/workloadIdentityPools/proj.svc.id.goog/subject/ns/ate-system/sa/ate-api-server" + for _, tc := range []struct { + name string + bindings []*iam.Binding + wantChanged bool + wantMembers []string + }{ + {name: "no binding yet", wantChanged: true, wantMembers: []string{member}}, + {name: "joins the role's binding", bindings: []*iam.Binding{{Role: tokenCreatorRole, Members: []string{"user:a@example.com"}}}, + wantChanged: true, wantMembers: []string{"user:a@example.com", member}}, + {name: "already a member", bindings: []*iam.Binding{{Role: tokenCreatorRole, Members: []string{member}}}, + wantMembers: []string{member}}, + {name: "a conditional binding does not count", bindings: []*iam.Binding{{Role: tokenCreatorRole, Members: []string{member}, Condition: &iam.Expr{Expression: "false"}}}, + wantChanged: true, wantMembers: []string{member}}, + } { + t.Run(tc.name, func(t *testing.T) { + policy := &iam.Policy{Bindings: tc.bindings} + if got := addServiceAccountBinding(policy, tokenCreatorRole, member); got != tc.wantChanged { + t.Errorf("changed = %v, want %v", got, tc.wantChanged) + } + var got []string + for _, b := range policy.Bindings { + if b.Role == tokenCreatorRole && b.Condition == nil { + got = append(got, b.Members...) + } + } + if !slices.Equal(got, tc.wantMembers) { + t.Errorf("unconditional token creator members = %v, want %v", got, tc.wantMembers) + } + }) + } +} + +func TestTransferServiceAccountMember(t *testing.T) { + cfg := &Config{ProjectID: "proj", TransferGSAName: "xfer"} + if got, want := transferServiceAccountMember(cfg), "serviceAccount:xfer@proj.iam.gserviceaccount.com"; got != want { + t.Errorf("member = %q, want %q", got, want) + } +} From 3e2797bc15a7b0cfdabaf94042f19a425c6c0453 Mon Sep 17 00:00:00 2001 From: pmandewalkar Date: Wed, 21 Oct 2026 23:59:48 +0000 Subject: [PATCH 06/12] tarutil: drop fsync of rootfs-upper and durable-dir tars (#1954) (#3026) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #1954 , depends on #3050 The `f.Sync()` in `tarutil` was a high cost of microVM suspend under load, and per the discussion on the issue local-only snapshots aren't guaranteed to survive a node restart, and local snapshots from an earlier boot aren't trusted (#3050 ). Since all actors share one filesystem, the fsync also waits behind other actors' writes in the ext4 journal. On a bare-metal microVM node (sweperf, one worker), the worker spent ~95% of its blocked time in this fsync (`jbd2_log_wait_commit`) at saturation. Removing it cut SuspendActor p50 from 10.6 s → 0.81 s at 40 users (full table in #1954). It doesn't change where the node saturates. - [ X] Tests pass - [ ] Docs updated --- internal/tarutil/tarutil.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/internal/tarutil/tarutil.go b/internal/tarutil/tarutil.go index 2940c5ac2f..9c0150a3b4 100644 --- a/internal/tarutil/tarutil.go +++ b/internal/tarutil/tarutil.go @@ -141,15 +141,16 @@ func create(ctx context.Context, tarPath, srcDir string, skip SkipFunc, includeR if err := tw.Close(); err != nil { return fmt.Errorf("closing tar %q: %w", tarPath, err) } - // The buffer has to reach the file before the sync below, or the sync - // durably persists a truncated archive. + // The end of the archive is still in the buffer; flush it before returning. if err := bw.Flush(); err != nil { return fmt.Errorf("flushing tar %q: %w", tarPath, err) } - // Durable-dir tars are handed to atelet for upload as soon as we return, so - // flush to disk rather than trusting the page cache to outlive us. - if err := f.Sync(); err != nil { - return fmt.Errorf("syncing tar %q: %w", tarPath, err) + // No fsync: local-only snapshots aren't guaranteed to survive a node + // restart, and atelet doesn't trust a local snapshot written before the + // current boot (#3050). fsync here also waits behind other actors' writes + // in the shared ext4 journal. See #1954. + if err := f.Close(); err != nil { + return fmt.Errorf("closing %q: %w", tarPath, err) } return nil } From 931d993b08c7558a91556fa7fd66d8c5200dbc39 Mon Sep 17 00:00:00 2001 From: Yufan Su Date: Thu, 22 Oct 2026 03:26:57 +0000 Subject: [PATCH 07/12] atenet-router: raise the tunnel circuit breaker to 1m (#3087) At 1k nodes and 200k actors, atenet-router starts failing ingress with `503 upstream connect error or disconnect/reset before headers. reset reason: remote connection failure`. The requests that fail are the ones that need a new tunnel. `actor_connect` refuses the tunnel once a circuit breaker set by `tunnelCircuitBreakerLimit` (65536) is full. - Split the limit into tunnelCircuitBreakerLimit and actorConnectCircuitBreakerLimit, so each cluster can be tuned on its own. - Set both to 1048576. For actor_connect that covers 11,000 workers on up to about 95 Envoy threads. Leaving the breakers unset would mean Envoy's default of 1024. - Tuning actor_connect's limits is tracked in #3100. - [ ] Tests pass - [ ] Docs updated --- cmd/atenet/internal/router/xds.go | 39 +++++++++++-------- .../router/xds_upstream_connect_test.go | 10 ++--- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index fa0f982858..8f9e798eda 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -217,9 +217,13 @@ const upstreamSANsFilterStateKey = "envoy.network.upstream_subject_alt_names" // domain. const trustDomainSPIFFEPrefix = "spiffe://" + installdefaults.AteletTrustDomain + "/" -// tunnelCircuitBreakerLimit lifts Envoy's default of 1024: there is a tunnel -// per active actor and port, and idle ones count too. -const tunnelCircuitBreakerLimit = 65536 +// tunnelCircuitBreakerLimit lifts Envoy's default of 1024 on the actor_tunnel +// clusters. +const tunnelCircuitBreakerLimit = 1048576 + +// actorConnectCircuitBreakerLimit lifts Envoy's default of 1024 on +// actor_connect. +const actorConnectCircuitBreakerLimit = 1048576 // tunnelIdleTimeout closes unused tunnels, which hold actor connections. const tunnelIdleTimeout = 60 * time.Second @@ -939,7 +943,14 @@ func buildActorTunnelCluster(name string, http2 bool) *clusterv3.Cluster { }), }, }, - CircuitBreakers: tunnelCircuitBreakers(), + CircuitBreakers: &clusterv3.CircuitBreakers{ + Thresholds: []*clusterv3.CircuitBreakers_Thresholds{{ + Priority: corev3.RoutingPriority_DEFAULT, + MaxConnections: wrapperspb.UInt32(tunnelCircuitBreakerLimit), + MaxPendingRequests: wrapperspb.UInt32(tunnelCircuitBreakerLimit), + MaxRequests: wrapperspb.UInt32(tunnelCircuitBreakerLimit), + }}, + }, TypedExtensionProtocolOptions: map[string]*anypb.Any{ httpProtocolOptionsName: newAny(&httpv3.HttpProtocolOptions{ CommonHttpProtocolOptions: &corev3.HttpProtocolOptions{ @@ -970,7 +981,14 @@ func (x *XdsServer) buildActorConnectCluster() *clusterv3.Cluster { }, }, TransportSocket: x.buildUpstreamTransportSocket(), - CircuitBreakers: tunnelCircuitBreakers(), + CircuitBreakers: &clusterv3.CircuitBreakers{ + Thresholds: []*clusterv3.CircuitBreakers_Thresholds{{ + Priority: corev3.RoutingPriority_DEFAULT, + MaxConnections: wrapperspb.UInt32(actorConnectCircuitBreakerLimit), + MaxPendingRequests: wrapperspb.UInt32(actorConnectCircuitBreakerLimit), + MaxRequests: wrapperspb.UInt32(actorConnectCircuitBreakerLimit), + }}, + }, TypedExtensionProtocolOptions: map[string]*anypb.Any{ httpProtocolOptionsName: newAny(&httpv3.HttpProtocolOptions{ UpstreamProtocolOptions: &httpv3.HttpProtocolOptions_ExplicitHttpConfig_{ @@ -985,17 +1003,6 @@ func (x *XdsServer) buildActorConnectCluster() *clusterv3.Cluster { } } -func tunnelCircuitBreakers() *clusterv3.CircuitBreakers { - return &clusterv3.CircuitBreakers{ - Thresholds: []*clusterv3.CircuitBreakers_Thresholds{{ - Priority: corev3.RoutingPriority_DEFAULT, - MaxConnections: wrapperspb.UInt32(tunnelCircuitBreakerLimit), - MaxPendingRequests: wrapperspb.UInt32(tunnelCircuitBreakerLimit), - MaxRequests: wrapperspb.UInt32(tunnelCircuitBreakerLimit), - }}, - } -} - func (x *XdsServer) buildRoutes() *routev3.RouteConfiguration { var routes []*routev3.Route if x.upstreamConnect { diff --git a/cmd/atenet/internal/router/xds_upstream_connect_test.go b/cmd/atenet/internal/router/xds_upstream_connect_test.go index 28017f150a..f11090db42 100644 --- a/cmd/atenet/internal/router/xds_upstream_connect_test.go +++ b/cmd/atenet/internal/router/xds_upstream_connect_test.go @@ -365,7 +365,7 @@ func TestXdsServer_ActorTunnelClusters(t *testing.T) { } return opts } - checkCircuitBreakers := func(t *testing.T, c *clusterv3.Cluster) { + checkCircuitBreakers := func(t *testing.T, c *clusterv3.Cluster, want uint32) { t.Helper() limits := c.GetCircuitBreakers().GetThresholds()[0] for name, got := range map[string]uint32{ @@ -373,8 +373,8 @@ func TestXdsServer_ActorTunnelClusters(t *testing.T) { "max_pending_requests": limits.GetMaxPendingRequests().GetValue(), "max_requests": limits.GetMaxRequests().GetValue(), } { - if got != tunnelCircuitBreakerLimit { - t.Errorf("cluster %s %s = %d, want %d", c.GetName(), name, got, tunnelCircuitBreakerLimit) + if got != want { + t.Errorf("cluster %s %s = %d, want %d", c.GetName(), name, got, want) } } } @@ -393,7 +393,7 @@ func TestXdsServer_ActorTunnelClusters(t *testing.T) { if got := opts.GetCommonHttpProtocolOptions().GetIdleTimeout().AsDuration(); got != tunnelIdleTimeout { t.Errorf("http2=%v: tunnel idle timeout = %v, want %v", http2, got, tunnelIdleTimeout) } - checkCircuitBreakers(t, c) + checkCircuitBreakers(t, c, tunnelCircuitBreakerLimit) } x := NewXdsServer(0) @@ -419,5 +419,5 @@ func TestXdsServer_ActorTunnelClusters(t *testing.T) { if got := protocolOptions(t, c).GetExplicitHttpConfig().GetHttp2ProtocolOptions(); got == nil { t.Error("actor_connect is not HTTP/2; tunnels to one worker would each need a connection") } - checkCircuitBreakers(t, c) + checkCircuitBreakers(t, c, actorConnectCircuitBreakerLimit) } From ab4f3dbec79b77771a7cca820173a249b551c493 Mon Sep 17 00:00:00 2001 From: Yufan Su Date: Thu, 22 Oct 2026 03:27:10 +0000 Subject: [PATCH 08/12] atenet-egress: reload actor-identity CA bundle in atenet-egress when it rotates (#3081) Part of #3032 atenet-egress checks actor certificates in Envoy and again in the router ext_proc sidecar. Both read `/run/actor-id-ca-certs/ca.crt` once at startup, so after a CA rotation they reject certificates from the new CA until the pod restarts. - Envoy loads the CA from a filesystem SDS resource with `watched_directory`, like the serving cert. - The router passes a `credbundle.PoolLoader` to `egress.New`, which checks the file on each CONNECT. AI use: used Claude Code to write the change and tests. I reviewed the diff. - [x] Tests pass - [ ] Docs updated --- cmd/ate-setup/internal/steps/overlay_test.go | 3 +- cmd/atenet/internal/router/egress/egress.go | 63 ++++++++-------- .../internal/router/egress/egress_test.go | 74 ++++++++++++++----- .../internal/router/egress/request_test.go | 2 +- cmd/atenet/internal/router/router.go | 21 +++--- manifests/ate-install/atenet-egress.yaml | 26 +++++-- 6 files changed, 118 insertions(+), 71 deletions(-) diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index dd5202e5c9..9faa884ee0 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -225,6 +225,7 @@ func TestPatchAtenetEgressManifest(t *testing.T) { "sds-servicedns-cert.yaml", "sds-podidentity-cert.yaml", "sds-servicedns-validation.yaml", + "sds-actor-identity-validation.yaml", } { content, ok := data[key] if !ok { @@ -256,7 +257,7 @@ func TestPatchAtenetEgressManifest(t *testing.T) { t.Errorf("patched envoy.yaml contains watched_directory outside the SDS resource files: %q", line) } } - for _, key := range []string{"sds-servicedns-cert.yaml", "sds-podidentity-cert.yaml", "sds-servicedns-validation.yaml"} { + for _, key := range []string{"sds-servicedns-cert.yaml", "sds-podidentity-cert.yaml", "sds-servicedns-validation.yaml", "sds-actor-identity-validation.yaml"} { if !strings.Contains(data[key], "watched_directory") { t.Errorf("SDS resource %q is missing watched_directory; rotation would be silently broken", key) } diff --git a/cmd/atenet/internal/router/egress/egress.go b/cmd/atenet/internal/router/egress/egress.go index 1fd82c42dc..207c0a6d03 100644 --- a/cmd/atenet/internal/router/egress/egress.go +++ b/cmd/atenet/internal/router/egress/egress.go @@ -78,10 +78,9 @@ const deniedBody = "egress denied" // the traffic inside the tunnel against the actor's EgressPolicy. type Handler struct { apiClient ateapipb.ControlClient - // actorIdentityRoots is the actor-identity CA bundle every actor - // certificate must chain to. Nil means the gateway cannot authenticate - // anyone, and every CONNECT fails closed. - actorIdentityRoots *x509.CertPool + // loadActorIdentityRoots yields the actor-identity CA bundle every actor + // certificate must chain to. An error fails the CONNECT closed. + loadActorIdentityRoots func() (*x509.CertPool, error) // policies is the per-actor EgressPolicy cache every leg reads through. policies *policyCache // actorJWTs mints the actor JWTs that credential injections ask for. @@ -95,8 +94,10 @@ type Handler struct { providerName string } -// New builds the egress handler. actorIdentityRoots is the egress listener's -// trusted_ca; see verifyActorCertificate for why it is checked again here. +// New builds the egress handler. loadActorIdentityRoots yields the egress +// listener's trusted_ca; see verifyActorCertificate for why it is checked +// again here. It is called on every CONNECT, so a rotated bundle applies to +// the next tunnel. A nil loadActorIdentityRoots fails every CONNECT with 503. // policyCacheTTL of 0 fetches the policy on every callout. // // provider resolves an allowed rule's credential_uri injections on the @@ -104,14 +105,19 @@ type Handler struct { // denied. Actor JWT injections need no provider. providerName, when set, is // the provider this gateway serves; a credential URI naming another provider // is refused. -func New(apiClient ateapipb.ControlClient, actorIdentityRoots *x509.CertPool, policyCacheTTL time.Duration, provider credproviderpb.CredentialProviderClient, providerName string) *Handler { +func New(apiClient ateapipb.ControlClient, loadActorIdentityRoots func() (*x509.CertPool, error), policyCacheTTL time.Duration, provider credproviderpb.CredentialProviderClient, providerName string) *Handler { + if loadActorIdentityRoots == nil { + loadActorIdentityRoots = func() (*x509.CertPool, error) { + return nil, errors.New("no actor-identity CA configured") + } + } return &Handler{ - apiClient: apiClient, - actorIdentityRoots: actorIdentityRoots, - policies: newPolicyCache(apiClient, policyCacheTTL), - actorJWTs: actorjwt.New(apiClient), - provider: provider, - providerName: providerName, + apiClient: apiClient, + loadActorIdentityRoots: loadActorIdentityRoots, + policies: newPolicyCache(apiClient, policyCacheTTL), + actorJWTs: actorjwt.New(apiClient), + provider: provider, + providerName: providerName, } } @@ -146,14 +152,15 @@ func (h *Handler) handleConnect(ctx context.Context, md *extproc.RequestMetadata "egress denied: expected CONNECT, got %q", md.Method) } - // No roots means the gateway cannot authenticate anyone. Fail closed, and + // Without roots the gateway cannot authenticate anyone. Fail closed, and // as 503 rather than 403: this is our misconfiguration, not the actor's. - if h.actorIdentityRoots == nil { - return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_ServiceUnavailable, - "egress unavailable: no actor-identity CA configured") + roots, err := h.loadActorIdentityRoots() + if err != nil { + return extproc.Result{}, extproc.WrapReqError(envoy_type.StatusCode_ServiceUnavailable, err, + "egress unavailable: actor-identity CA failed to load") } - actorRef, err := h.authenticateActorCertificate(md) + actorRef, err := authenticateActorCertificate(md, roots) if err != nil { // The body stays generic on purpose: an actor that fails authentication // has not proven it is anyone, so it gets no detail about why. The @@ -292,13 +299,13 @@ func (h *Handler) validateActor(ctx context.Context, actorRef resources.ActorRef // authenticateActorCertificate turns the mTLS peer certificate Envoy recorded // on the request into a verified ActorRef, or an error describing why it // cannot be trusted. -func (h *Handler) authenticateActorCertificate(md *extproc.RequestMetadata) (resources.ActorRef, error) { +func authenticateActorCertificate(md *extproc.RequestMetadata, roots *x509.CertPool) (resources.ActorRef, error) { if certificate := md.Attribute(agentgatewayClientCertificateAttribute); certificate != "" { chain, err := parseCertificateChainPEM([]byte(certificate)) if err != nil { return resources.ActorRef{}, err } - return h.verifyActorCertificate(chain) + return verifyActorCertificate(chain, roots) } header := md.Header(forwardedClientCertHeader) if header == "" { @@ -308,7 +315,7 @@ func (h *Handler) authenticateActorCertificate(md *extproc.RequestMetadata) (res if err != nil { return resources.ActorRef{}, err } - return h.verifyActorCertificate(chain) + return verifyActorCertificate(chain, roots) } // verifyActorCertificate checks that chain[0] is a live, non-CA, client-auth @@ -322,7 +329,7 @@ func (h *Handler) authenticateActorCertificate(md *extproc.RequestMetadata) (res // It also keeps the handler safe if the Envoy config is ever loosened, and costs // one signature check per CONNECT rather than per request. The IsCA, ClientAuth-EKU, // and worker SPIFFE URI checks below have no Envoy-side equivalent at all. -func (h *Handler) verifyActorCertificate(chain []*x509.Certificate) (resources.ActorRef, error) { +func verifyActorCertificate(chain []*x509.Certificate, roots *x509.CertPool) (resources.ActorRef, error) { leaf := chain[0] intermediates := x509.NewCertPool() for _, cert := range chain[1:] { @@ -342,7 +349,7 @@ func (h *Handler) verifyActorCertificate(chain []*x509.Certificate) (resources.A } if _, err := leaf.Verify(x509.VerifyOptions{ - Roots: h.actorIdentityRoots, + Roots: roots, Intermediates: intermediates, CurrentTime: now, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, @@ -505,13 +512,3 @@ func mapEgressIdentityError(atespace, actorName string, err error) error { "egress denied for %q/%q: %v", atespace, actorName, err) } } - -// LoadActorIdentityRoots reads the actor-identity CA trust bundle the egress -// gateway verifies actor client certificates against. -func LoadActorIdentityRoots(pemBytes []byte) (*x509.CertPool, error) { - roots := x509.NewCertPool() - if !roots.AppendCertsFromPEM(pemBytes) { - return nil, fmt.Errorf("actor-identity CA bundle contains no certificates") - } - return roots, nil -} diff --git a/cmd/atenet/internal/router/egress/egress_test.go b/cmd/atenet/internal/router/egress/egress_test.go index 8fe77bc535..7f367cd3bf 100644 --- a/cmd/atenet/internal/router/egress/egress_test.go +++ b/cmd/atenet/internal/router/egress/egress_test.go @@ -86,10 +86,11 @@ func newTestCA(t *testing.T, commonName string) *testCA { return &testCA{cert: cert, key: key} } -func (ca *testCA) roots() *x509.CertPool { +// loadRoots is an actor-identity roots loader for New that trusts only ca. +func (ca *testCA) loadRoots() (*x509.CertPool, error) { pool := x509.NewCertPool() pool.AddCert(ca.cert) - return pool + return pool, nil } // actorCertOptions mutates the leaf template so each test can break exactly one @@ -165,8 +166,8 @@ func xfccHeaderDER(chain ...[]byte) string { // egressHandler builds a Handler with an allow-everything policy, so the // identity tests are about identity alone. -func egressHandler(roots *x509.CertPool, actor *ateapipb.Actor, err error) *Handler { - return New(&egressMockClient{actor: actor, err: err, policy: allowAllPolicy()}, roots, DefaultPolicyCacheTTL, nil, "") +func egressHandler(loadRoots func() (*x509.CertPool, error), actor *ateapipb.Actor, err error) *Handler { + return New(&egressMockClient{actor: actor, err: err, policy: allowAllPolicy()}, loadRoots, DefaultPolicyCacheTTL, nil, "") } // egressMockClient is the slice of ateapi the egress handler talks to. @@ -332,7 +333,7 @@ func wantStatus(t *testing.T, err error, want envoy_type.StatusCode) { func TestHandleRequestHeadersAllowsVerifiedActor(t *testing.T) { ca := newTestCA(t, "actor-identity-ca") leaf := ca.issueActorCert(t, "spiffe://substrate-actor.local/worker-for-actor/foo/bar", actorCertOptions{}) - h := egressHandler(ca.roots(), runningActor(), nil) + h := egressHandler(ca.loadRoots, runningActor(), nil) res, err := h.HandleRequestHeaders(context.Background(), egressMetadata(xfccHeader(leaf))) if err != nil { @@ -408,7 +409,7 @@ func TestConnectLegOpensForAnyRules(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - h := New(&egressMockClient{actor: runningActor(), policy: tc.policy}, ca.roots(), 0, nil, "") + h := New(&egressMockClient{actor: runningActor(), policy: tc.policy}, ca.loadRoots, 0, nil, "") md := egressMetadata(xfccHeader(leaf)) md.Host = "93.184.216.34:443" if tc.dialed != "" { @@ -467,7 +468,7 @@ func TestConnectLegWithoutRequestLegs(t *testing.T) { leaf := ca.issueActorCert(t, "spiffe://substrate-actor.local/worker-for-actor/foo/bar", actorCertOptions{}) certificate := string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leaf.Raw})) - h := New(&egressMockClient{actor: runningActor(), policy: httpPolicy("api.example.com")}, ca.roots(), 0, nil, "") + h := New(&egressMockClient{actor: runningActor(), policy: httpPolicy("api.example.com")}, ca.loadRoots, 0, nil, "") res, err := h.HandleRequestHeaders(context.Background(), agentgatewayEgressMetadata(certificate)) if err != nil { t.Fatalf("HandleRequestHeaders() error = %v, want the tunnel to open", err) @@ -475,7 +476,7 @@ func TestConnectLegWithoutRequestLegs(t *testing.T) { if got := dialedPortOf(res); got != "80" { t.Errorf("dialed port = %q, want %q", got, "80") } - h = New(&egressMockClient{actor: runningActor()}, ca.roots(), 0, nil, "") + h = New(&egressMockClient{actor: runningActor()}, ca.loadRoots, 0, nil, "") _, err = h.HandleRequestHeaders(context.Background(), agentgatewayEgressMetadata(certificate)) wantStatus(t, err, envoy_type.StatusCode_Forbidden) } @@ -483,7 +484,7 @@ func TestConnectLegWithoutRequestLegs(t *testing.T) { func TestHandleRequestHeadersAllowsAgentgatewayCertificateAttribute(t *testing.T) { ca := newTestCA(t, "actor-identity-ca") leaf := ca.issueActorCert(t, "spiffe://substrate-actor.local/worker-for-actor/foo/bar", actorCertOptions{}) - h := egressHandler(ca.roots(), runningActor(), nil) + h := egressHandler(ca.loadRoots, runningActor(), nil) certificate := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leaf.Raw}) if _, err := h.HandleRequestHeaders(context.Background(), agentgatewayEgressMetadata(string(certificate))); err != nil { @@ -701,7 +702,7 @@ func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - h := egressHandler(ca.roots(), runningActor(), nil) + h := egressHandler(ca.loadRoots, runningActor(), nil) _, err := h.HandleRequestHeaders(context.Background(), egressMetadata(tc.xfcc(t))) wantStatus(t, err, tc.want) }) @@ -745,7 +746,7 @@ func TestHandleRequestHeadersAuthorization(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - h := egressHandler(ca.roots(), tc.actor, tc.err) + h := egressHandler(ca.loadRoots, tc.actor, tc.err) leaf := ca.issueActorCert(t, "spiffe://substrate-actor.local/worker-for-actor/foo/bar", actorCertOptions{}) _, err := h.HandleRequestHeaders(context.Background(), egressMetadata(xfccHeader(leaf))) wantStatus(t, err, tc.want) @@ -753,15 +754,52 @@ func TestHandleRequestHeadersAuthorization(t *testing.T) { } } -// An ingress-only router has no actor-identity CA. If an egress CONNECT somehow -// reaches it, it must fail closed rather than tunnel unauthenticated traffic. -func TestHandleRequestHeadersWithoutConfiguredCA(t *testing.T) { +// Without actor-identity roots the gateway must fail closed rather than tunnel +// unauthenticated traffic. An ingress-only router has no CA configured, and a +// configured bundle can become unreadable after startup. +func TestHandleRequestHeadersWithoutUsableCA(t *testing.T) { ca := newTestCA(t, "actor-identity-ca") leaf := ca.issueActorCert(t, "spiffe://substrate-actor.local/worker-for-actor/foo/bar", actorCertOptions{}) - h := egressHandler(nil, runningActor(), nil) + + tests := []struct { + name string + loadRoots func() (*x509.CertPool, error) + }{ + { + name: "no CA configured", + }, + { + name: "CA fails to load", + loadRoots: func() (*x509.CertPool, error) { + return nil, errors.New("trust bundle contains no certificates") + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + h := egressHandler(tc.loadRoots, runningActor(), nil) + _, err := h.HandleRequestHeaders(context.Background(), egressMetadata(xfccHeader(leaf))) + wantStatus(t, err, envoy_type.StatusCode_ServiceUnavailable) + }) + } +} + +// A rotated actor-identity CA applies to the next CONNECT without a restart. +func TestHandleRequestHeadersPicksUpRotatedCA(t *testing.T) { + oldCA := newTestCA(t, "actor-identity-ca") + newCA := newTestCA(t, "actor-identity-ca-rotated") + leaf := newCA.issueActorCert(t, "spiffe://substrate-actor.local/worker-for-actor/foo/bar", actorCertOptions{}) + load := oldCA.loadRoots + h := egressHandler(func() (*x509.CertPool, error) { return load() }, runningActor(), nil) _, err := h.HandleRequestHeaders(context.Background(), egressMetadata(xfccHeader(leaf))) - wantStatus(t, err, envoy_type.StatusCode_ServiceUnavailable) + wantStatus(t, err, envoy_type.StatusCode_Forbidden) + + load = newCA.loadRoots + if _, err := h.HandleRequestHeaders(context.Background(), egressMetadata(xfccHeader(leaf))); err != nil { + t.Fatalf("HandleRequestHeaders() after rotation error = %v, want nil", err) + } } // atunnel always sends the address the actor's kernel dialed; a name here is @@ -769,7 +807,7 @@ func TestHandleRequestHeadersWithoutConfiguredCA(t *testing.T) { func TestHandleRequestHeadersRejectsNonAddressAuthority(t *testing.T) { ca := newTestCA(t, "actor-identity-ca") leaf := ca.issueActorCert(t, "spiffe://substrate-actor.local/worker-for-actor/foo/bar", actorCertOptions{}) - h := egressHandler(ca.roots(), runningActor(), nil) + h := egressHandler(ca.loadRoots, runningActor(), nil) for _, authority := range []string{"example.com:443", "93.184.216.34", ""} { md := egressMetadata(xfccHeader(leaf)) @@ -783,7 +821,7 @@ func TestHandleRequestHeadersRejectsNonAddressAuthority(t *testing.T) { func TestHandleRequestHeadersRejectsNonConnect(t *testing.T) { ca := newTestCA(t, "actor-identity-ca") leaf := ca.issueActorCert(t, "spiffe://substrate-actor.local/worker-for-actor/foo/bar", actorCertOptions{}) - h := egressHandler(ca.roots(), runningActor(), nil) + h := egressHandler(ca.loadRoots, runningActor(), nil) md := egressMetadata(xfccHeader(leaf)) md.Method = "GET" diff --git a/cmd/atenet/internal/router/egress/request_test.go b/cmd/atenet/internal/router/egress/request_test.go index 18df8950b1..548d3075ab 100644 --- a/cmd/atenet/internal/router/egress/request_test.go +++ b/cmd/atenet/internal/router/egress/request_test.go @@ -446,7 +446,7 @@ func TestConnectLegRequiresAPolicy(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - h := New(tc.client, ca.roots(), DefaultPolicyCacheTTL, nil, "") + h := New(tc.client, ca.loadRoots, DefaultPolicyCacheTTL, nil, "") res, err := h.HandleRequestHeaders(context.Background(), egressMetadata(xfccHeader(leaf))) if tc.want == 0 { wantAllowed(t, res, err) diff --git a/cmd/atenet/internal/router/router.go b/cmd/atenet/internal/router/router.go index 5c84e1964a..73b5b24dfd 100644 --- a/cmd/atenet/internal/router/router.go +++ b/cmd/atenet/internal/router/router.go @@ -22,7 +22,6 @@ import ( "log/slog" "net" "net/http" - "os" "os/signal" "strconv" "syscall" @@ -40,6 +39,7 @@ import ( "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" "go.ate.dev/substrate/cmd/atenet/internal/router/ingress" "go.ate.dev/substrate/internal/ateapiauth" + "go.ate.dev/substrate/internal/credbundle" "go.ate.dev/substrate/internal/preview" "go.ate.dev/substrate/internal/serverboot" "go.ate.dev/substrate/internal/version" @@ -206,17 +206,14 @@ func (s *RouterServer) Run(ctx context.Context) error { if s.cfg.Mode.ServesEgress() { // Load the actor-identity CA up front so a missing or unusable bundle // fails startup, rather than turning into a 503 on the first actor - // egress attempt. An unset flag leaves the handler with no roots, which - // denies every CONNECT — see egress.New. - var actorIdentityRoots *x509.CertPool + // egress attempt. The handler reloads it when the file changes. An + // unset flag leaves the handler with no roots, which denies every + // CONNECT — see egress.New. + var loadActorIdentityRoots func() (*x509.CertPool, error) if s.cfg.ActorIdentityCAFile != "" { - pemBytes, err := os.ReadFile(s.cfg.ActorIdentityCAFile) - if err != nil { - return fmt.Errorf("reading --actor-identity-ca-file: %w", err) - } - actorIdentityRoots, err = egress.LoadActorIdentityRoots(pemBytes) - if err != nil { - return fmt.Errorf("loading --actor-identity-ca-file %q: %w", s.cfg.ActorIdentityCAFile, err) + loadActorIdentityRoots = credbundle.PoolLoader(s.cfg.ActorIdentityCAFile) + if _, err := loadActorIdentityRoots(); err != nil { + return fmt.Errorf("loading --actor-identity-ca-file: %w", err) } } // Enable egress credential injection when a credential provider address @@ -247,7 +244,7 @@ func (s *RouterServer) Run(ctx context.Context) error { slog.String("provider_name", s.cfg.CredentialProvider.Name)) } - egressHandler := egress.New(s.apiClient, actorIdentityRoots, s.cfg.EgressPolicyCacheTTL, provider, providerName) + egressHandler := egress.New(s.apiClient, loadActorIdentityRoots, s.cfg.EgressPolicyCacheTTL, provider, providerName) handlers[egressHandler.Direction()] = egressHandler } diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index f95cc86897..994b2fae6f 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -98,12 +98,17 @@ data: resource_api_version: V3 path_config_source: path: /etc/envoy/sds-servicedns-cert.yaml - validation_context: - # Actors authenticate with an actor-identity client cert. - # Envoy enforces chain, signature, and validity period here, so - # an unsigned or expired actor cert never reaches ext_proc, - # which reads the actor from its SPIFFE URI SAN. - trusted_ca: { filename: /run/actor-id-ca-certs/ca.crt } + # Actors authenticate with an actor-identity client cert. + # Envoy enforces chain, signature, and validity period here, so + # an unsigned or expired actor cert never reaches ext_proc, + # which reads the actor from its SPIFFE URI SAN. Filesystem SDS + # for the same reason as the serving cert above. + validation_context_sds_secret_config: + name: actor_identity_validation_context + sds_config: + resource_api_version: V3 + path_config_source: + path: /etc/envoy/sds-actor-identity-validation.yaml filters: - name: envoy.filters.network.http_connection_manager typed_config: @@ -1127,6 +1132,15 @@ data: certificate_chain: { filename: /run/servicedns.podcert.ate.dev/credential-bundle.pem } private_key: { filename: /run/servicedns.podcert.ate.dev/credential-bundle.pem } watched_directory: { path: /run/servicedns.podcert.ate.dev } + # SDS resource for the actor-identity CA roots the egress listener trusts. + # The actor-id-ca-certs Secret volume rotates the same way. + sds-actor-identity-validation.yaml: | + resources: + - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret + name: actor_identity_validation_context + validation_context: + trusted_ca: { filename: /run/actor-id-ca-certs/ca.crt } + watched_directory: { path: /run/actor-id-ca-certs } # The next two keys are used only by the additional_egress_ext_proc cluster # spliced over the #ATE_MITM_EXTPROC_CLUSTER marker. They ship # unconditionally because the splice can only rewrite envoy.yaml, not add From d7c074b5812cdf7274a1adf073f3382ad0ee7883 Mon Sep 17 00:00:00 2001 From: Walter Fender Date: Thu, 22 Oct 2026 09:04:55 +0000 Subject: [PATCH 09/12] objectstorage: fail a ranged read cut short by cancellation (#3049) Why is this needed? When ctx was done, schedule closed ordered and Read returned io.EOF, even with bytes of the object still unread. A plain zstd stream cut at a frame boundary then decoded as a clean, truncated success. schedule now records the context's cause when it stops early, and Read returns it instead of io.EOF. Fixes #3047 AI use: Claude assisted. - [ ] Tests pass - [ ] Docs updated --- pkg/objectstorage/rangedget.go | 12 ++++++++++++ pkg/objectstorage/rangedget_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/pkg/objectstorage/rangedget.go b/pkg/objectstorage/rangedget.go index 42bde25446..1ff433f035 100644 --- a/pkg/objectstorage/rangedget.go +++ b/pkg/objectstorage/rangedget.go @@ -53,6 +53,9 @@ type rangedReader struct { cur []byte curFull []byte err error + // stopErr is why schedule stopped before issuing every range. It is set + // before ordered is closed, so Read may use it once ordered is closed. + stopErr error } type rangeResult struct { @@ -87,12 +90,14 @@ func (r *rangedReader) schedule(ctx context.Context, size int64, head io.Reader, select { case buf = <-r.free: case <-ctx.Done(): + r.stop(ctx, off, size) return } out := make(chan rangeResult, 1) select { case r.ordered <- out: case <-ctx.Done(): + r.stop(ctx, off, size) return } n := min(int64(downloadChunkSize), size-off) @@ -115,6 +120,10 @@ func (r *rangedReader) schedule(ctx context.Context, size int64, head io.Reader, } } +func (r *rangedReader) stop(ctx context.Context, off, size int64) { + r.stopErr = fmt.Errorf("ranged read stopped at offset %d of %d: %w", off, size, context.Cause(ctx)) +} + func (r *rangedReader) Read(p []byte) (int, error) { if r.err != nil { return 0, r.err @@ -123,6 +132,9 @@ func (r *rangedReader) Read(p []byte) (int, error) { out, ok := <-r.ordered if !ok { r.err = io.EOF + if r.stopErr != nil { + r.err = r.stopErr + } return 0, r.err } res := <-out diff --git a/pkg/objectstorage/rangedget_test.go b/pkg/objectstorage/rangedget_test.go index 82e2b30c21..49a7e58638 100644 --- a/pkg/objectstorage/rangedget_test.go +++ b/pkg/objectstorage/rangedget_test.go @@ -15,6 +15,7 @@ package objectstorage import ( + "bytes" "context" "errors" "io" @@ -60,3 +61,26 @@ func TestRangedReaderCloseStopsRequests(t *testing.T) { t.Fatal("Close left the first range reader open") } } + +// A read cut short by cancellation must fail, not end in a clean EOF that a +// decoder could take for the end of a frame. +func TestRangedReaderCancelIsNotEOF(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + fetched := make(chan struct{}, downloadConcurrency) + head := io.NopCloser(bytes.NewReader(make([]byte, downloadChunkSize))) + r := newRangedReader(ctx, (downloadConcurrency+1)*downloadChunkSize, head, func(context.Context, int, int64, int64, []byte) error { + fetched <- struct{}{} + return nil + }) + defer r.Close() + + // Every buffer holds a fetched range, so the last range waits for one. + for range downloadConcurrency - 1 { + <-fetched + } + cancel() + n, err := io.Copy(io.Discard, r) + if !errors.Is(err, context.Canceled) { + t.Errorf("read %d of %d bytes, err %v, want context.Canceled", n, r.ObjectSize(), err) + } +} From 6c94cf316e781747a1be14e1e2aa32f303ef7ee5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2026 09:54:39 +0000 Subject: [PATCH 10/12] build(deps): bump github.com/klauspost/compress from 1.18.6 to 1.18.7 in /hack/tools/ko (#2980) Bumps [github.com/klauspost/compress](https://github.com/klauspost/compress) from 1.18.6 to 1.18.7.
Release notes

Sourced from github.com/klauspost/compress's releases.

v1.18.7

  • Fix s2.NewDict OOB read.

Security release without other changes.

Full Changelog: https://github.com/klauspost/compress/compare/v1.18.6...v1.18.7

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/klauspost/compress&package-manager=go_modules&previous-version=1.18.6&new-version=1.18.7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/agent-substrate/substrate/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- hack/tools/ko/go.mod | 2 +- hack/tools/ko/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hack/tools/ko/go.mod b/hack/tools/ko/go.mod index 68d4123a63..6b8d492b05 100644 --- a/hack/tools/ko/go.mod +++ b/hack/tools/ko/go.mod @@ -97,7 +97,7 @@ require ( github.com/in-toto/in-toto-golang v0.11.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jedisct1/go-minisign v0.0.0-20241212093149-d2f9f49435c7 // indirect - github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/compress v1.18.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect diff --git a/hack/tools/ko/go.sum b/hack/tools/ko/go.sum index 4f80ab511d..0714065511 100644 --- a/hack/tools/ko/go.sum +++ b/hack/tools/ko/go.sum @@ -302,8 +302,8 @@ github.com/jedisct1/go-minisign v0.0.0-20241212093149-d2f9f49435c7 h1:FWpSWRD8Fb github.com/jedisct1/go-minisign v0.0.0-20241212093149-d2f9f49435c7/go.mod h1:BMxO138bOokdgt4UaxZiEfypcSHX0t6SIFimVP1oRfk= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= github.com/jellydator/ttlcache/v3 v3.4.0/go.mod h1:Hw9EgjymziQD3yGsQdf1FqFdpp7YjFMd4Srg5EJlgD4= -github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= -github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw= +github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= From 06737b8f3bdaba0fa332cd2d411fdd5d5c0a5566 Mon Sep 17 00:00:00 2001 From: Scott Funkenhauser <85249587+sfunkenhauser@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:07:25 +0000 Subject: [PATCH 11/12] Detect worker restarts by an epoch the worker process registers (#2988) Restart detection relied on the worker container's restartCount, which can restart from zero or stay the same. It was also recorded at bind time, so a resume whose Restore reached the restarted process was later crashed as if its sandbox had been lost. Each worker process now acquires an epoch from ateapi and activates it with RegisterWorker, once, along with its pod IPs and capacity. Sandbox runtimes move to SetWorkerSandboxRuntimes. Run and Restore return the worker's epoch, and finalizeRunning records it, or returns Unavailable. A worker refuses workloads with Unavailable until it registers, and atelet passes a worker's Unavailable on, so the resume is retried rather than the actor crashed. The reconciler crashes actors recorded below the active epoch, and leaves RESUMING actors and claims never committed to an actor alone. A worker reports ready only once registered. Worker.ips moves to status.ips and follows a recreated pod sandbox. Not handled yet: a worker that crashes before registering leaves its lost actors RUNNING. Fixes #2692 Fixes #1837 AI use: AI-assisted, I reviewed and tested it. - [x] Tests pass - [x] Docs updated --- benchmarking/README.md | 9 +- .../internal/apivalidation/actor_test.go | 11 +- cmd/ateapi/internal/apivalidation/worker.go | 30 +- .../internal/apivalidation/worker_test.go | 244 +++-- .../apivalidation/zz_generated.validation.go | 502 ++++++--- cmd/ateapi/internal/authz/registry.go | 2 + .../controlapi/functionaltest/actor_test.go | 6 +- .../controlapi/functionaltest/common_test.go | 1 - .../controlapi/functionaltest/main_test.go | 12 +- .../functionaltest/worker_contention_test.go | 18 +- .../functionaltest/worker_restart_test.go | 156 +++ .../controlapi/functionaltest/worker_test.go | 10 +- .../internal/controlapi/hosting_test.go | 12 +- .../controlapi/mint_after_release_test.go | 8 +- cmd/ateapi/internal/controlapi/worker.go | 12 +- .../worker_assignment_reconciler_test.go | 27 +- cmd/ateapi/internal/controlapi/worker_test.go | 10 +- .../workflow_reconcile_assignments.go | 73 +- .../workflow_reconcile_assignments_test.go | 254 ++--- .../internal/controlapi/workflow_resume.go | 72 +- .../controlapi/workflow_resume_test.go | 167 ++- .../controlapi/workflow_testutil_test.go | 22 + cmd/ateapi/internal/scheduling/scheduling.go | 16 +- .../internal/scheduling/scheduling_test.go | 33 +- .../internal/store/atepg/outbox_test.go | 7 +- .../internal/store/atepg/worker_assignment.go | 3 - .../store/atepg/worker_assignment_test.go | 2 +- cmd/ateapi/internal/workerservice/register.go | 176 +++- .../internal/workerservice/register_test.go | 556 +++++++--- .../internal/workerservice/service_test.go | 23 +- .../internal/controllers/workerpool_apply.go | 4 + .../controllers/workerpool_apply_test.go | 6 + .../internal/workersync/fakecontrol_test.go | 27 +- .../internal/workersync/syncer.go | 58 +- .../workersync/syncer_restart_test.go | 246 ----- .../internal/workersync/syncer_test.go | 116 ++- cmd/atelet/internal/apivalidation/validate.go | 5 + .../internal/apivalidation/validation_test.go | 89 +- .../apivalidation/zz_generated.validation.go | 170 ++- cmd/atelet/lifecycle_test.go | 27 +- cmd/atelet/main.go | 24 +- cmd/atelet/main_test.go | 22 + cmd/atelet/workersupport.go | 71 +- cmd/atelet/workersupport_test.go | 241 ++++- cmd/ateworker-gvisor/main.go | 23 +- cmd/ateworker-gvisor/main_test.go | 22 + cmd/ateworker-microvm/main.go | 11 +- cmd/ateworker-microvm/main_test.go | 21 + cmd/ateworker-microvm/restore.go | 6 +- cmd/ateworker-microvm/restore_test.go | 1 + cmd/ateworker-microvm/run.go | 6 +- .../isolate/fake-atelet/epochs.go | 58 ++ .../isolate/fake-atelet/herder.go | 7 +- .../isolate/fake-atelet/herder_test.go | 32 + cmd/benchmarking/isolate/fake-atelet/main.go | 5 +- cmd/benchmarking/isolate/fake-atelet/relay.go | 25 + .../isolate/fake-atelet/relay_test.go | 120 ++- .../isolate/fake-workersync/controller.go | 42 +- .../fake-workersync/controller_test.go | 96 +- .../isolate/fake-workersync/relayclient.go | 29 +- .../fake-workersync/relayclient_test.go | 96 +- docs/authorization.md | 2 + internal/ateworker/register/register.go | 289 +++-- internal/ateworker/register/register_test.go | 240 ++++- internal/proto/ateletpb/atelet.pb.go | 933 ++++++++++------- internal/proto/ateletpb/atelet.proto | 53 +- internal/proto/ateletpb/atelet_grpc.pb.go | 106 +- internal/proto/ateworkerpb/ateworker.pb.go | 32 +- internal/proto/ateworkerpb/ateworker.proto | 4 + pkg/proto/ateapipb/ateapi.pb.go | 984 +++++++++++------- pkg/proto/ateapipb/ateapi.proto | 208 ++-- pkg/proto/ateapipb/ateapi_grpc.pb.go | 148 ++- 72 files changed, 4970 insertions(+), 2209 deletions(-) create mode 100644 cmd/ateapi/internal/controlapi/functionaltest/worker_restart_test.go delete mode 100644 cmd/atecontroller/internal/workersync/syncer_restart_test.go create mode 100644 cmd/benchmarking/isolate/fake-atelet/epochs.go diff --git a/benchmarking/README.md b/benchmarking/README.md index 14be6d41c3..5f17b2466b 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -658,7 +658,14 @@ fakes from `cmd/benchmarking/isolate/` instead of worker pods: time, so it adds to `--delay`, the data plane's share of the call. It also relays fake-workersync's capacity reports, as atelet relays ateworker's: ate-api-server accepts a report only from the atelet on the Worker's node. - The relay admits only fake-workersync's pod identity. + The relay admits only fake-workersync's pod identity. fake-atelet remembers + the epoch each Worker registers with and reports it from every run and + restore, as a worker process does; ate-api-server commits a resumed actor + only under its Worker's current epoch. It keeps the epochs in memory only, + and fake-workersync registers a Worker once, so a fake-atelet that restarts + after its Workers registered, in a DaemonSet rollout for example, fails + every resume on those Workers with Unavailable until they are replaced, + such as by redeploying the benchmark. The install also scales ate-controller to zero, whose worker syncer would delete the fake Workers. Its other controllers stop with it: no NetworkPolicy diff --git a/cmd/ateapi/internal/apivalidation/actor_test.go b/cmd/ateapi/internal/apivalidation/actor_test.go index 652d4e8fb6..e52c315db6 100644 --- a/cmd/ateapi/internal/apivalidation/actor_test.go +++ b/cmd/ateapi/internal/apivalidation/actor_test.go @@ -395,10 +395,17 @@ func TestValidateActorUpdate(t *testing.T) { oldVal: validInput(withStatus(withWorkerAssignment())), newVal: validOutput(withStatus(func(s *ateapipb.ActorStatus) { s.WorkerAssignment = nil })), }, { - name: "modify actor.status.worker_assignment", + name: "modify actor.status.worker_assignment identity", oldVal: validInput(withStatus(withWorkerAssignment())), newVal: validOutput(withStatus(withWorkerAssignment(func(wa *ateapipb.WorkerAssignment) { wa.WorkerPod = "pod2" }))), - want: field.ErrorList{field.Invalid(field.NewPath("status", "worker_assignment"), nil, "").WithOrigin("update")}, + want: field.ErrorList{field.Invalid(field.NewPath("status", "worker_assignment", "worker_pod"), nil, "").WithOrigin("update")}, + }, { + name: "modify actor.status.worker_assignment worker process", + oldVal: validInput(withStatus(withWorkerAssignment())), + newVal: validOutput(withStatus(withWorkerAssignment(func(wa *ateapipb.WorkerAssignment) { + wa.WorkerEpoch = 2 + wa.WorkerPodIps = []string{"10.9.8.7"} + }))), }, { name: "empty actor.status.worker_assignment", oldVal: validInput(), diff --git a/cmd/ateapi/internal/apivalidation/worker.go b/cmd/ateapi/internal/apivalidation/worker.go index 4d3ce0b5fa..00e8567d4d 100644 --- a/cmd/ateapi/internal/apivalidation/worker.go +++ b/cmd/ateapi/internal/apivalidation/worker.go @@ -16,7 +16,6 @@ package apivalidation import ( "context" - "fmt" "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/pkg/proto/ateapipb" @@ -72,11 +71,21 @@ func ValidateDrainWorkerRequest(ctx context.Context, req *ateapipb.DrainWorkerRe return Validate_DrainWorkerRequest(ctx, op, nil, req, nil) } +func ValidateAcquireWorkerEpochRequest(ctx context.Context, req *ateapipb.AcquireWorkerEpochRequest) field.ErrorList { + op := MakeCreateOp(ctx) + return Validate_AcquireWorkerEpochRequest(ctx, op, nil, req, nil) +} + func ValidateRegisterWorkerRequest(ctx context.Context, req *ateapipb.RegisterWorkerRequest) field.ErrorList { op := MakeCreateOp(ctx) return Validate_RegisterWorkerRequest(ctx, op, nil, req, nil) } +func ValidateSetWorkerSandboxRuntimesRequest(ctx context.Context, req *ateapipb.SetWorkerSandboxRuntimesRequest) field.ErrorList { + op := MakeCreateOp(ctx) + return Validate_SetWorkerSandboxRuntimesRequest(ctx, op, nil, req, nil) +} + func ValidateRequestActorSuspendRequest(ctx context.Context, req *ateapipb.RequestActorSuspendRequest) field.ErrorList { op := MakeCreateOp(ctx) return Validate_RequestActorSuspendRequest(ctx, op, nil, req, nil) @@ -135,7 +144,12 @@ func ValidateCustom_WorkerSandboxClass_SandboxClass(_ context.Context, _ operati } // This is needed because DV doesn't have a standard format for IP addresses yet. -func ValidateCustom_Worker_Ips(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ []string) field.ErrorList { +func ValidateCustom_WorkerStatus_Ips(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ []string) field.ErrorList { + return validateDualStackIPs(fldPath, value) +} + +// This is needed because DV doesn't have a standard format for IP addresses yet. +func ValidateCustom_RegisterWorkerRequest_Ips(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ []string) field.ErrorList { return validateDualStackIPs(fldPath, value) } @@ -144,18 +158,6 @@ func ValidateCustom_Worker_WorkerSpiffeId(_ context.Context, _ operation.Operati return validateSPIFFEID(fldPath, *value) } -// epoch only moves forward: a lower value would make Actors placed during the -// current run look older than it, and crash them. -func ValidateCustom_Worker_Epoch(_ context.Context, op operation.Operation, fldPath *field.Path, value, oldValue *int64) field.ErrorList { - if op.Type != operation.Update || value == nil || oldValue == nil { - return nil - } - if *value < *oldValue { - return field.ErrorList{field.Invalid(fldPath, *value, fmt.Sprintf("may not decrease from %d", *oldValue))} - } - return nil -} - // This exists only because nested subfield tags are not supported yet. func ValidateCustom_UpdateWorkerRequest_Worker(ctx context.Context, op operation.Operation, fldPath *field.Path, worker, _ *ateapipb.Worker) field.ErrorList { if worker == nil || worker.Metadata == nil { diff --git a/cmd/ateapi/internal/apivalidation/worker_test.go b/cmd/ateapi/internal/apivalidation/worker_test.go index 05948affc6..c32dd82269 100644 --- a/cmd/ateapi/internal/apivalidation/worker_test.go +++ b/cmd/ateapi/internal/apivalidation/worker_test.go @@ -122,26 +122,32 @@ func TestValidateCreateWorkerRequest(t *testing.T) { req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { w.NodeName = "NODE_NAME" })), want: field.ErrorList{field.Invalid(field.NewPath("worker", "node_name"), nil, "").WithOrigin("format=k8s-long-name")}, }, { - name: "valid dual-stack ips", - req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { w.Ips = []string{"fd00::1", "10.1.2.3"} })), - }, { - name: "missing ips", - req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { w.Ips = nil })), - want: field.ErrorList{field.Required(field.NewPath("worker", "ips"), "")}, - }, { - name: "invalid ip", - req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { w.Ips = []string{"not-an-ip"} })), - want: field.ErrorList{field.Invalid(field.NewPath("worker", "ips").Index(0), nil, "").WithOrigin("format=ip-strict")}, - }, { - name: "two ips in one family", - req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { w.Ips = []string{"10.1.2.3", "10.1.2.4"} })), - want: field.ErrorList{field.Invalid(field.NewPath("worker", "ips").Index(1), nil, "")}, - }, { - name: "too many ips", - req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { - w.Ips = []string{"10.1.2.3", "fd00::1", "10.1.2.4"} - })), - want: field.ErrorList{field.TooMany(field.NewPath("worker", "ips"), 3, 2).WithOrigin("maxItems")}, + name: "valid dual-stack status.ips", + req: validReq(validWorker(apiWorkerName, withStatus(func(s *ateapipb.WorkerStatus) { s.Ips = []string{"fd00::1", "10.1.2.3"} }))), + }, { + name: "invalid status.ips", + req: validReq(validWorker(apiWorkerName, withStatus(func(s *ateapipb.WorkerStatus) { s.Ips = []string{"not-an-ip"} }))), + want: field.ErrorList{field.Invalid(field.NewPath("worker", "status", "ips").Index(0), nil, "").WithOrigin("format=ip-strict")}, + }, { + name: "two status.ips in one family", + req: validReq(validWorker(apiWorkerName, withStatus(func(s *ateapipb.WorkerStatus) { s.Ips = []string{"10.1.2.3", "10.1.2.4"} }))), + want: field.ErrorList{field.Invalid(field.NewPath("worker", "status", "ips").Index(1), nil, "")}, + }, { + name: "too many status.ips", + req: validReq(validWorker(apiWorkerName, withStatus(func(s *ateapipb.WorkerStatus) { + s.Ips = []string{"10.1.2.3", "fd00::1", "10.1.2.4"} + }))), + want: field.ErrorList{field.TooMany(field.NewPath("worker", "status", "ips"), 3, 2).WithOrigin("maxItems")}, + }, { + name: "negative status epochs", + req: validReq(validWorker(apiWorkerName, withStatus(func(s *ateapipb.WorkerStatus) { + s.IssuedEpoch = -1 + s.RegisteredEpoch = -1 + }))), + want: field.ErrorList{ + field.Invalid(field.NewPath("worker", "status", "issued_epoch"), nil, "").WithOrigin("minimum"), + field.Invalid(field.NewPath("worker", "status", "registered_epoch"), nil, "").WithOrigin("minimum"), + }, }, { name: "valid worker_spiffe_id", req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { @@ -337,26 +343,50 @@ func TestValidateWorkerUpdate_RequireStatus(t *testing.T) { assertValidateErr(t, ValidateWorkerUpdate(context.Background(), field.NewPath("worker"), newVal, oldVal, false), nil) } +func TestValidateAcquireWorkerEpochRequest(t *testing.T) { + tests := []struct { + name string + req *ateapipb.AcquireWorkerEpochRequest + want field.ErrorList + }{{ + name: "valid", + req: &ateapipb.AcquireWorkerEpochRequest{Worker: workerRef(apiWorkerName)}, + }, { + name: "missing worker", + req: &ateapipb.AcquireWorkerEpochRequest{}, + want: field.ErrorList{field.Required(field.NewPath("worker"), "")}, + }, { + name: "worker.atespace must be empty", + req: &ateapipb.AcquireWorkerEpochRequest{Worker: &ateapipb.ObjectRef{Atespace: "team-a", Name: apiWorkerName}}, + want: field.ErrorList{field.Forbidden(field.NewPath("worker", "atespace"), "")}, + }, { + name: "missing worker.name", + req: &ateapipb.AcquireWorkerEpochRequest{Worker: &ateapipb.ObjectRef{}}, + want: field.ErrorList{field.Required(field.NewPath("worker", "name"), "")}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assertValidateErr(t, ValidateAcquireWorkerEpochRequest(context.Background(), tt.req), tt.want) + }) + } +} + func TestValidateRegisterWorkerRequest(t *testing.T) { valid := func(mutate ...func(*ateapipb.RegisterWorkerRequest)) *ateapipb.RegisterWorkerRequest { r := &ateapipb.RegisterWorkerRequest{ Worker: workerRef(apiWorkerName), + Epoch: 1, + Ips: []string{"10.1.2.3"}, Capacity: &ateapipb.WorkerResources{ Actors: 10, CpuMillicores: 2000, MemoryBytes: 4 << 30, }, - SandboxClasses: []*ateapipb.WorkerSandboxClass{validSandboxClass()}, } for _, m := range mutate { m(r) } return r } - classesPath := field.NewPath("sandbox_classes") - classPath := classesPath.Index(0) - defaultRuntimePath := classPath.Child("default_runtime") - defaultVersionPath := defaultRuntimePath.Child("version") - restorablePath := classPath.Child("restorable_runtimes") tests := []struct { name string req *ateapipb.RegisterWorkerRequest @@ -364,19 +394,12 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { }{{ name: "valid", req: valid(), - }, { - name: "valid with restorable runtimes", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { - r.SandboxClasses[0].RestorableRuntimes = []*ateapipb.WorkerSandboxRuntime{{Version: &ateapipb.VersionedSandboxCompat{ - SchemaVersion: "v1", - Attributes: []*ateapipb.AttributeEntry{ - {Key: "architecture", Value: "amd64"}, - }, - }}} - }), }, { name: "valid empty capacity", req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.Capacity = &ateapipb.WorkerResources{} }), + }, { + name: "valid dual-stack ips", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.Ips = []string{"fd00::1", "10.1.2.3"} }), }, { name: "missing worker", req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.Worker = nil }), @@ -393,6 +416,30 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { name: "invalid worker.name", req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.Worker.Name = "UPPER" }), want: field.ErrorList{field.Invalid(field.NewPath("worker", "name"), nil, "").WithOrigin("format=k8s-short-name")}, + }, { + name: "missing epoch", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.Epoch = 0 }), + want: field.ErrorList{field.Required(field.NewPath("epoch"), "")}, + }, { + name: "negative epoch", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.Epoch = -1 }), + want: field.ErrorList{field.Invalid(field.NewPath("epoch"), nil, "").WithOrigin("minimum")}, + }, { + name: "missing ips", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.Ips = nil }), + want: field.ErrorList{field.Required(field.NewPath("ips"), "")}, + }, { + name: "invalid ip", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.Ips = []string{"not-an-ip"} }), + want: field.ErrorList{field.Invalid(field.NewPath("ips").Index(0), nil, "").WithOrigin("format=ip-strict")}, + }, { + name: "two ips in one family", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.Ips = []string{"10.1.2.3", "10.1.2.4"} }), + want: field.ErrorList{field.Invalid(field.NewPath("ips").Index(1), nil, "")}, + }, { + name: "too many ips", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.Ips = []string{"10.1.2.3", "fd00::1", "10.1.2.4"} }), + want: field.ErrorList{field.TooMany(field.NewPath("ips"), 3, 2).WithOrigin("maxItems")}, }, { name: "missing capacity", req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.Capacity = nil }), @@ -405,17 +452,75 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { name: "negative capacity.memory_bytes", req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.Capacity.MemoryBytes = -1 }), want: field.ErrorList{field.Invalid(field.NewPath("capacity", "memory_bytes"), nil, "").WithOrigin("minimum")}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assertValidateErr(t, ValidateRegisterWorkerRequest(context.Background(), tt.req), tt.want) + }) + } +} + +func TestValidateSetWorkerSandboxRuntimesRequest(t *testing.T) { + valid := func(mutate ...func(*ateapipb.SetWorkerSandboxRuntimesRequest)) *ateapipb.SetWorkerSandboxRuntimesRequest { + r := &ateapipb.SetWorkerSandboxRuntimesRequest{ + Worker: workerRef(apiWorkerName), + Epoch: 1, + SandboxClasses: []*ateapipb.WorkerSandboxClass{validSandboxClass()}, + } + for _, m := range mutate { + m(r) + } + return r + } + classesPath := field.NewPath("sandbox_classes") + classPath := classesPath.Index(0) + defaultRuntimePath := classPath.Child("default_runtime") + defaultVersionPath := defaultRuntimePath.Child("version") + restorablePath := classPath.Child("restorable_runtimes") + tests := []struct { + name string + req *ateapipb.SetWorkerSandboxRuntimesRequest + want field.ErrorList + }{{ + name: "valid", + req: valid(), + }, { + name: "valid with restorable runtimes", + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { + r.SandboxClasses[0].RestorableRuntimes = []*ateapipb.WorkerSandboxRuntime{{Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + }, + }}} + }), + }, { + name: "missing worker", + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.Worker = nil }), + want: field.ErrorList{field.Required(field.NewPath("worker"), "")}, + }, { + name: "worker.atespace must be empty", + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.Worker.Atespace = "team-a" }), + want: field.ErrorList{field.Forbidden(field.NewPath("worker", "atespace"), "")}, + }, { + name: "missing epoch", + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.Epoch = 0 }), + want: field.ErrorList{field.Required(field.NewPath("epoch"), "")}, + }, { + name: "negative epoch", + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.Epoch = -1 }), + want: field.ErrorList{field.Invalid(field.NewPath("epoch"), nil, "").WithOrigin("minimum")}, }, { name: "missing sandbox_classes", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses = nil }), + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.SandboxClasses = nil }), want: field.ErrorList{field.Required(classesPath, "")}, }, { name: "empty sandbox_classes", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses = []*ateapipb.WorkerSandboxClass{} }), + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.SandboxClasses = []*ateapipb.WorkerSandboxClass{} }), want: field.ErrorList{field.Required(classesPath, "")}, }, { name: "too many sandbox_classes", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { second := validSandboxClass() second.SandboxClass = "microvm" r.SandboxClasses = append(r.SandboxClasses, second) @@ -423,44 +528,50 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { want: field.ErrorList{field.TooMany(classesPath, 2, 1).WithOrigin("maxItems")}, }, { name: "missing sandbox_classes.sandbox_class", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses[0].SandboxClass = "" }), + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.SandboxClasses[0].SandboxClass = "" }), want: field.ErrorList{field.Required(classPath.Child("sandbox_class"), "")}, }, { name: "sandbox_classes.sandbox_class too long", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses[0].SandboxClass = strings.Repeat("c", 64) }), + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { + r.SandboxClasses[0].SandboxClass = strings.Repeat("c", 64) + }), want: field.ErrorList{ field.TooLong(classPath.Child("sandbox_class"), "", 63).WithOrigin("maxLength"), field.NotSupported[string](classPath.Child("sandbox_class"), nil, nil), }, }, { name: "unsupported sandbox_classes.sandbox_class", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses[0].SandboxClass = "kata" }), + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.SandboxClasses[0].SandboxClass = "kata" }), want: field.ErrorList{field.NotSupported[string](classPath.Child("sandbox_class"), nil, nil)}, }, { name: "missing sandbox_classes.default_runtime", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses[0].DefaultRuntime = nil }), + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.SandboxClasses[0].DefaultRuntime = nil }), want: field.ErrorList{field.Required(defaultRuntimePath, "")}, }, { name: "missing default_runtime.version", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses[0].DefaultRuntime.Version = nil }), + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.SandboxClasses[0].DefaultRuntime.Version = nil }), want: field.ErrorList{field.Required(defaultVersionPath, "")}, }, { name: "missing default_runtime.schema_version", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses[0].DefaultRuntime.Version.SchemaVersion = "" }), + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { + r.SandboxClasses[0].DefaultRuntime.Version.SchemaVersion = "" + }), want: field.ErrorList{field.Required(defaultVersionPath.Child("schema_version"), "")}, }, { name: "default_runtime.schema_version too long", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.SandboxClasses[0].DefaultRuntime.Version.SchemaVersion = strings.Repeat("v", 65) }), want: field.ErrorList{field.TooLong(defaultVersionPath.Child("schema_version"), "", 64).WithOrigin("maxLength")}, }, { name: "missing default_runtime.attributes", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses[0].DefaultRuntime.Version.Attributes = nil }), + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { + r.SandboxClasses[0].DefaultRuntime.Version.Attributes = nil + }), want: field.ErrorList{field.Required(defaultVersionPath.Child("attributes"), "")}, }, { name: "duplicate attribute key", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.SandboxClasses[0].DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{ {Key: "architecture", Value: "amd64"}, {Key: "architecture", Value: "arm64"}, @@ -469,77 +580,81 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { want: field.ErrorList{field.Duplicate(defaultVersionPath.Child("attributes").Index(1), nil)}, }, { name: "missing attribute key", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.SandboxClasses[0].DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Value: "amd64"}} }), want: field.ErrorList{field.Required(defaultVersionPath.Child("attributes").Index(0).Child("key"), "")}, }, { name: "attribute key too long", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.SandboxClasses[0].DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Key: strings.Repeat("k", 129), Value: "v"}} }), want: field.ErrorList{field.TooLong(defaultVersionPath.Child("attributes").Index(0).Child("key"), "", 128).WithOrigin("maxLength")}, }, { name: "missing attribute value", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.SandboxClasses[0].DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Key: "k"}} }), want: field.ErrorList{field.Required(defaultVersionPath.Child("attributes").Index(0).Child("value"), "")}, }, { name: "attribute value too long", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.SandboxClasses[0].DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Key: "k", Value: strings.Repeat("v", 257)}} }), want: field.ErrorList{field.TooLong(defaultVersionPath.Child("attributes").Index(0).Child("value"), "", 256).WithOrigin("maxLength")}, }, { name: "attributes at the bound", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.SandboxClasses[0].DefaultRuntime.Version.Attributes = attributeEntries(32) }), }, { name: "too many attributes", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.SandboxClasses[0].DefaultRuntime.Version.Attributes = attributeEntries(33) }), want: field.ErrorList{field.TooMany(defaultVersionPath.Child("attributes"), 33, 32).WithOrigin("maxItems")}, }, { name: "nil restorable_runtimes entry", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.SandboxClasses[0].RestorableRuntimes = []*ateapipb.WorkerSandboxRuntime{nil} }), want: field.ErrorList{field.Required(restorablePath.Index(0), "")}, }, { name: "invalid restorable_runtimes entry", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.SandboxClasses[0].RestorableRuntimes = []*ateapipb.WorkerSandboxRuntime{{Version: &ateapipb.VersionedSandboxCompat{Attributes: attributeEntries(1)}}} }), want: field.ErrorList{field.Required(restorablePath.Index(0).Child("version", "schema_version"), "")}, }, { name: "restorable_runtimes entry without a version", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.SandboxClasses[0].RestorableRuntimes = []*ateapipb.WorkerSandboxRuntime{{}} }), want: field.ErrorList{field.Required(restorablePath.Index(0).Child("version"), "")}, }, { name: "restorable_runtimes at the bound", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses[0].RestorableRuntimes = sandboxRuntimes(32) }), + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { + r.SandboxClasses[0].RestorableRuntimes = sandboxRuntimes(32) + }), }, { name: "too many restorable_runtimes", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.SandboxClasses[0].RestorableRuntimes = sandboxRuntimes(33) }), + req: valid(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { + r.SandboxClasses[0].RestorableRuntimes = sandboxRuntimes(33) + }), want: field.ErrorList{field.TooMany(restorablePath, 33, 32).WithOrigin("maxItems")}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assertValidateErr(t, ValidateRegisterWorkerRequest(context.Background(), tt.req), tt.want) + assertValidateErr(t, ValidateSetWorkerSandboxRuntimesRequest(context.Background(), tt.req), tt.want) }) } } // TestValidateWorkerUpdate_SandboxClasses pins the status-side checks on the -// sandbox classes RegisterWorker records. They reach the store as status, -// which no request validator sees, so ValidateWorkerUpdate is the only check -// on them. The old worker already reports a class so that every mutation, -// clearing the list included, differs from it and is validated. +// sandbox classes SetWorkerSandboxRuntimes records. They reach the store as +// status, which no request validator sees, so ValidateWorkerUpdate is the only +// check on them. The old worker already reports a class so that every +// mutation, clearing the list included, differs from it and is validated. func TestValidateWorkerUpdate_SandboxClasses(t *testing.T) { classesPath := field.NewPath("worker", "status", "sandbox_classes") classPath := classesPath.Index(0) @@ -816,7 +931,6 @@ func validWorker(name string, mods ...func(*ateapipb.Worker)) *ateapipb.Worker { WorkerPod: "worker-pod-1", WorkerPodUid: name, NodeName: "node-1", - Ips: []string{"10.1.2.3"}, } for _, m := range mods { m(w) diff --git a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go index 3bd48c9c84..bb5572845b 100644 --- a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go +++ b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go @@ -114,6 +114,65 @@ func Validate_AccessPolicy( return errs } +// Validate_AcquireWorkerEpochRequest validates an instance of AcquireWorkerEpochRequest according +// to declarative validation rules in the API schema. +func Validate_AcquireWorkerEpochRequest( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.AcquireWorkerEpochRequest) (errs field.ErrorList) { + + { // field ateapipb.AcquireWorkerEpochRequest.Worker + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.ObjectRef, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + func() { // cohort = "atespace" + earlyReturn := false + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.ForbiddenValue).MarkBeta().MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.OptionalValue).MarkBeta().MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.OptionalValue).MarkBeta().MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + }() + // call the type's validation function + errs = append(errs, Validate_ObjectRef(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.AcquireWorkerEpochRequest) *ateapipb.ObjectRef { + return oldObj.Worker + }) + errs = append(errs, fn(fldPath.Child("worker"), obj.Worker, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_Actor validates an instance of Actor according // to declarative validation rules in the API schema. func Validate_Actor( @@ -891,10 +950,6 @@ func Validate_ActorStatus( if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { earlyReturn = true } - if e := validate.UpdatePointer(ctx, op, fldPath, obj, oldObj, validate.NoModify).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } if earlyReturn { return // do not proceed } @@ -6622,41 +6677,42 @@ func Validate_RegisterWorkerRequest( errs = append(errs, fn(fldPath.Child("worker"), obj.Worker, oldVal, oldObj != nil)...) } - { // field ateapipb.RegisterWorkerRequest.Capacity + { // field ateapipb.RegisterWorkerRequest.Epoch fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.WorkerResources, + obj, oldObj *int64, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { return nil } } // call field-attached validations earlyReturn := false - if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } if earlyReturn { return // do not proceed } - // call the type's validation function - errs = append(errs, Validate_WorkerResources(ctx, op, fldPath, obj, oldObj)...) + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.RegisterWorkerRequest) *ateapipb.WorkerResources { - return oldObj.Capacity + func(oldObj *ateapipb.RegisterWorkerRequest) *int64 { + return &oldObj.Epoch }) - errs = append(errs, fn(fldPath.Child("capacity"), obj.Capacity, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("epoch"), &obj.Epoch, oldVal, oldObj != nil)...) } - { // field ateapipb.RegisterWorkerRequest.SandboxClasses + { // field ateapipb.RegisterWorkerRequest.Ips fn := func( fldPath *field.Path, - obj, oldObj []*ateapipb.WorkerSandboxClass, + obj, oldObj []string, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -6666,11 +6722,7 @@ func Validate_RegisterWorkerRequest( } // call field-attached validations earlyReturn := false - if e := validate.PtrSliceNoNils[ateapipb.WorkerSandboxClass](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 1).MarkShortCircuit(); len(e) != 0 { + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 2).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } @@ -6681,30 +6733,48 @@ func Validate_RegisterWorkerRequest( if earlyReturn { return // do not proceed } - if e := validate.MinItems(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + // custom validation + if e := ValidateCustom_RegisterWorkerRequest_Ips(ctx, op, fldPath, obj, oldObj); len(e) != 0 { errs = append(errs, e...) } - // lists with map semantics require unique keys - if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, - func(a *ateapipb.WorkerSandboxClass, b *ateapipb.WorkerSandboxClass) bool { - return a.SandboxClass == b.SandboxClass - }); len(e) != 0 { - errs = append(errs, e...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.RegisterWorkerRequest) []string { + return oldObj.Ips + }) + errs = append(errs, fn(fldPath.Child("ips"), obj.Ips, oldVal, oldObj != nil)...) + } + + { // field ateapipb.RegisterWorkerRequest.Capacity + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.WorkerResources, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } } - // iterate the list and call the type's validation function - if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, - func(a *ateapipb.WorkerSandboxClass, b *ateapipb.WorkerSandboxClass) bool { - return a.SandboxClass == b.SandboxClass - }, ateDeepEqual, Validate_WorkerSandboxClass); len(e) != 0 { + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed } + // call the type's validation function + errs = append(errs, Validate_WorkerResources(ctx, op, fldPath, obj, oldObj)...) return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.RegisterWorkerRequest) []*ateapipb.WorkerSandboxClass { - return oldObj.SandboxClasses + func(oldObj *ateapipb.RegisterWorkerRequest) *ateapipb.WorkerResources { + return oldObj.Capacity }) - errs = append(errs, fn(fldPath.Child("sandbox_classes"), obj.SandboxClasses, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("capacity"), obj.Capacity, oldVal, oldObj != nil)...) } return errs @@ -7713,6 +7783,151 @@ func Validate_Selector( return errs } +// Validate_SetWorkerSandboxRuntimesRequest validates an instance of SetWorkerSandboxRuntimesRequest according +// to declarative validation rules in the API schema. +func Validate_SetWorkerSandboxRuntimesRequest( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.SetWorkerSandboxRuntimesRequest) (errs field.ErrorList) { + + { // field ateapipb.SetWorkerSandboxRuntimesRequest.Worker + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.ObjectRef, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + func() { // cohort = "atespace" + earlyReturn := false + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.ForbiddenValue).MarkBeta().MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.OptionalValue).MarkBeta().MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.OptionalValue).MarkBeta().MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + }() + // call the type's validation function + errs = append(errs, Validate_ObjectRef(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SetWorkerSandboxRuntimesRequest) *ateapipb.ObjectRef { + return oldObj.Worker + }) + errs = append(errs, fn(fldPath.Child("worker"), obj.Worker, oldVal, oldObj != nil)...) + } + + { // field ateapipb.SetWorkerSandboxRuntimesRequest.Epoch + fn := func( + fldPath *field.Path, + obj, oldObj *int64, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SetWorkerSandboxRuntimesRequest) *int64 { + return &oldObj.Epoch + }) + errs = append(errs, fn(fldPath.Child("epoch"), &obj.Epoch, oldVal, oldObj != nil)...) + } + + { // field ateapipb.SetWorkerSandboxRuntimesRequest.SandboxClasses + fn := func( + fldPath *field.Path, + obj, oldObj []*ateapipb.WorkerSandboxClass, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateapipb.WorkerSandboxClass](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 1).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.RequiredSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MinItems(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + // lists with map semantics require unique keys + if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, + func(a *ateapipb.WorkerSandboxClass, b *ateapipb.WorkerSandboxClass) bool { + return a.SandboxClass == b.SandboxClass + }); len(e) != 0 { + errs = append(errs, e...) + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, + func(a *ateapipb.WorkerSandboxClass, b *ateapipb.WorkerSandboxClass) bool { + return a.SandboxClass == b.SandboxClass + }, ateDeepEqual, Validate_WorkerSandboxClass); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SetWorkerSandboxRuntimesRequest) []*ateapipb.WorkerSandboxClass { + return oldObj.SandboxClasses + }) + errs = append(errs, fn(fldPath.Child("sandbox_classes"), obj.SandboxClasses, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_Snapshot validates an instance of Snapshot according // to declarative validation rules in the API schema. func Validate_Snapshot( @@ -10122,47 +10337,6 @@ func Validate_Worker( errs = append(errs, fn(fldPath.Child("node_name"), &obj.NodeName, oldVal, oldObj != nil)...) } - { // field ateapipb.Worker.Ips - fn := func( - fldPath *field.Path, - obj, oldObj []string, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.Immutable(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 2).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if e := validate.RequiredSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - // custom validation - if e := ValidateCustom_Worker_Ips(ctx, op, fldPath, obj, oldObj); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.Worker) []string { - return oldObj.Ips - }) - errs = append(errs, fn(fldPath.Child("ips"), obj.Ips, oldVal, oldObj != nil)...) - } - { // field ateapipb.Worker.WorkerSpiffeId fn := func( fldPath *field.Path, @@ -10240,46 +10414,6 @@ func Validate_Worker( errs = append(errs, fn(fldPath.Child("labels"), obj.Labels, oldVal, oldObj != nil)...) } - { // field ateapipb.Worker.Epoch - fn := func( - fldPath *field.Path, - obj, oldObj *int64, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } - if e := validate.UpdateValue(ctx, op, fldPath, obj, oldObj, - func(a int64, b int64) bool { return a == b }, validate.NoUnset).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - // custom validation - if e := ValidateCustom_Worker_Epoch(ctx, op, fldPath, obj, oldObj); len(e) != 0 { - errs = append(errs, e...) - } - if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 0); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.Worker) *int64 { - return &oldObj.Epoch - }) - errs = append(errs, fn(fldPath.Child("epoch"), &obj.Epoch, oldVal, oldObj != nil)...) - } - { // field ateapipb.Worker.Status fn := func( fldPath *field.Path, @@ -10336,6 +10470,10 @@ func Validate_WorkerAssignment( errs = append(errs, e...) earlyReturn = true } + if e := validate.UpdatePointer(ctx, op, fldPath, obj, oldObj, validate.NoModify).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } if earlyReturn { return // do not proceed } @@ -10386,6 +10524,11 @@ func Validate_WorkerAssignment( errs = append(errs, e...) earlyReturn = true } + if e := validate.UpdateValue(ctx, op, fldPath, obj, oldObj, + func(a string, b string) bool { return a == b }, validate.NoModify).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } if earlyReturn { return // do not proceed } @@ -10418,6 +10561,11 @@ func Validate_WorkerAssignment( errs = append(errs, e...) earlyReturn = true } + if e := validate.UpdateValue(ctx, op, fldPath, obj, oldObj, + func(a string, b string) bool { return a == b }, validate.NoModify).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } if earlyReturn { return // do not proceed } @@ -10450,6 +10598,11 @@ func Validate_WorkerAssignment( errs = append(errs, e...) earlyReturn = true } + if e := validate.UpdateValue(ctx, op, fldPath, obj, oldObj, + func(a string, b string) bool { return a == b }, validate.NoModify).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } if earlyReturn { return // do not proceed } @@ -10482,6 +10635,11 @@ func Validate_WorkerAssignment( errs = append(errs, e...) earlyReturn = true } + if e := validate.UpdateValue(ctx, op, fldPath, obj, oldObj, + func(a string, b string) bool { return a == b }, validate.NoModify).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } if earlyReturn { return // do not proceed } @@ -10551,6 +10709,11 @@ func Validate_WorkerAssignment( errs = append(errs, e...) earlyReturn = true } + if e := validate.UpdateValue(ctx, op, fldPath, obj, oldObj, + func(a string, b string) bool { return a == b }, validate.NoModify).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } if earlyReturn { return // do not proceed } @@ -10613,6 +10776,10 @@ func Validate_WorkerAssignment( if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { earlyReturn = true } + if e := validate.UpdatePointer(ctx, op, fldPath, obj, oldObj, validate.NoModify).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } if earlyReturn { return // do not proceed } @@ -10643,6 +10810,11 @@ func Validate_WorkerAssignment( if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { earlyReturn = true } + if e := validate.UpdateValue(ctx, op, fldPath, obj, oldObj, + func(a string, b string) bool { return a == b }, validate.NoModify).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } if earlyReturn { return // do not proceed } @@ -11108,5 +11280,103 @@ func Validate_WorkerStatus( errs = append(errs, fn(fldPath.Child("sandbox_classes"), obj.SandboxClasses, oldVal, oldObj != nil)...) } + { // field ateapipb.WorkerStatus.Ips + fn := func( + fldPath *field.Path, + obj, oldObj []string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 2).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // custom validation + if e := ValidateCustom_WorkerStatus_Ips(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.WorkerStatus) []string { + return oldObj.Ips + }) + errs = append(errs, fn(fldPath.Child("ips"), obj.Ips, oldVal, oldObj != nil)...) + } + + { // field ateapipb.WorkerStatus.IssuedEpoch + fn := func( + fldPath *field.Path, + obj, oldObj *int64, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 0); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.WorkerStatus) *int64 { + return &oldObj.IssuedEpoch + }) + errs = append(errs, fn(fldPath.Child("issued_epoch"), &obj.IssuedEpoch, oldVal, oldObj != nil)...) + } + + { // field ateapipb.WorkerStatus.RegisteredEpoch + fn := func( + fldPath *field.Path, + obj, oldObj *int64, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 0); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.WorkerStatus) *int64 { + return &oldObj.RegisteredEpoch + }) + errs = append(errs, fn(fldPath.Child("registered_epoch"), &obj.RegisteredEpoch, oldVal, oldObj != nil)...) + } + return errs } diff --git a/cmd/ateapi/internal/authz/registry.go b/cmd/ateapi/internal/authz/registry.go index d850d03911..531d915a7c 100644 --- a/cmd/ateapi/internal/authz/registry.go +++ b/cmd/ateapi/internal/authz/registry.go @@ -326,7 +326,9 @@ var defaultRPCPermissions = map[string]rpcRule{ // WorkerService is served only to atelet, which each handler // authenticates by its mTLS SPIFFE ID. + ateapipb.WorkerService_AcquireWorkerEpoch_FullMethodName: inHandler("ateletauth: caller is atelet on the Worker's node"), ateapipb.WorkerService_RegisterWorker_FullMethodName: inHandler("ateletauth: caller is atelet on the Worker's node"), + ateapipb.WorkerService_SetWorkerSandboxRuntimes_FullMethodName: inHandler("ateletauth: caller is atelet on the Worker's node"), ateapipb.WorkerService_ReportActorCrashed_FullMethodName: inHandler("ateletauth: caller is atelet on the node of the Worker hosting the Actor"), ateapipb.WorkerService_RequestActorSuspend_FullMethodName: inHandler("ateletauth: caller is atelet on the node of the Worker hosting the Actor"), ateapipb.WorkerService_MintWorkerActorCertificate_FullMethodName: inHandler("ateletauth: caller is atelet on the node of the Worker the Actor is assigned to"), diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index d13dc17c6a..f29d58278d 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -3221,6 +3221,7 @@ func TestResumeActor(t *testing.T) { WorkerPodUid: podUID, WorkerPodIps: []string{"127.0.0.1"}, NodeName: "node1", + WorkerEpoch: reportedEpoch, SandboxRuntime: defaultWorkerRuntime("gvisor"), }, }, @@ -3251,11 +3252,12 @@ func TestResumeActor(t *testing.T) { WorkerPool: "pool1", WorkerPod: "worker-1", WorkerPodUid: podUID, - Ips: []string{"127.0.0.1"}, NodeName: "node1", Labels: map[string]string{poolLabelKey: ns}, Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + RegisteredEpoch: reportedEpoch, + Ips: []string{"127.0.0.1"}, // Only the ceiling the worker reported. Capacity: &ateapipb.WorkerResources{Actors: 1, CpuMillicores: 1000, MemoryBytes: 512 << 20}, // All a listing reports of the assignments: one actor and the diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index 019b065b1d..bf2670e7c0 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -692,7 +692,6 @@ func createWorkerPod(t *testing.T, tc *testContext, ns string, name string, node WorkerPool: poolName, WorkerPod: name, WorkerPodUid: string(createdPod.UID), - Ips: []string{"127.0.0.1"}, NodeName: nodeName, Labels: pool.GetLabels(), // Capacity is not settable here: a Worker gets it from its own diff --git a/cmd/ateapi/internal/controlapi/functionaltest/main_test.go b/cmd/ateapi/internal/controlapi/functionaltest/main_test.go index a40b5674d0..12b651eee7 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/main_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/main_test.go @@ -133,6 +133,9 @@ type FakeAteletServer struct { RestoreRequest *ateletpb.RestoreRequest FailRestore error RestoreDelay time.Duration + // RestoreWorkerEpoch, when set, is the epoch a successful Restore returns, + // so a test can stand in for the worker process that ran it. + RestoreWorkerEpoch func() int64 PromoteCalled bool PromoteRequest *ateletpb.PromoteSnapshotRequest @@ -191,6 +194,7 @@ func (f *FakeAteletServer) Reset() { f.RestoreRequest = nil f.FailRestore = nil f.RestoreDelay = 0 + f.RestoreWorkerEpoch = nil f.PromoteCalled = false f.PromoteRequest = nil @@ -260,7 +264,8 @@ func (f *FakeAteletServer) Run(ctx context.Context, req *ateletpb.RunRequest) (* return nil, f.FailRun } - return &ateletpb.RunResponse{}, nil + // The worker process every test Worker registers, see reportWorkerCapacity. + return &ateletpb.RunResponse{WorkerEpoch: reportedEpoch}, nil } func (f *FakeAteletServer) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest) (*ateletpb.CheckpointResponse, error) { @@ -307,7 +312,10 @@ func (f *FakeAteletServer) Restore(ctx context.Context, req *ateletpb.RestoreReq if f.FailRestore != nil { return nil, f.FailRestore } - return &ateletpb.RestoreResponse{}, nil + if f.RestoreWorkerEpoch != nil { + return &ateletpb.RestoreResponse{WorkerEpoch: f.RestoreWorkerEpoch()}, nil + } + return &ateletpb.RestoreResponse{WorkerEpoch: reportedEpoch}, nil } func (f *FakeAteletServer) Terminate(ctx context.Context, req *ateletpb.TerminateRequest) (*ateletpb.TerminateResponse, error) { diff --git a/cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go b/cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go index 0acfe318f5..e520268332 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go @@ -154,6 +154,10 @@ func setWorkerActorCapacity(t *testing.T, tc *testContext, pool string, actors i } } +// reportedEpoch is the epoch reportWorkerCapacity registers, and the one the +// fake atelet reports its workloads ran under. +const reportedEpoch = 1 + // defaultWorkerRuntime returns the default ActorSandboxRuntime that // reportWorkerCapacity records on a test worker of the given sandbox class. func defaultWorkerRuntime(class string) *ateapipb.ActorSandboxRuntime { @@ -185,10 +189,12 @@ func workerSandboxClasses(def *ateapipb.ActorSandboxRuntime, restorable ...*atea return []*ateapipb.WorkerSandboxClass{class} } -// reportWorkerCapacity stands in for the worker's capacity report, which atelet -// forwards to WorkerService in a real cluster, with compute for actors of the -// size test templates declare. It waits for the worker cache, which placement -// reads, to catch up. +// reportWorkerCapacity stands in for the worker process's registration and +// runtimes report, which atelet forwards to WorkerService in a real cluster, +// with compute for actors of the size test templates declare. A Worker not yet +// registered gets reportedEpoch, an IP and a default runtime; one already +// registered keeps them. It waits for the worker cache, which placement reads, +// to catch up. func reportWorkerCapacity(t *testing.T, tc *testContext, name string, actors int32) { t.Helper() ctx := context.Background() @@ -197,6 +203,10 @@ func reportWorkerCapacity(t *testing.T, tc *testContext, name string, actors int t.Fatalf("getting worker %s: %v", name, err) } if _, err := tc.persistence.UpdateWorker(ctx, name, store.PreconditionFrom(worker), func(toUpdate *ateapipb.Worker) error { + if toUpdate.Status.RegisteredEpoch == 0 { + toUpdate.Status.RegisteredEpoch = reportedEpoch + toUpdate.Status.Ips = []string{"127.0.0.1"} + } toUpdate.Status.Capacity = &ateapipb.WorkerResources{ Actors: actors, CpuMillicores: int64(actors) * 1000, MemoryBytes: int64(actors) * 512 << 20, diff --git a/cmd/ateapi/internal/controlapi/functionaltest/worker_restart_test.go b/cmd/ateapi/internal/controlapi/functionaltest/worker_restart_test.go new file mode 100644 index 0000000000..60f5d0eb94 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/functionaltest/worker_restart_test.go @@ -0,0 +1,156 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package functionaltest + +import ( + "context" + "slices" + "testing" + + "go.ate.dev/substrate/cmd/ateapi/internal/controlapi" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// TestResumeActor_RetriedAfterWorkerRestartDuringRestore drives a resume whose +// restore finished on a worker process that was replaced before the resume +// committed RUNNING. The resume reports Unavailable and keeps its assignment; +// the reconciler for the new epoch leaves the RESUMING actor to it; a retry +// while the new process has yet to report its runtimes waits; and the next +// retry restores the actor on the new process, stamped with its epoch and IPs. +func TestResumeActor_RetriedAfterWorkerRestartDuringRestore(t *testing.T) { + ns := namespaceForTest("ns-restart-restore") + tc := setupTest(t, ns) + defer tc.cleanup() + ctx := context.Background() + + createTemplate(t, tc, ns) + podUID := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + ref := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"} + if _, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: ref.GetAtespace(), Name: ref.GetName()}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor failed: %v", err) + } + + const newEpoch = reportedEpoch + 1 + newIPs := []string{"127.0.0.2"} + updateWorker := func(mutate func(*ateapipb.WorkerStatus)) { + t.Helper() + worker, err := tc.persistence.GetWorker(ctx, podUID) + if err != nil { + t.Fatalf("GetWorker: %v", err) + } + if _, err := tc.persistence.UpdateWorker(ctx, podUID, store.PreconditionFrom(worker), func(w *ateapipb.Worker) error { + mutate(w.Status) + return nil + }); err != nil { + t.Fatalf("UpdateWorker: %v", err) + } + } + getActor := func() *ateapipb.Actor { + t.Helper() + actor, err := tc.client.GetActor(ctx, &ateapipb.GetActorRequest{Actor: ref}) + if err != nil { + t.Fatalf("GetActor failed: %v", err) + } + return actor + } + + // The restore finishes on the original process, which then exits; a new + // process registers before the resume commits. Registering activates its + // epoch, replaces the IPs, and clears the runtimes until it reports them. + var restores int + tc.fakeAtelet.RestoreWorkerEpoch = func() int64 { + restores++ + if restores == 1 { + updateWorker(func(s *ateapipb.WorkerStatus) { + s.RegisteredEpoch = newEpoch + s.Ips = newIPs + s.SandboxClasses = nil + }) + return reportedEpoch + } + return newEpoch + } + + _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: ref}) + if got := status.Code(err); got != codes.Unavailable { + t.Fatalf("first ResumeActor = %v (err %v), want Unavailable", got, err) + } + if got := getActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RESUMING { + t.Fatalf("actor state after the first resume = %v, want RESUMING", got) + } + + // The reconciler for the new epoch leaves the RESUMING actor, and its + // claim, to the resume. + if err := controlapi.NewWorkerWorkflow(tc.persistence, nil).ReconcileAssignments(ctx, podUID); err != nil { + t.Fatalf("ReconcileAssignments: %v", err) + } + if got := getActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RESUMING { + t.Fatalf("actor state after reconcile = %v, want RESUMING", got) + } + + // Until the new process reports its runtimes the retry waits, without + // restoring or crashing anything. + _, err = tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: ref}) + if got := status.Code(err); got != codes.Unavailable { + t.Fatalf("ResumeActor before runtimes are reported = %v (err %v), want Unavailable", got, err) + } + if restores != 1 { + t.Fatalf("restores = %d before runtimes are reported, want 1", restores) + } + + updateWorker(func(s *ateapipb.WorkerStatus) { + s.SandboxClasses = workerSandboxClasses(defaultWorkerRuntime("gvisor")) + }) + resp, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: ref}) + if err != nil { + t.Fatalf("ResumeActor after the restart: %v", err) + } + if restores != 2 { + t.Errorf("restores = %d, want the retry to restore again", restores) + } + actor := resp.GetActor() + if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RUNNING { + t.Fatalf("actor state = %v, want RUNNING", got) + } + assignment := actor.GetStatus().GetWorkerAssignment() + if got := assignment.GetWorkerEpoch(); got != newEpoch { + t.Errorf("worker_epoch = %d, want %d", got, newEpoch) + } + if got := assignment.GetWorkerPodIps(); !slices.Equal(got, newIPs) { + t.Errorf("worker_pod_ips = %v, want %v", got, newIPs) + } + + // The claim taken by the first attempt is the one the actor runs under: + // the worker still books one actor, and a later reconcile keeps it. + if err := controlapi.NewWorkerWorkflow(tc.persistence, nil).ReconcileAssignments(ctx, podUID); err != nil { + t.Fatalf("ReconcileAssignments: %v", err) + } + if got := getActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RUNNING { + t.Errorf("actor state after a later reconcile = %v, want RUNNING", got) + } + worker, err := tc.persistence.GetWorker(ctx, podUID) + if err != nil { + t.Fatalf("GetWorker: %v", err) + } + if got := worker.GetStatus().GetAllocated().GetActors(); got != 1 { + t.Errorf("worker allocated actors = %d, want 1", got) + } +} diff --git a/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go b/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go index e51a7a0513..09164728c3 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go @@ -48,7 +48,6 @@ func newTestWorker(ns string) *ateapipb.Worker { WorkerPod: "worker-api-1", WorkerPodUid: testWorkerName, NodeName: "node1", - Ips: []string{"10.1.2.3"}, } } @@ -162,12 +161,13 @@ func TestListWorkers(t *testing.T) { WorkerPod: "worker-1", WorkerPodUid: podUID, NodeName: "node1", - Ips: []string{"127.0.0.1"}, Labels: map[string]string{"foo": "bar"}, Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - Capacity: &ateapipb.WorkerResources{Actors: 1, CpuMillicores: 1000, MemoryBytes: 512 << 20}, - SandboxClasses: workerSandboxClasses(defaultWorkerRuntime("gvisor")), + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + RegisteredEpoch: reportedEpoch, + Ips: []string{"127.0.0.1"}, + Capacity: &ateapipb.WorkerResources{Actors: 1, CpuMillicores: 1000, MemoryBytes: 512 << 20}, + SandboxClasses: workerSandboxClasses(defaultWorkerRuntime("gvisor")), }, }, } diff --git a/cmd/ateapi/internal/controlapi/hosting_test.go b/cmd/ateapi/internal/controlapi/hosting_test.go index a963441391..e6f8c3f164 100644 --- a/cmd/ateapi/internal/controlapi/hosting_test.go +++ b/cmd/ateapi/internal/controlapi/hosting_test.go @@ -59,11 +59,11 @@ func TestHostingNodeAfterWorkerRelease(t *testing.T) { WorkerPod: wa.GetWorkerPod(), WorkerPodUid: wa.GetWorkerPodUid(), NodeName: node, - Status: &ateapipb.WorkerStatus{ + Status: registered(&ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, SandboxClasses: []*ateapipb.WorkerSandboxClass{{SandboxClass: "gvisor"}}, - }, + }), }); err != nil { t.Fatalf("CreateWorker: %v", err) } @@ -168,11 +168,11 @@ func TestHostingNodeAfterResumeOntoNewWorker(t *testing.T) { WorkerPod: "stale-pod", WorkerPodUid: testWorkerUID("stale-pod"), NodeName: "node1", - Status: &ateapipb.WorkerStatus{ + Status: registered(&ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, SandboxClasses: []*ateapipb.WorkerSandboxClass{{SandboxClass: "microvm", DefaultRuntime: &ateapipb.WorkerSandboxRuntime{Version: testSandboxRuntime().GetVersion()}}}, - }, + }), } free := &ateapipb.Worker{ Metadata: &ateapipb.ResourceMetadata{Name: testWorkerUID("free-pod")}, @@ -182,11 +182,11 @@ func TestHostingNodeAfterResumeOntoNewWorker(t *testing.T) { WorkerPodUid: testWorkerUID("free-pod"), NodeName: "node2", Labels: map[string]string{"pool": "b"}, - Status: &ateapipb.WorkerStatus{ + Status: registered(&ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, SandboxClasses: workerSandboxClasses(testSandboxRuntime()), - }, + }), } for _, w := range []*ateapipb.Worker{stale, free} { if _, err := persistence.CreateWorker(ctx, w); err != nil { diff --git a/cmd/ateapi/internal/controlapi/mint_after_release_test.go b/cmd/ateapi/internal/controlapi/mint_after_release_test.go index a33c8a59ba..7d0ecbff03 100644 --- a/cmd/ateapi/internal/controlapi/mint_after_release_test.go +++ b/cmd/ateapi/internal/controlapi/mint_after_release_test.go @@ -70,10 +70,12 @@ func TestMintWorkerActorCertificateAfterWorkerRelease(t *testing.T) { WorkerPod: w.pod, WorkerPodUid: w.podUID, NodeName: node, + // Registered, with a sandbox class, so the scheduler takes it. Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - Capacity: &ateapipb.WorkerResources{Actors: 1}, - SandboxClasses: []*ateapipb.WorkerSandboxClass{{SandboxClass: "gvisor"}}, + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + RegisteredEpoch: 1, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + SandboxClasses: []*ateapipb.WorkerSandboxClass{{SandboxClass: "gvisor"}}, }, }); err != nil { t.Fatalf("CreateWorker(%s): %v", w.name, err) diff --git a/cmd/ateapi/internal/controlapi/worker.go b/cmd/ateapi/internal/controlapi/worker.go index 9d0c6ba68e..660d2fa92e 100644 --- a/cmd/ateapi/internal/controlapi/worker.go +++ b/cmd/ateapi/internal/controlapi/worker.go @@ -124,18 +124,10 @@ func (s *RPCService) CreateWorker(ctx context.Context, req *ateapipb.CreateWorke func (s *ServiceImpl) CreateWorker(ctx context.Context, inWorker *ateapipb.Worker) (*ateapipb.Worker, error) { // A Worker takes no Actors until MarkWorkerAvailable. outWorker := proto.CloneOf(inWorker) - // A new Worker hosts no Actors, so none are left from an earlier epoch. outWorker.Status = &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE, - ObservedEpoch: inWorker.GetEpoch(), + State: ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE, } - // Capacity and runtimes are left unset: a Worker holds nothing and runs no - // sandbox until its own worker says what it has, through - // WorkerService.RegisterWorker. Nothing is placed on it in the meantime, - // which is the point -- the alternative is guessing on the Worker's behalf - // and placing against the guess. - // Verify that the result is properly valid before storing it. if errs := apivalidation.ValidateWorkerUpdate(ctx, field.NewPath("worker"), outWorker, inWorker, true); len(errs) > 0 { return nil, resources.ToInternalError(errs) @@ -153,7 +145,7 @@ func (s *ServiceImpl) CreateWorker(ctx context.Context, inWorker *ateapipb.Worke } // UpdateWorker replaces the stored Worker with the one the request carries. -// Only labels and epoch are the caller's to change; a request that alters an +// Only labels are the caller's to change; a request that alters an // immutable field — including by leaving it unset, which would clear it — is // rejected. // The service layer enforces that with declarative validation against the diff --git a/cmd/ateapi/internal/controlapi/worker_assignment_reconciler_test.go b/cmd/ateapi/internal/controlapi/worker_assignment_reconciler_test.go index 6962f9a882..af834070c5 100644 --- a/cmd/ateapi/internal/controlapi/worker_assignment_reconciler_test.go +++ b/cmd/ateapi/internal/controlapi/worker_assignment_reconciler_test.go @@ -30,11 +30,10 @@ type noWorkers struct{} func (noWorkers) AddHandler(func(*ateapipb.Worker)) {} -func epochWorker(name string, epoch, observed int64) *ateapipb.Worker { +func epochWorker(name string, registered, observed int64) *ateapipb.Worker { return &ateapipb.Worker{ Metadata: &ateapipb.ResourceMetadata{Name: name}, - Epoch: epoch, - Status: &ateapipb.WorkerStatus{ObservedEpoch: observed}, + Status: &ateapipb.WorkerStatus{RegisteredEpoch: registered, ObservedEpoch: observed}, } } @@ -46,7 +45,7 @@ func TestWorkerAssignmentReconciler_QueuesUnobservedWorkers(t *testing.T) { epochWorker("observed", 2, 2), epochWorker("restarted", 3, 2), epochWorker("unreported", 0, 0), - epochWorker("registered-before-epochs", 4, 0), + epochWorker("first-registration", 1, 0), } { r.enqueue(w) } @@ -57,8 +56,8 @@ func TestWorkerAssignmentReconciler_QueuesUnobservedWorkers(t *testing.T) { got[name] = true r.queue.Done(name) } - want := map[string]bool{"restarted": true, "registered-before-epochs": true} - if len(got) != len(want) || !got["restarted"] || !got["registered-before-epochs"] { + want := map[string]bool{"restarted": true, "first-registration": true} + if len(got) != len(want) || !got["restarted"] || !got["first-registration"] { t.Errorf("queued %v, want %v", got, want) } } @@ -67,10 +66,10 @@ func TestWorkerAssignmentReconciler_QueuesUnobservedWorkers(t *testing.T) { // retried. func TestWorkerAssignmentReconciler_SkipsWorkerLeasedElsewhere(t *testing.T) { ctx := context.Background() - svc, persistence := newWorkerAPIService(t) + _, persistence := newWorkerAPIService(t) seedEpochWorker(t, ctx, persistence, 1, 1) seedRunningActor(t, ctx, persistence) - mustRaiseEpoch(t, ctx, svc, persistence, 2) + mustRaiseEpoch(t, ctx, persistence, 2) lease, err := persistence.AcquireLease(ctx, workerAssignmentLeaseKey(apiWorkerName)) if err != nil { @@ -116,12 +115,12 @@ func startAssignmentReconciler(t *testing.T, ctx context.Context, persistence st func TestWorkerAssignmentReconciler_ReleasesRestartedWorker(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - svc, persistence := newWorkerAPIService(t) + _, persistence := newWorkerAPIService(t) seedEpochWorker(t, ctx, persistence, 1, 1) seedRunningActor(t, ctx, persistence) startAssignmentReconciler(t, ctx, persistence) - mustRaiseEpoch(t, ctx, svc, persistence, 2) + mustRaiseEpoch(t, ctx, persistence, 2) waitForObservedEpoch(t, ctx, persistence, 2) if got := mustGetActor(t, ctx, persistence).GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_CRASHED { @@ -137,7 +136,7 @@ func TestWorkerAssignmentReconciler_ReleasesRestartedWorker(t *testing.T) { func TestWorkerAssignmentReconciler_RetriesAfterLeaseFrees(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - svc, persistence := newWorkerAPIService(t) + _, persistence := newWorkerAPIService(t) seedEpochWorker(t, ctx, persistence, 1, 1) seedRunningActor(t, ctx, persistence) @@ -146,7 +145,7 @@ func TestWorkerAssignmentReconciler_RetriesAfterLeaseFrees(t *testing.T) { t.Fatalf("AcquireLease() failed: %v", err) } startAssignmentReconciler(t, ctx, persistence) - mustRaiseEpoch(t, ctx, svc, persistence, 2) + mustRaiseEpoch(t, ctx, persistence, 2) time.Sleep(200 * time.Millisecond) lease.Close() @@ -160,10 +159,10 @@ func TestWorkerAssignmentReconciler_RetriesAfterLeaseFrees(t *testing.T) { func TestWorkerAssignmentReconciler_ReleasesWorkerRaisedBeforeStart(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - svc, persistence := newWorkerAPIService(t) + _, persistence := newWorkerAPIService(t) seedEpochWorker(t, ctx, persistence, 1, 1) seedRunningActor(t, ctx, persistence) - mustRaiseEpoch(t, ctx, svc, persistence, 2) + mustRaiseEpoch(t, ctx, persistence, 2) startAssignmentReconciler(t, ctx, persistence) diff --git a/cmd/ateapi/internal/controlapi/worker_test.go b/cmd/ateapi/internal/controlapi/worker_test.go index bce229aa69..54e307323c 100644 --- a/cmd/ateapi/internal/controlapi/worker_test.go +++ b/cmd/ateapi/internal/controlapi/worker_test.go @@ -46,7 +46,6 @@ func validWorker(name string, mods ...func(*ateapipb.Worker)) *ateapipb.Worker { WorkerPod: "worker-pod-1", WorkerPodUid: name, NodeName: "node-1", - Ips: []string{"10.1.2.3"}, } for _, m := range mods { m(w) @@ -291,10 +290,6 @@ func TestCreateWorker_InvalidArgument(t *testing.T) { {name: "no worker"}, {name: "no name", mutate: func(w *ateapipb.Worker) { w.Metadata = &ateapipb.ResourceMetadata{} }}, {name: "atespace set", mutate: func(w *ateapipb.Worker) { w.Metadata.Atespace = "team-a" }}, - {name: "no ips", mutate: func(w *ateapipb.Worker) { w.Ips = nil }}, - {name: "bad ip", mutate: func(w *ateapipb.Worker) { w.Ips = []string{"not-an-ip"} }}, - {name: "two ipv4 ips", mutate: func(w *ateapipb.Worker) { w.Ips = []string{"10.1.2.3", "10.1.2.4"} }}, - {name: "three ips", mutate: func(w *ateapipb.Worker) { w.Ips = []string{"10.1.2.3", "fd00::1", "fd00::2"} }}, {name: "no node", mutate: func(w *ateapipb.Worker) { w.NodeName = "" }}, {name: "no pool", mutate: func(w *ateapipb.Worker) { w.WorkerPool = "" }}, {name: "no pod", mutate: func(w *ateapipb.Worker) { w.WorkerPod = "" }}, @@ -460,8 +455,6 @@ func TestUpdateWorker_Errors(t *testing.T) { }, codes.NotFound}, // Immutable fields, changed. A replacement update carries the whole // worker, so these are the cases where it carries a different one. - {"ips changed", func(w *ateapipb.Worker) { w.Ips = []string{"10.9.9.9"} }, codes.InvalidArgument}, - {"ips family added", func(w *ateapipb.Worker) { w.Ips = append(w.Ips, "fd00::1") }, codes.InvalidArgument}, {"worker_pod changed", func(w *ateapipb.Worker) { w.WorkerPod = "worker-pod-2" }, codes.InvalidArgument}, {"node_name changed", func(w *ateapipb.Worker) { w.NodeName = "node-2" }, codes.InvalidArgument}, // capacity is deliberately absent here: it may change (the pool's actor @@ -470,7 +463,7 @@ func TestUpdateWorker_Errors(t *testing.T) { // // And immutable fields dropped, which a replacement update reads as a // request to clear them. Rejected rather than silently applied. - {"ips omitted", func(w *ateapipb.Worker) { w.Ips = nil }, codes.InvalidArgument}, + {"node_name omitted", func(w *ateapipb.Worker) { w.NodeName = "" }, codes.InvalidArgument}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -825,7 +818,6 @@ func TestServiceImplUpdateWorker_ImmutableFields(t *testing.T) { {"worker_pod", "worker_pod", func(w *ateapipb.Worker) { w.WorkerPod = "other-pod" }}, {"worker_pod_uid", "worker_pod_uid", func(w *ateapipb.Worker) { w.WorkerPodUid = apiOtherWorkerName }}, {"node_name", "node_name", func(w *ateapipb.Worker) { w.NodeName = "other-node" }}, - {"ips", "ips", func(w *ateapipb.Worker) { w.Ips = []string{"10.0.0.9"} }}, // capacity is absent: it is reported into status, which a client // cannot write. See TestUpdateWorker_CannotChangeCapacity. } { diff --git a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments.go b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments.go index fc101af121..28eb5c2ec8 100644 --- a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments.go +++ b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments.go @@ -25,17 +25,18 @@ import ( "go.ate.dev/substrate/pkg/proto/ateapipb" ) -// needsAssignmentReconcile reports whether worker's epoch has risen past the -// last one its earlier Actors were released for. +// needsAssignmentReconcile reports whether worker's registered epoch has risen +// past the last one its earlier Actors were released for. func needsAssignmentReconcile(worker *ateapipb.Worker) bool { - return worker.GetEpoch() > worker.GetStatus().GetObservedEpoch() + return worker.GetStatus().GetRegisteredEpoch() > worker.GetStatus().GetObservedEpoch() } -// ReconcileAssignments brings a Worker's status.observed_epoch up to its epoch. -// A raised epoch means its worker restarted, taking with it the sandboxes of the -// Actors placed during an earlier epoch, so those Actors are crashed and their -// assignments released before observed_epoch records it. A failure leaves -// observed_epoch where it was, so the next pass redoes the release. +// ReconcileAssignments brings a Worker's status.observed_epoch up to its +// registered_epoch. A raised registered_epoch means a new worker process registered, +// and every earlier one has exited and lost the sandboxes of the Actors that +// ran under it, so those Actors are crashed and their assignments released +// before observed_epoch records it. A failure leaves observed_epoch where it +// was, so the next pass redoes the release. // // Nothing to do, including a Worker that is gone, is success. func (w *WorkerWorkflow) ReconcileAssignments(ctx context.Context, name string) (err error) { @@ -55,15 +56,9 @@ func (w *WorkerWorkflow) ReconcileAssignments(ctx context.Context, name string) markSkipped(ctx, "epoch already observed") return nil } - epoch := worker.GetEpoch() - - // A Worker whose observed_epoch is 0 was registered before epochs were - // reported, so its current epoch is recorded without releasing anything: - // nothing says which of its Actors predate it. - if worker.GetStatus().GetObservedEpoch() != 0 { - if err := w.releaseAssignmentsBefore(ctx, worker, epoch); err != nil { - return err - } + epoch := worker.GetStatus().GetRegisteredEpoch() + if err := w.releaseAssignmentsBefore(ctx, worker, epoch); err != nil { + return err } return w.recordObservedEpoch(ctx, name, worker.GetMetadata().GetUid(), epoch) } @@ -77,11 +72,9 @@ const recordObservedEpochAttempts = 5 // alone. // // A write that lands on the Worker after the sweep does not make the record -// wrong, so a version conflict is retried against a fresh read. Every -// assignment is stamped with the Worker's epoch under its row lock, the same -// lock that raises the epoch, so a row stamped before epoch was committed -// before epoch was read and the sweep found it. Later binds are stamped epoch -// or higher. +// wrong, so a version conflict is retried against a fresh read. An Actor +// recorded RUNNING under an epoch before epoch was bound, and its assignment +// row written, before that epoch was activated, so the sweep found the row. func (w *WorkerWorkflow) recordObservedEpoch(ctx context.Context, name, uid string, epoch int64) error { for attempt := 1; ; attempt++ { worker, err := w.store.GetWorker(ctx, name) @@ -110,9 +103,9 @@ func (w *WorkerWorkflow) recordObservedEpoch(ctx context.Context, name, uid stri // Actor's lease. var errActorBusy = errors.New("actor has an operation in progress") -// releaseAssignmentsBefore releases every assignment on worker made during an -// epoch before epoch. An assignment whose Actor is busy is skipped until the -// rest are released, then reported, so the pass is retried. +// releaseAssignmentsBefore releases every assignment on worker whose Actor ran +// under an epoch before epoch. An assignment whose Actor is busy is skipped +// until the rest are released, then reported, so the pass is retried. func (w *WorkerWorkflow) releaseAssignmentsBefore(ctx context.Context, worker *ateapipb.Worker, epoch int64) error { name := worker.GetMetadata().GetName() busy := 0 @@ -124,9 +117,6 @@ func (w *WorkerWorkflow) releaseAssignmentsBefore(ctx context.Context, worker *a return fmt.Errorf("while listing the assignments of worker %s: %w", name, err) } for _, assignment := range page.Items { - if assignment.GetWorkerEpoch() >= epoch { - continue - } err := w.releaseEarlierAssignment(ctx, worker, assignment, epoch) if errors.Is(err, errActorBusy) { busy++ @@ -147,15 +137,15 @@ func (w *WorkerWorkflow) releaseAssignmentsBefore(ctx context.Context, worker *a return nil } -// releaseEarlierAssignment clears one assignment made during an epoch of the -// Worker before epoch, under its Actor's lease so no operation on the Actor is -// midway. The Actor, read under the lease, may have been bound here again in -// the current epoch since the list; its assignment then carries that epoch, as -// the rebound row does, and is left alone. An Actor still running on it is -// crashed first and its row released after, so a failure in between leaves the -// row for a retry to find, and the Actor already CRASHED. A row whose Actor -// does not point here is left over from an operation that failed partway, and -// is released. +// releaseEarlierAssignment clears one assignment whose Actor ran under an epoch +// of the Worker before epoch, under its Actor's lease so no operation on the +// Actor is midway. An Actor recorded RUNNING under epoch or later is left +// alone, and so is a RESUMING Actor, which is left to its resume. An Actor +// still running on it is crashed first and its row released after, so a +// failure in between leaves the row for a retry to find, and the Actor already +// CRASHED. A row whose Actor does not point here names nothing that ran on the +// Worker, so the restart lost nothing of it and it is left alone: it is a claim +// a resume is still recording, or one the Actor's next resume recovers. func (w *WorkerWorkflow) releaseEarlierAssignment(ctx context.Context, worker *ateapipb.Worker, assignment *ateapipb.ActorAssignment, epoch int64) error { name := worker.GetMetadata().GetName() release := func() error { @@ -191,11 +181,18 @@ func (w *WorkerWorkflow) releaseEarlierAssignment(ctx context.Context, worker *a } if actor.GetStatus().GetWorkerAssignment().GetWorker().GetName() != name { - return release() + return nil } if actor.GetStatus().GetWorkerAssignment().GetWorkerEpoch() >= epoch { return nil } + if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_RESUMING { + // A RESUMING Actor is left to its resume, which records the epoch of + // the worker process that restores it, and restores it again if that + // process is gone, whether finalizeRunning finds it replaced or the + // Restore was cut short by the restart. + return nil + } if actor.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { if err := w.crashBoundActor(ctx, worker, actorRef, actor, "Releasing actor from a restarted worker", crashMessageWorkerRestarted); err != nil { return err diff --git a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go index b3a1b1a13a..23bb30a155 100644 --- a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go @@ -24,44 +24,33 @@ import ( "go.ate.dev/substrate/cmd/ateapi/internal/store" "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" "go.ate.dev/substrate/cmd/ateapi/internal/workercache" - "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/pkg/proto/ateapipb" - "google.golang.org/grpc/codes" ) -// withEpoch returns a modifier func (see validWorker) which sets the worker's -// epoch. -func withEpoch(epoch int64) func(*ateapipb.Worker) { - return func(w *ateapipb.Worker) { w.Epoch = epoch } -} - -// seedEpochWorker stores apiWorkerName at epoch, with observed as the epoch -// its earlier Actors were last released for. +// seedEpochWorker stores apiWorkerName with epoch active, and observed as the +// epoch its earlier Actors were last released for. func seedEpochWorker(t *testing.T, ctx context.Context, persistence store.Interface, epoch, observed int64) { t.Helper() - seedAPIWorker(t, ctx, persistence, validWorker(apiWorkerName, withEpoch(epoch), func(w *ateapipb.Worker) { + seedAPIWorker(t, ctx, persistence, validWorker(apiWorkerName, func(w *ateapipb.Worker) { w.Status = &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - ObservedEpoch: observed, + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + RegisteredEpoch: epoch, + ObservedEpoch: observed, } })) } -// raiseEpoch sends the UpdateWorker the syncer sends when the worker pod's -// worker restarts. -func raiseEpoch(t *testing.T, ctx context.Context, svc *RPCService, persistence store.Interface, epoch int64) (*ateapipb.Worker, error) { - t.Helper() - return svc.UpdateWorker(ctx, &ateapipb.UpdateWorkerRequest{ - Worker: updateFrom(mustGetWorker(t, ctx, persistence), func(w *ateapipb.Worker) { w.Epoch = epoch }), - }) -} - -// mustRaiseEpoch is raiseEpoch for a raise the test expects to succeed. -func mustRaiseEpoch(t *testing.T, ctx context.Context, svc *RPCService, persistence store.Interface, epoch int64) { +// mustRaiseEpoch activates epoch, as WorkerService.RegisterWorker does when a +// new worker process registers with it. +func mustRaiseEpoch(t *testing.T, ctx context.Context, persistence store.Interface, epoch int64) { t.Helper() - if _, err := raiseEpoch(t, ctx, svc, persistence, epoch); err != nil { - t.Fatalf("UpdateWorker() raising epoch to %d failed: %v", epoch, err) + worker := mustGetWorker(t, ctx, persistence) + if _, err := persistence.UpdateWorker(ctx, apiWorkerName, store.PreconditionFrom(worker), func(toUpdate *ateapipb.Worker) error { + toUpdate.Status.RegisteredEpoch = epoch + return nil + }); err != nil { + t.Fatalf("UpdateWorker() activating epoch %d failed: %v", epoch, err) } } @@ -97,84 +86,9 @@ func seedRunningActor(t *testing.T, ctx context.Context, persistence store.Inter assignAPIWorker(t, ctx, persistence, apiWorkerName, actor.GetMetadata().GetUid()) } -func TestBindActorToWorker_StampsWorkerEpoch(t *testing.T) { - ctx := context.Background() - _, persistence := newWorkerAPIService(t) - seedEpochWorker(t, ctx, persistence, 4, 4) - - assignment := newAPIAssignment("uid-1") - assignment.WorkerEpoch = 99 - if err := persistence.BindActorToWorker(ctx, apiWorkerName, assignment, nil); err != nil { - t.Fatalf("BindActorToWorker() failed: %v", err) - } - if got := assignment.GetWorkerEpoch(); got != 4 { - t.Errorf("assignment worker_epoch read back = %d, want 4", got) - } - if got := firstAssignment(t, persistence, apiWorkerName).GetWorkerEpoch(); got != 4 { - t.Errorf("stored worker_epoch = %d, want 4", got) - } -} - -// A new Worker hosts nothing, so its first epoch is observed from the start. -func TestCreateWorker_ObservesInitialEpoch(t *testing.T) { - ctx := context.Background() - svc, _ := newWorkerAPIService(t) - - got, err := svc.CreateWorker(ctx, &ateapipb.CreateWorkerRequest{Worker: validWorker(apiWorkerName, withEpoch(3))}) - if err != nil { - t.Fatalf("CreateWorker() failed: %v", err) - } - if got.GetEpoch() != 3 || got.GetStatus().GetObservedEpoch() != 3 { - t.Errorf("epoch = %d, observed_epoch = %d, want both 3", got.GetEpoch(), got.GetStatus().GetObservedEpoch()) - } -} - -// Raising the epoch only records it; the release is the reconciler's. -func TestUpdateWorker_RaisedEpochLeavesActors(t *testing.T) { - ctx := context.Background() - svc, persistence := newWorkerAPIService(t) - seedEpochWorker(t, ctx, persistence, 1, 1) - seedRunningActor(t, ctx, persistence) - - updated, err := raiseEpoch(t, ctx, svc, persistence, 2) - if err != nil { - t.Fatalf("UpdateWorker() failed: %v", err) - } - if updated.GetEpoch() != 2 || updated.GetStatus().GetObservedEpoch() != 1 { - t.Errorf("epoch = %d, observed_epoch = %d, want 2 and 1", updated.GetEpoch(), updated.GetStatus().GetObservedEpoch()) - } - if got := mustGetActor(t, ctx, persistence).GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RUNNING { - t.Errorf("actor state = %v, want RUNNING", got) - } -} - -func TestUpdateWorker_EpochCannotDecrease(t *testing.T) { - for _, tc := range []struct { - name string - to int64 - }{ - {name: "lowered", to: 2}, - {name: "cleared", to: 0}, - } { - t.Run(tc.name, func(t *testing.T) { - ctx := context.Background() - svc, persistence := newWorkerAPIService(t) - seedEpochWorker(t, ctx, persistence, 3, 3) - - _, err := raiseEpoch(t, ctx, svc, persistence, tc.to) - if got := apierror.Code(err); got != codes.InvalidArgument { - t.Fatalf("UpdateWorker() = %v (err %v), want %v", got, err, codes.InvalidArgument) - } - if got := mustGetWorker(t, ctx, persistence).GetEpoch(); got != 3 { - t.Errorf("worker epoch = %d, want 3", got) - } - }) - } -} - func TestReconcileAssignments_CrashesEarlierActors(t *testing.T) { ctx := context.Background() - svc, persistence := newWorkerAPIService(t) + _, persistence := newWorkerAPIService(t) seedEpochWorker(t, ctx, persistence, 1, 1) actor := seedAPIActor(t, ctx, persistence, ateapipb.ActorState_ACTOR_STATE_RUNNING, func(a *ateapipb.Actor) { a.Status.LastAssignedGeneration = 1 @@ -183,7 +97,7 @@ func TestReconcileAssignments_CrashesEarlierActors(t *testing.T) { } }) assignAPIWorker(t, ctx, persistence, apiWorkerName, actor.GetMetadata().GetUid()) - mustRaiseEpoch(t, ctx, svc, persistence, 2) + mustRaiseEpoch(t, ctx, persistence, 2) mustReconcileAssignments(t, ctx, persistence) @@ -215,14 +129,17 @@ func TestReconcileAssignments_CrashesEarlierActors(t *testing.T) { } } -// An Actor placed after the raise was stamped with the new epoch, so it runs on -// the restarted worker and is kept. -func TestReconcileAssignments_KeepsActorsPlacedAfterRaise(t *testing.T) { +// An Actor restored on the new worker process records that process's epoch +// when it becomes RUNNING, so it is kept. +func TestReconcileAssignments_KeepsActorRestoredOnNewProcess(t *testing.T) { ctx := context.Background() - svc, persistence := newWorkerAPIService(t) + _, persistence := newWorkerAPIService(t) seedEpochWorker(t, ctx, persistence, 1, 1) - mustRaiseEpoch(t, ctx, svc, persistence, 2) seedRunningActor(t, ctx, persistence) + mustRaiseEpoch(t, ctx, persistence, 2) + mustUpdateActorStatus(t, ctx, persistence, mustGetActor(t, ctx, persistence), func(s *ateapipb.ActorStatus) { + s.WorkerAssignment.WorkerEpoch = 2 + }) mustReconcileAssignments(t, ctx, persistence) @@ -237,31 +154,6 @@ func TestReconcileAssignments_KeepsActorsPlacedAfterRaise(t *testing.T) { } } -// A Worker registered before epochs were reported has observed_epoch 0. Its -// first epoch is recorded without crashing anything, and a later restart -// treats its unstamped assignments as older. -func TestReconcileAssignments_FirstEpochCrashesNothing(t *testing.T) { - ctx := context.Background() - svc, persistence := newWorkerAPIService(t) - seedAPIWorker(t, ctx, persistence, validWorker(apiWorkerName)) - seedRunningActor(t, ctx, persistence) - - mustRaiseEpoch(t, ctx, svc, persistence, 3) - mustReconcileAssignments(t, ctx, persistence) - if got := mustGetActor(t, ctx, persistence).GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RUNNING { - t.Fatalf("actor state after the first epoch = %v, want RUNNING", got) - } - if got := mustGetWorker(t, ctx, persistence).GetStatus().GetObservedEpoch(); got != 3 { - t.Fatalf("observed_epoch = %d, want 3", got) - } - - mustRaiseEpoch(t, ctx, svc, persistence, 4) - mustReconcileAssignments(t, ctx, persistence) - if got := mustGetActor(t, ctx, persistence).GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_CRASHED { - t.Errorf("actor state after a restart = %v, want CRASHED", got) - } -} - func TestReconcileAssignments_AlreadyObservedWritesNothing(t *testing.T) { ctx := context.Background() _, persistence := newWorkerAPIService(t) @@ -297,19 +189,20 @@ func TestReconcileAssignments_ReleasesByActorState(t *testing.T) { // Suspended cleanly before the restart: still resumable. {name: "suspended stays suspended", state: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, wantState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, wantRelease: true}, {name: "crashed is released", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantRelease: true}, - // Left behind by an operation that failed partway: released without - // touching the Actor. - {name: "crashed unassigned is released", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, pointsAt: unassigned, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantRelease: true}, - {name: "suspended unassigned is released", state: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, pointsAt: unassigned, wantState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, wantRelease: true}, - {name: "running elsewhere is released", state: ateapipb.ActorState_ACTOR_STATE_RUNNING, pointsAt: otherWorker, wantState: ateapipb.ActorState_ACTOR_STATE_RUNNING, wantRelease: true}, - {name: "suspended elsewhere is released", state: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, pointsAt: otherWorker, wantState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, wantRelease: true}, - {name: "resuming crashes", state: ateapipb.ActorState_ACTOR_STATE_RESUMING, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantRelease: true}, + // Nothing it ran on the Worker: a claim a resume is still recording, + // or one its next resume recovers. Left alone. + {name: "crashed unassigned is left", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, pointsAt: unassigned, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED}, + {name: "suspended unassigned is left", state: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, pointsAt: unassigned, wantState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}, + {name: "running elsewhere is left", state: ateapipb.ActorState_ACTOR_STATE_RUNNING, pointsAt: otherWorker, wantState: ateapipb.ActorState_ACTOR_STATE_RUNNING}, + {name: "suspended elsewhere is left", state: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, pointsAt: otherWorker, wantState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}, + // Nothing it ran is kept: its resume restores it on the new process. + {name: "resuming is left to its resume", state: ateapipb.ActorState_ACTOR_STATE_RESUMING, wantState: ateapipb.ActorState_ACTOR_STATE_RESUMING}, {name: "deleting stays deleting", state: ateapipb.ActorState_ACTOR_STATE_DELETING, wantState: ateapipb.ActorState_ACTOR_STATE_DELETING, wantRelease: true}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { ctx := context.Background() - svc, persistence := newWorkerAPIService(t) + _, persistence := newWorkerAPIService(t) seedEpochWorker(t, ctx, persistence, 1, 1) actor := seedAPIActor(t, ctx, persistence, tc.state, func(a *ateapipb.Actor) { switch tc.pointsAt { @@ -320,7 +213,7 @@ func TestReconcileAssignments_ReleasesByActorState(t *testing.T) { } }) assignAPIWorker(t, ctx, persistence, apiWorkerName, actor.GetMetadata().GetUid()) - mustRaiseEpoch(t, ctx, svc, persistence, 2) + mustRaiseEpoch(t, ctx, persistence, 2) mustReconcileAssignments(t, ctx, persistence) @@ -339,7 +232,7 @@ func TestReconcileAssignments_ReleasesByActorState(t *testing.T) { // running, so it is released without touching the live Actor. func TestReconcileAssignments_ReleasesOrphanedAssignments(t *testing.T) { ctx := context.Background() - svc, persistence := newWorkerAPIService(t) + _, persistence := newWorkerAPIService(t) seedEpochWorker(t, ctx, persistence, 1, 1) seedAPIActor(t, ctx, persistence, ateapipb.ActorState_ACTOR_STATE_RUNNING) assignAPIWorker(t, ctx, persistence, apiWorkerName, "11111111-1111-1111-1111-111111111111") @@ -348,7 +241,7 @@ func TestReconcileAssignments_ReleasesOrphanedAssignments(t *testing.T) { if err := persistence.BindActorToWorker(ctx, apiWorkerName, orphan, nil); err != nil { t.Fatalf("BindActorToWorker() failed: %v", err) } - mustRaiseEpoch(t, ctx, svc, persistence, 2) + mustRaiseEpoch(t, ctx, persistence, 2) mustReconcileAssignments(t, ctx, persistence) @@ -364,7 +257,7 @@ func TestReconcileAssignments_ReleasesOrphanedAssignments(t *testing.T) { // are released, but observed_epoch stays put until a pass gets the Actor too. func TestReconcileAssignments_WaitsForBusyActor(t *testing.T) { ctx := context.Background() - svc, persistence := newWorkerAPIService(t) + _, persistence := newWorkerAPIService(t) seedEpochWorker(t, ctx, persistence, 1, 1) seedRunningActor(t, ctx, persistence) orphan := newAPIAssignment("22222222-2222-2222-2222-222222222222") @@ -372,7 +265,7 @@ func TestReconcileAssignments_WaitsForBusyActor(t *testing.T) { if err := persistence.BindActorToWorker(ctx, apiWorkerName, orphan, nil); err != nil { t.Fatalf("BindActorToWorker() failed: %v", err) } - mustRaiseEpoch(t, ctx, svc, persistence, 2) + mustRaiseEpoch(t, ctx, persistence, 2) lease, err := persistence.AcquireLease(ctx, actorLeaseKey(apiActorRef)) if err != nil { @@ -434,10 +327,10 @@ func (s *hookedStore) UpdateWorker(ctx context.Context, name string, preconditio // assignment runs on the restarted worker, so it is kept. func TestReconcileAssignments_KeepsActorReboundSinceList(t *testing.T) { ctx := context.Background() - svc, persistence := newWorkerAPIService(t) + _, persistence := newWorkerAPIService(t) seedEpochWorker(t, ctx, persistence, 1, 1) seedRunningActor(t, ctx, persistence) - mustRaiseEpoch(t, ctx, svc, persistence, 2) + mustRaiseEpoch(t, ctx, persistence, 2) actorUID := mustGetActor(t, ctx, persistence).GetMetadata().GetUid() rebinding := &hookedStore{Interface: persistence, beforeLease: func() { @@ -445,8 +338,8 @@ func TestReconcileAssignments_KeepsActorReboundSinceList(t *testing.T) { t.Fatalf("ReleaseActorFromWorker() failed: %v", err) } assignAPIWorker(t, ctx, persistence, apiWorkerName, actorUID) - // As assignWorkerAttempt does, the Actor records the epoch it was - // bound in. + // As finalizeRunning does, the Actor records the epoch of the + // process it was restored on. actor := mustGetActor(t, ctx, persistence) if _, err := persistence.UpdateActor(ctx, apiActorRef, store.PreconditionFrom(actor), func(a *ateapipb.Actor) error { a.Status.WorkerAssignment.WorkerEpoch = 2 @@ -462,8 +355,8 @@ func TestReconcileAssignments_KeepsActorReboundSinceList(t *testing.T) { if got := mustGetActor(t, ctx, persistence).GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RUNNING { t.Errorf("actor state = %v, want RUNNING", got) } - if a := firstAssignment(t, persistence, apiWorkerName); a.GetWorkerEpoch() != 2 { - t.Errorf("assignment = %v, want the epoch 2 binding kept", a) + if firstAssignment(t, persistence, apiWorkerName) == nil { + t.Error("assignment released, want the new binding kept") } if got := mustGetWorker(t, ctx, persistence).GetStatus().GetObservedEpoch(); got != 2 { t.Errorf("observed_epoch = %d, want 2", got) @@ -471,14 +364,14 @@ func TestReconcileAssignments_KeepsActorReboundSinceList(t *testing.T) { } // A bind that moves the Worker on after the pass read it only makes recording -// observed_epoch read the Worker again: the bind is stamped with the current -// epoch, so it is kept. +// observed_epoch read the Worker again: the bind came after the sweep, so it is +// kept. func TestReconcileAssignments_RetriesRecordAfterConcurrentBind(t *testing.T) { ctx := context.Background() - svc, persistence := newWorkerAPIService(t) + _, persistence := newWorkerAPIService(t) seedEpochWorker(t, ctx, persistence, 1, 1) seedRunningActor(t, ctx, persistence) - mustRaiseEpoch(t, ctx, svc, persistence, 2) + mustRaiseEpoch(t, ctx, persistence, 2) const boundUID = "33333333-3333-3333-3333-333333333333" binding := &hookedStore{Interface: persistence, beforeUpdateWorker: func() { @@ -494,13 +387,9 @@ func TestReconcileAssignments_RetriesRecordAfterConcurrentBind(t *testing.T) { if got := mustGetActor(t, ctx, persistence).GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_CRASHED { t.Errorf("actor state = %v, want CRASHED", got) } - a, err := persistence.GetWorkerAssignment(ctx, apiWorkerName, boundUID) - if err != nil { + if _, err := persistence.GetWorkerAssignment(ctx, apiWorkerName, boundUID); err != nil { t.Fatalf("GetWorkerAssignment() = %v, want the concurrent bind kept", err) } - if a.GetWorkerEpoch() != 2 { - t.Errorf("concurrent bind epoch = %d, want 2", a.GetWorkerEpoch()) - } } // A Worker replaced under the same name after the pass read it is a new @@ -508,9 +397,9 @@ func TestReconcileAssignments_RetriesRecordAfterConcurrentBind(t *testing.T) { // own restarts would then look already observed. func TestReconcileAssignments_LeavesReplacedWorker(t *testing.T) { ctx := context.Background() - svc, persistence := newWorkerAPIService(t) + _, persistence := newWorkerAPIService(t) seedEpochWorker(t, ctx, persistence, 1, 1) - mustRaiseEpoch(t, ctx, svc, persistence, 5) + mustRaiseEpoch(t, ctx, persistence, 5) replacing := &hookedStore{Interface: persistence, beforeUpdateWorker: func() { if _, err := persistence.DeleteWorker(ctx, apiWorkerName, store.DeletePreconditions{}); err != nil { @@ -531,10 +420,10 @@ func TestReconcileAssignments_LeavesReplacedWorker(t *testing.T) { // redoes it. func TestReconcileAssignments_FailureKeepsObservedEpoch(t *testing.T) { ctx := context.Background() - svc, persistence := newWorkerAPIService(t) + _, persistence := newWorkerAPIService(t) seedEpochWorker(t, ctx, persistence, 1, 1) seedRunningActor(t, ctx, persistence) - mustRaiseEpoch(t, ctx, svc, persistence, 2) + mustRaiseEpoch(t, ctx, persistence, 2) failing := NewWorkerWorkflow(failingUpdateActorStore{Interface: persistence, err: errors.New("crash failed")}, nil) if err := failing.ReconcileAssignments(ctx, apiWorkerName); err == nil { @@ -562,24 +451,24 @@ func TestReconcileAssignments_WorkerGone(t *testing.T) { mustReconcileAssignments(t, ctx, persistence) } -// The Actor's assignment carries the epoch the bind read under the Worker's -// row lock, not the one in the scheduler's cache. -func TestAssignWorkerAttempt_StampsWorkerEpoch(t *testing.T) { +// Placing an Actor records no epoch: only the worker process that restores it +// knows which epoch it runs under. +func TestAssignWorkerAttempt_LeavesWorkerEpochUnset(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) name := testWorkerUID("pod-1") - created, err := persistence.CreateWorker(ctx, &ateapipb.Worker{ + _, err := persistence.CreateWorker(ctx, &ateapipb.Worker{ Metadata: &ateapipb.ResourceMetadata{Name: name}, WorkerNamespace: "worker-ns", WorkerPool: "pool", WorkerPod: "pod-1", WorkerPodUid: name, NodeName: "node-1", - Epoch: 1, Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - Capacity: &ateapipb.WorkerResources{Actors: 1}, - SandboxClasses: workerSandboxClasses(testSandboxRuntime()), + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + RegisteredEpoch: 1, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + SandboxClasses: workerSandboxClasses(testSandboxRuntime()), }, }) if err != nil { @@ -596,28 +485,13 @@ func TestAssignWorkerAttempt_StampsWorkerEpoch(t *testing.T) { t.Fatalf("workercache.Start: %v", err) } - precondition := store.Precondition{UID: created.GetMetadata().GetUid(), Version: created.GetMetadata().GetVersion()} - if _, err := persistence.UpdateWorker(ctx, name, precondition, func(w *ateapipb.Worker) error { - w.Epoch = 2 - return nil - }); err != nil { - t.Fatalf("UpdateWorker: %v", err) - } - w := &ActorWorkflow{store: persistence, workerCache: wc, scheduler: scheduling.New(wc)} tmpl := &ateapipb.ActorTemplate{} stored, _, err := w.assignWorkerAttempt(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, tmpl) if err != nil { t.Fatalf("assignWorkerAttempt: %v", err) } - if got := stored.GetStatus().GetWorkerAssignment().GetWorkerEpoch(); got != 2 { - t.Errorf("actor worker_epoch = %d, want 2", got) - } - assignment, err := persistence.GetWorkerAssignment(ctx, name, actor.GetMetadata().GetUid()) - if err != nil { - t.Fatalf("GetWorkerAssignment: %v", err) - } - if got := assignment.GetWorkerEpoch(); got != 2 { - t.Errorf("assignment worker_epoch = %d, want 2", got) + if got := stored.GetStatus().GetWorkerAssignment().GetWorkerEpoch(); got != 0 { + t.Errorf("actor worker_epoch = %d, want 0 until the Actor is RUNNING", got) } } diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index 65f8e1f57b..85ba36756c 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -118,11 +118,12 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, actorRef resources.Acto if err != nil { return nil, false, err } - if tele, err = w.ensureAteletRestored(leaseCtx, actorRef, actor, actorTemplate, volumePublishContexts); err != nil { + var workerEpoch int64 + if tele, workerEpoch, err = w.ensureAteletRestored(leaseCtx, actorRef, actor, actorTemplate, volumePublishContexts); err != nil { return nil, false, err } var running *ateapipb.Actor - if running, err = w.finalizeRunning(leaseCtx, actorRef); err != nil { + if running, err = w.finalizeRunning(leaseCtx, actorRef, workerEpoch); err != nil { return nil, false, err } actor = running @@ -370,10 +371,14 @@ func (w *ActorWorkflow) validateAssignedWorker(ctx context.Context, actorRef res return nil, apierror.Aborted("actor %s crashed", actorRef.String()) } // An unavailable worker may come back with its Actors intact. If its worker - // restarted instead, the raised epoch has this Actor crashed for it. + // process restarted instead, a retried resume restores this Actor on the + // new process. if worker.GetStatus().GetState() == ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE { return nil, apierror.Unavailable("worker %s assigned to actor %s is unavailable", worker.GetMetadata().GetName(), actorRef) } + if !scheduling.Registered(worker) { + return nil, apierror.Unavailable("worker %s assigned to actor %s has not finished registering", worker.GetMetadata().GetName(), actorRef) + } // Verify the worker is still hosting this Actor. hosted, err := workerHostsActor(ctx, w.store, worker.GetMetadata().GetName(), actor.GetMetadata().GetUid()) if err != nil { @@ -541,9 +546,6 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour } newAssignment := workerAssignmentFrom(bound, snapRuntime) - // The cached Worker may predate a raised epoch; the bind read it under the - // Worker's row lock. - newAssignment.WorkerEpoch = assignment.GetWorkerEpoch() storedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_RESUMING toUpdate.Status.WorkerAssignment = newAssignment @@ -592,7 +594,7 @@ func workerAssignmentFrom(w *ateapipb.Worker, snapRuntime hardware.Requirement) WorkerPool: w.GetWorkerPool(), WorkerPod: w.GetWorkerPod(), WorkerPodUid: w.GetWorkerPodUid(), - WorkerPodIps: slices.Clone(w.GetIps()), + WorkerPodIps: slices.Clone(w.GetStatus().GetIps()), NodeName: w.GetNodeName(), WorkerSpiffeId: w.GetWorkerSpiffeId(), // The runtime the actor is started or restored under. Snapshots taken @@ -755,25 +757,25 @@ func checkVolumesAttachable(volumes []*ateapipb.ExternalVolumeStatus) error { // the worker pod UID, so a re-entered workflow re-sends the same semantic // request; once atelet's Restore/Run are idempotent on those keys this step // becomes fully reentrant with no changes here. -func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate, volumePublishContexts map[string]map[string]string) (tele restoreTelemetry, err error) { +func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate, volumePublishContexts map[string]map[string]string) (tele restoreTelemetry, workerEpoch int64, err error) { ctx, done := stepSpan(ctx, "CallAteletRestore") defer func() { err = done(err) }() // A re-entered resume skips assignWorkerAttempt, which checks this too. if err := checkSnapshotURI(actorRef, actor); err != nil { - return tele, err + return tele, 0, err } assignment := actor.GetStatus().GetWorkerAssignment() ateletConn, err := w.dialer.DialForAteletOnNode(assignment.GetNodeName()) if err != nil { - return tele, err + return tele, 0, err } client := ateletpb.NewAteletClient(ateletConn) workloadSpec, err := workloadSpecFromActorTemplate(actorTemplate, actor, volumePublishContexts) if err != nil { - return tele, err + return tele, 0, err } egressGateway := w.egressGateway() @@ -804,10 +806,11 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou EgressGateway: egressGateway, Resources: actorResources, } - if _, err = client.Run(ctx, req); err != nil { - return tele, handleAteletError(ctx, w.store, actorRef, ateattr.OperationResume, "Run", false, err) + resp, err := client.Run(ctx, req) + if err != nil { + return tele, 0, handleAteletError(ctx, w.store, actorRef, ateattr.OperationResume, "Run", false, err) } - return tele, nil + return tele, resp.GetWorkerEpoch(), nil } slog.InfoContext(ctx, "Actor has snapshot; Restoring from snapshot", slog.String("snapshot_uuid", snap.GetUuid())) @@ -857,10 +860,11 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou EgressGateway: egressGateway, Resources: actorResources, } - if _, err = client.Restore(ctx, req); err != nil { - return tele, handleAteletError(ctx, w.store, actorRef, ateattr.OperationResume, "Restore", false, err) + resp, err := client.Restore(ctx, req) + if err != nil { + return tele, 0, handleAteletError(ctx, w.store, actorRef, ateattr.OperationResume, "Restore", false, err) } - return tele, nil + return tele, resp.GetWorkerEpoch(), nil } // logRuntimeFallback records a resume that restores without memory state @@ -881,8 +885,16 @@ func (w *ActorWorkflow) egressGateway() *ateletpb.EgressGateway { return &ateletpb.EgressGateway{Address: w.egressGatewayAddress} } -// finalizeRunning re-reads the actor for a fresh version and commits RUNNING. -func (w *ActorWorkflow) finalizeRunning(ctx context.Context, actorRef resources.ActorRef) (_ *ateapipb.Actor, err error) { +// finalizeRunning re-reads the actor for a fresh version and commits RUNNING, +// stamped with workerEpoch, the epoch of the worker process that ran it. +// +// The worker process registered that epoch, with its pod IPs, before it took +// the workload. If it is still the Worker's registered epoch, the Actor is +// stamped with it and with those IPs, which a resume bound before the process +// started did not carry. If a later epoch is registered, that process has exited and taken +// the sandbox with it, so RUNNING is not committed: the error is Unavailable, +// and the retried resume restores the Actor on the current process. +func (w *ActorWorkflow) finalizeRunning(ctx context.Context, actorRef resources.ActorRef, workerEpoch int64) (_ *ateapipb.Actor, err error) { ctx, done := stepSpan(ctx, "FinalizeRunning") defer func() { err = done(err) }() @@ -891,8 +903,30 @@ func (w *ActorWorkflow) finalizeRunning(ctx context.Context, actorRef resources. return nil, err } + workerName := latestActor.GetStatus().GetWorkerAssignment().GetWorker().GetName() + worker, err := w.store.GetWorker(ctx, workerName) + if errors.Is(err, store.ErrNotFound) { + // Deleting the Worker crashes the Actors bound to it. + return nil, apierror.Aborted("worker %s of actor %s is gone", workerName, actorRef) + } + if err != nil { + return nil, fmt.Errorf("while getting worker %s: %w", workerName, err) + } + switch registered := worker.GetStatus().GetRegisteredEpoch(); { + case workerEpoch < registered: + slog.WarnContext(ctx, "Worker process that ran the actor has been replaced; restoring it again", + slog.String("worker", workerName), + slog.Int64("worker_epoch", workerEpoch), + slog.Int64("registered_epoch", registered)) + return nil, apierror.Unavailable("worker %s restarted while actor %s was being restored, please retry", workerName, actorRef) + case workerEpoch > registered: + return nil, apierror.Internal("worker %s reported epoch %d, above its registered epoch %d", workerName, workerEpoch, registered) + } + storedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(latestActor), func(toUpdate *ateapipb.Actor) error { toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_RUNNING + toUpdate.Status.WorkerAssignment.WorkerEpoch = workerEpoch + toUpdate.Status.WorkerAssignment.WorkerPodIps = slices.Clone(worker.GetStatus().GetIps()) return nil }) if err != nil { diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index efd45a0fd5..2a872e441f 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -115,13 +115,27 @@ func TestResumeActor_RunningFastPathDoesNotAcquireLease(t *testing.T) { } } -// TestFinalizeRunning_CommitsRunning verifies the last step of a resume moves -// the actor out of RESUMING against a freshly read version, and keeps the -// locality of the local snapshot it restored from: resume does not delete the -// snapshot's files from the node. -func TestFinalizeRunning_CommitsRunning(t *testing.T) { - ctx := context.Background() - persistence := newTestPersistence(t) +// seedFinalizeActor stores a RESUMING actor bound to a Worker whose registered +// epoch is registered and whose registered IPs are newIPs. The actor's +// assignment still carries epoch 1 and the IPs it was bound with. +func seedFinalizeActor(t *testing.T, ctx context.Context, persistence store.Interface, registered int64, newIPs []string) { + t.Helper() + workerName := testWorkerUID("pod-1") + if _, err := persistence.CreateWorker(ctx, &ateapipb.Worker{ + Metadata: &ateapipb.ResourceMetadata{Name: workerName}, + WorkerNamespace: "worker-ns", + WorkerPool: "pool", + WorkerPod: "pod-1", + WorkerPodUid: workerName, + NodeName: "node-1", + Status: &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + RegisteredEpoch: registered, + Ips: newIPs, + }, + }); err != nil { + t.Fatalf("CreateWorker: %v", err) + } storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "id1"}, ActorTemplate: &ateapipb.ObjectRef{Atespace: "team-a", Name: "tmpl-2"}, @@ -129,14 +143,29 @@ func TestFinalizeRunning_CommitsRunning(t *testing.T) { State: ateapipb.ActorState_ACTOR_STATE_RESUMING, AssignedNode: "node-1", LastAssignedGeneration: 1, + WorkerAssignment: &ateapipb.WorkerAssignment{ + Worker: &ateapipb.ObjectRef{Name: workerName}, + WorkerPodIps: []string{"10.0.0.1"}, + WorkerEpoch: 1, + }, Snapshots: []*ateapipb.Snapshot{ newPausedSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-1", "node-1"), }, }, }) +} + +// TestFinalizeRunning_CommitsRunning verifies the last step of a resume moves +// the actor out of RESUMING against a freshly read version, and keeps the +// locality of the local snapshot it restored from: resume does not delete the +// snapshot's files from the node. +func TestFinalizeRunning_CommitsRunning(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + seedFinalizeActor(t, ctx, persistence, 1, []string{"10.0.0.1"}) w := &ActorWorkflow{store: persistence} - got, err := w.finalizeRunning(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}) + got, err := w.finalizeRunning(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, 1) if err != nil { t.Fatalf("finalizeRunning: %v", err) } @@ -148,6 +177,60 @@ func TestFinalizeRunning_CommitsRunning(t *testing.T) { } } +// A resume bound before the worker restarted, whose restore ran on the new +// worker process, is stamped with that process's epoch and the IPs it +// registered, so the reconciler keeps it and traffic reaches it. +func TestFinalizeRunning_StampsReportedEpochAndIPs(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + seedFinalizeActor(t, ctx, persistence, 2, []string{"10.0.0.2"}) + w := &ActorWorkflow{store: persistence} + + got, err := w.finalizeRunning(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, 2) + if err != nil { + t.Fatalf("finalizeRunning: %v", err) + } + assignment := got.GetStatus().GetWorkerAssignment() + if assignment.GetWorkerEpoch() != 2 { + t.Errorf("worker_epoch = %d, want 2", assignment.GetWorkerEpoch()) + } + if diff := cmp.Diff([]string{"10.0.0.2"}, assignment.GetWorkerPodIps()); diff != "" { + t.Errorf("worker_pod_ips mismatch (-want +got):\n%s", diff) + } +} + +// A restore that ran on a worker process that has since been replaced left no +// sandbox behind, so RUNNING is not committed and the resume is retried. +func TestFinalizeRunning_ReplacedWorkerProcess(t *testing.T) { + for _, tc := range []struct { + name string + reported int64 + want codes.Code + }{ + {name: "replaced process", reported: 1, want: codes.Unavailable}, + {name: "epoch above the active one", reported: 3, want: codes.Internal}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + seedFinalizeActor(t, ctx, persistence, 2, []string{"10.0.0.2"}) + w := &ActorWorkflow{store: persistence} + + _, err := w.finalizeRunning(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, tc.reported) + if got := apierror.Code(err); got != tc.want { + t.Fatalf("finalizeRunning() = %v (err %v), want %v", got, err, tc.want) + } + actor, err := persistence.GetActor(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}) + if err != nil { + t.Fatalf("GetActor: %v", err) + } + if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RESUMING { + t.Errorf("state = %v, want RESUMING", got) + } + }) + } +} + // bindErrorStore fails every claim, standing in for a worker that moved or // vanished between the pick and the write. type bindErrorStore struct { @@ -206,11 +289,11 @@ func TestAssignWorkerAttempt_SkipsWorkerAssignedInOtherAtespace(t *testing.T) { WorkerPool: "pool", WorkerPod: "pod-1", WorkerPodUid: testWorkerUID("pod-1"), - Status: &ateapipb.WorkerStatus{ + Status: registered(&ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, SandboxClasses: workerSandboxClasses(testSandboxRuntime()), - }, + }), } if _, err := persistence.CreateWorker(ctx, worker); err != nil { t.Fatalf("CreateWorker: %v", err) @@ -330,11 +413,11 @@ func TestAssignWorkerAttempt_ReleasesIneligibleStaleWorker(t *testing.T) { WorkerPod: "stale-pod", WorkerPodUid: testWorkerUID("stale-pod"), NodeName: "node-1", - Status: &ateapipb.WorkerStatus{ + Status: registered(&ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, SandboxClasses: workerSandboxClasses(testSandboxRuntime()), - }, + }), } free := &ateapipb.Worker{ Metadata: &ateapipb.ResourceMetadata{Name: testWorkerUID("free-pod")}, @@ -344,11 +427,11 @@ func TestAssignWorkerAttempt_ReleasesIneligibleStaleWorker(t *testing.T) { WorkerPodUid: testWorkerUID("free-pod"), NodeName: "node-1", Labels: map[string]string{"pool": "b"}, - Status: &ateapipb.WorkerStatus{ + Status: registered(&ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, SandboxClasses: workerSandboxClasses(testSandboxRuntime()), - }, + }), } for _, w := range []*ateapipb.Worker{stale, free} { if _, err := persistence.CreateWorker(ctx, w); err != nil { @@ -404,11 +487,11 @@ func TestAssignWorkerAttempt_RetryAfterConflictPicksFreshWorker(t *testing.T) { WorkerPod: "contested-pod", WorkerPodUid: testWorkerUID("contested-pod"), NodeName: "node-1", - Status: &ateapipb.WorkerStatus{ + Status: registered(&ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, SandboxClasses: workerSandboxClasses(wantRuntime), - }, + }), } fallback := &ateapipb.Worker{ Metadata: &ateapipb.ResourceMetadata{Name: testWorkerUID("fallback-pod")}, @@ -417,11 +500,11 @@ func TestAssignWorkerAttempt_RetryAfterConflictPicksFreshWorker(t *testing.T) { WorkerPod: "fallback-pod", WorkerPodUid: testWorkerUID("fallback-pod"), NodeName: "node-1", - Status: &ateapipb.WorkerStatus{ + Status: registered(&ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, SandboxClasses: workerSandboxClasses(wantRuntime), - }, + }), } for _, w := range []*ateapipb.Worker{contested, fallback} { if _, err := persistence.CreateWorker(ctx, w); err != nil { @@ -512,11 +595,11 @@ func seedAssignFixture(t *testing.T, ctx context.Context, persistence store.Inte WorkerPod: "pod-1", WorkerPodUid: testWorkerUID("pod-1"), NodeName: "node-1", - Status: &ateapipb.WorkerStatus{ + Status: registered(&ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, SandboxClasses: workerSandboxClasses(testSandboxRuntime()), - }, + }), }); err != nil { t.Fatalf("CreateWorker: %v", err) } @@ -821,11 +904,11 @@ func TestValidateAssignedWorker(t *testing.T) { Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "shared"}, ActorUid: "stale-incarnation-uid", } - activeStatus := &ateapipb.WorkerStatus{ + activeStatus := registered(&ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, SandboxClasses: workerSandboxClasses(testSandboxRuntime()), - } + }) drainingStatus := &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_DRAINING, Capacity: &ateapipb.WorkerResources{Actors: 1}, SandboxClasses: workerSandboxClasses(testSandboxRuntime())} unavailableStatus := &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE, Capacity: &ateapipb.WorkerResources{Actors: 1}, SandboxClasses: workerSandboxClasses(testSandboxRuntime())} @@ -870,6 +953,20 @@ func TestValidateAssignedWorker(t *testing.T) { wantActorState: ateapipb.ActorState_ACTOR_STATE_RESUMING, wantAssignment: ownAssignment, }, + { + // A restarted worker process between registering and reporting + // its runtimes takes Actors again shortly. + name: "leaves actor and worker untouched while the worker process finishes registering", + workerStatus: &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + RegisteredEpoch: 2, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + }, + assignment: ownAssignment, + wantCode: codes.Unavailable, + wantActorState: ateapipb.ActorState_ACTOR_STATE_RESUMING, + wantAssignment: ownAssignment, + }, { name: "crashes actor and leaves worker untouched when assigned to another actor", workerStatus: activeStatus, @@ -1060,14 +1157,14 @@ func (f *capturingAtelet) Restore(ctx context.Context, req *ateletpb.RestoreRequ f.mu.Lock() defer f.mu.Unlock() f.restore = proto.Clone(req).(*ateletpb.RestoreRequest) - return &ateletpb.RestoreResponse{}, nil + return &ateletpb.RestoreResponse{WorkerEpoch: registeredEpoch}, nil } func (f *capturingAtelet) Run(ctx context.Context, req *ateletpb.RunRequest) (*ateletpb.RunResponse, error) { f.mu.Lock() defer f.mu.Unlock() f.run = proto.Clone(req).(*ateletpb.RunRequest) - return &ateletpb.RunResponse{}, nil + return &ateletpb.RunResponse{WorkerEpoch: registeredEpoch}, nil } func (f *capturingAtelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest) (*ateletpb.CheckpointResponse, error) { @@ -1870,7 +1967,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { actor, loadedTmpl, err := w.loadActorForResume(ctx, actorRef) var tele restoreTelemetry if err == nil { - tele, err = w.ensureAteletRestored(ctx, actorRef, actor, loadedTmpl, nil) + tele, _, err = w.ensureAteletRestored(ctx, actorRef, actor, loadedTmpl, nil) } if got := apierror.Code(err); got != tt.want.code { t.Fatalf("apierror.Code(err) = %v, want %v (err: %v)", got, tt.want.code, err) @@ -2216,11 +2313,11 @@ func contendedWorker(podName string) *ateapipb.Worker { WorkerPod: podName, WorkerPodUid: testWorkerUID(podName), NodeName: "node-1", - Status: &ateapipb.WorkerStatus{ + Status: registered(&ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, SandboxClasses: workerSandboxClasses(testSandboxRuntime()), - }, + }), } } @@ -2535,9 +2632,10 @@ func seedRuntimeWorker(t *testing.T, ctx context.Context, st store.Interface, po WorkerPodUid: testWorkerUID(pod), NodeName: node, Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - Capacity: &ateapipb.WorkerResources{Actors: 1}, - SandboxClasses: workerSandboxClasses(def, restorable...), + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + RegisteredEpoch: registeredEpoch, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + SandboxClasses: workerSandboxClasses(def, restorable...), }, }) if err != nil { @@ -2567,8 +2665,9 @@ func seedMicroVMWorker(t *testing.T, ctx context.Context, st store.Interface, po WorkerPodUid: testWorkerUID(pod), NodeName: node, Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - Capacity: &ateapipb.WorkerResources{Actors: 1}, + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + RegisteredEpoch: registeredEpoch, + Capacity: &ateapipb.WorkerResources{Actors: 1}, SandboxClasses: []*ateapipb.WorkerSandboxClass{{ SandboxClass: "microvm", DefaultRuntime: &ateapipb.WorkerSandboxRuntime{Version: &ateapipb.VersionedSandboxCompat{SchemaVersion: "v1", Attributes: []*ateapipb.AttributeEntry{ @@ -3052,11 +3151,11 @@ func seedVolumeAssignFixture(t *testing.T, ctx context.Context, persistence stor WorkerPod: "pod-1", WorkerPodUid: testWorkerUID("pod-1"), NodeName: node, - Status: &ateapipb.WorkerStatus{ + Status: registered(&ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, SandboxClasses: workerSandboxClasses(testSandboxRuntime()), - }, + }), }); err != nil { t.Fatalf("CreateWorker: %v", err) } diff --git a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go index 2476323ef9..a4e4437d58 100644 --- a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go @@ -284,6 +284,28 @@ func seedSuspendTemplate(t *testing.T, ctx context.Context, st store.Interface, return stored } +// registeredEpoch is the epoch registered sets, and the one the fake atelets +// report their workloads ran under. +const registeredEpoch = 1 + +// registered fills in what a Worker's worker process reports when it +// registers, which the scheduler requires: a registered epoch and, unless +// status names them, sandbox classes with a default runtime no snapshot +// requires. +func registered(status *ateapipb.WorkerStatus) *ateapipb.WorkerStatus { + status.RegisteredEpoch = max(status.RegisteredEpoch, registeredEpoch) + if len(status.SandboxClasses) == 0 { + status.SandboxClasses = workerSandboxClasses(&ateapipb.ActorSandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{{Key: "architecture", Value: "fixture"}}, + }, + }) + } + return status +} + // seedWireWorker stores the Worker wireTestAssignment names and places // assignment on it (none when nil), so a finalize step has a real assignment // to release. diff --git a/cmd/ateapi/internal/scheduling/scheduling.go b/cmd/ateapi/internal/scheduling/scheduling.go index 6cbfdccbc2..bdeaba6ed8 100644 --- a/cmd/ateapi/internal/scheduling/scheduling.go +++ b/cmd/ateapi/internal/scheduling/scheduling.go @@ -212,15 +212,19 @@ func utilization(capacity, used *ateapipb.WorkerResources) float64 { return maxRatio } +// Registered reports whether worker's current worker process has registered +// and reported its sandbox class and runtimes. The assignment records the +// runtime and atelet requires it on Run/Restore, so a worker without one +// cannot host anything yet. +func Registered(worker *ateapipb.Worker) bool { + return worker.GetStatus().GetRegisteredEpoch() != 0 && hardware.SandboxClass(worker.GetStatus()) != "" +} + func (s *scheduler) Applies(worker *ateapipb.Worker, constraints Constraints) bool { - // The class is the one the worker registered with WorkerService. Until - // that report lands the Worker has no class, and nothing is placed on it - // (see ServiceImpl.CreateWorker): the assignment records the runtime, - // and atelet requires it on Run and Restore. - workerClass := hardware.SandboxClass(worker.GetStatus()) - if workerClass == "" { + if !Registered(worker) { return false } + workerClass := hardware.SandboxClass(worker.GetStatus()) if constraints.SandboxClass != "" && workerClass != constraints.SandboxClass { return false } diff --git a/cmd/ateapi/internal/scheduling/scheduling_test.go b/cmd/ateapi/internal/scheduling/scheduling_test.go index e04084253c..efa9436a94 100644 --- a/cmd/ateapi/internal/scheduling/scheduling_test.go +++ b/cmd/ateapi/internal/scheduling/scheduling_test.go @@ -366,6 +366,26 @@ func TestSchedule(t *testing.T) { } } +func TestRegistered(t *testing.T) { + tests := []struct { + name string + mutate func(*ateapipb.Worker) + want bool + }{ + {name: "registered with runtimes", mutate: func(*ateapipb.Worker) {}, want: true}, + {name: "no registered epoch", mutate: func(w *ateapipb.Worker) { w.Status.RegisteredEpoch = 0 }}, + {name: "no sandbox class", mutate: func(w *ateapipb.Worker) { w.Status.SandboxClasses = nil }}, + {name: "no status", mutate: func(w *ateapipb.Worker) { w.Status = nil }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := Registered(worker("w", "gvisor", "node-a", nil, tc.mutate)); got != tc.want { + t.Errorf("Registered() = %v, want %v", got, tc.want) + } + }) + } +} + func TestApplies(t *testing.T) { tierSel := labels.SelectorFromSet(labels.Set{"tier": "1"}) workloadSel := labels.SelectorFromSet(labels.Set{"workload": "a"}) @@ -395,6 +415,14 @@ func TestApplies(t *testing.T) { constraints: Constraints{}, want: false, }, + { + name: "worker process not registered", + worker: worker("w", "gvisor", "node-a", nil, func(w *ateapipb.Worker) { + w.Status.RegisteredEpoch = 0 + }), + constraints: Constraints{}, + want: false, + }, { name: "snapshot class pins the worker class", worker: worker("w", "microvm", "node-a", nil), @@ -513,8 +541,9 @@ func worker(pod, class, node string, lbls map[string]string, opts ...func(*ateap Labels: lbls, // A stored Worker always carries a ceiling; CreateWorker reifies one. Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - Capacity: &ateapipb.WorkerResources{Actors: 1}, + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + RegisteredEpoch: 1, + Capacity: &ateapipb.WorkerResources{Actors: 1}, }, } if class != "" { diff --git a/cmd/ateapi/internal/store/atepg/outbox_test.go b/cmd/ateapi/internal/store/atepg/outbox_test.go index 9bbcee22a6..e700131ae7 100644 --- a/cmd/ateapi/internal/store/atepg/outbox_test.go +++ b/cmd/ateapi/internal/store/atepg/outbox_test.go @@ -21,7 +21,6 @@ package atepg import ( "context" "fmt" - "slices" "strings" "sync" "testing" @@ -1092,7 +1091,7 @@ func TestLocalPublishReachesWatchers(t *testing.T) { } updated, err := p.UpdateWorker(ctx, created.GetMetadata().GetName(), store.PreconditionFrom(created), func(toUpdate *ateapipb.Worker) error { - toUpdate.Ips = []string{"10.0.0.9"} + toUpdate.Labels = map[string]string{"tier": "batch"} return nil }) if err != nil { @@ -1102,8 +1101,8 @@ func TestLocalPublishReachesWatchers(t *testing.T) { if got, want := ev.Worker.GetMetadata().GetVersion(), updated.GetMetadata().GetVersion(); got != want { t.Errorf("updated event version = %d, want committed version %d", got, want) } - if !slices.Equal(ev.Worker.GetIps(), []string{"10.0.0.9"}) { - t.Errorf("updated event carries Ips %q, want the committed mutation", ev.Worker.GetIps()) + if got := ev.Worker.GetLabels()["tier"]; got != "batch" { + t.Errorf("updated event carries label tier=%q, want the committed mutation", got) } // The watcher's copy must be isolated from the caller's returned Worker. if ev.Worker == updated { diff --git a/cmd/ateapi/internal/store/atepg/worker_assignment.go b/cmd/ateapi/internal/store/atepg/worker_assignment.go index 9439982d18..2ddebcddf8 100644 --- a/cmd/ateapi/internal/store/atepg/worker_assignment.go +++ b/cmd/ateapi/internal/store/atepg/worker_assignment.go @@ -89,9 +89,6 @@ func (p *Persistence) BindActorToWorker(ctx context.Context, workerName string, worker.Status = &ateapipb.WorkerStatus{} } - // Read under the row lock, so an epoch raised concurrently is either - // seen here or raised after this bind commits. - assignment.WorkerEpoch = worker.GetEpoch() assignmentBytes, err := proto.Marshal(assignment) if err != nil { return nil, fmt.Errorf("marshaling assignment: %w", err) diff --git a/cmd/ateapi/internal/store/atepg/worker_assignment_test.go b/cmd/ateapi/internal/store/atepg/worker_assignment_test.go index ae3ea6dec2..90739baf1a 100644 --- a/cmd/ateapi/internal/store/atepg/worker_assignment_test.go +++ b/cmd/ateapi/internal/store/atepg/worker_assignment_test.go @@ -50,7 +50,7 @@ func TestSaveWorker_RejectsAStaleWrite(t *testing.T) { // Move the stored Worker on, so the copy above is a version behind. if _, err := p.UpdateWorker(ctx, created.GetMetadata().GetName(), store.PreconditionFrom(created), func(toUpdate *ateapipb.Worker) error { - toUpdate.Ips = []string{"10.0.0.1"} + toUpdate.Labels = map[string]string{"tier": "batch"} return nil }); err != nil { t.Fatalf("UpdateWorker failed: %v", err) diff --git a/cmd/ateapi/internal/workerservice/register.go b/cmd/ateapi/internal/workerservice/register.go index a5318bb5fc..fd58852eeb 100644 --- a/cmd/ateapi/internal/workerservice/register.go +++ b/cmd/ateapi/internal/workerservice/register.go @@ -32,9 +32,40 @@ import ( "google.golang.org/protobuf/proto" ) -// RegisterWorker records a Worker's reported capacity and sandbox classes in -// one write. As with MintCert, the caller must be an atelet running on the -// Worker's node. +// AcquireWorkerEpoch hands out the next epoch of a Worker. As with MintCert, +// the caller must be an atelet running on the Worker's node. +func (s *Server) AcquireWorkerEpoch(ctx context.Context, req *ateapipb.AcquireWorkerEpochRequest) (*ateapipb.AcquireWorkerEpochResponse, error) { + caller, err := ateletauth.Authenticate(ctx, s.ateletSPIFFEID) + if err != nil { + return nil, err + } + if errs := apivalidation.ValidateAcquireWorkerEpochRequest(ctx, req); len(errs) > 0 { + return nil, resources.ToAPIError(errs) + } + name := req.GetWorker().GetName() + worker, err := s.workerOnCallerNode(ctx, name, caller) + if err != nil { + return nil, err + } + + updated, err := s.store.UpdateWorker(ctx, name, store.PreconditionFrom(worker), func(toUpdate *ateapipb.Worker) error { + toUpdate.Status.IssuedEpoch++ + return nil + }) + if err != nil { + return nil, workerWriteError(name, "acquiring an epoch", err) + } + epoch := updated.GetStatus().GetIssuedEpoch() + slog.InfoContext(ctx, "Worker acquired an epoch", + slog.String("worker", name), + slog.Int64("epoch", epoch), + slog.Int64("registered_epoch", updated.GetStatus().GetRegisteredEpoch())) + return &ateapipb.AcquireWorkerEpochResponse{Epoch: epoch}, nil +} + +// RegisterWorker activates the epoch a worker process acquired and records its +// pod IPs and capacity in one write. As with MintCert, the caller must be an +// atelet running on the Worker's node. func (s *Server) RegisterWorker(ctx context.Context, req *ateapipb.RegisterWorkerRequest) (*ateapipb.RegisterWorkerResponse, error) { // TODO(identity): This check should be handled by OpenFGA. caller, err := ateletauth.Authenticate(ctx, s.ateletSPIFFEID) @@ -44,7 +75,65 @@ func (s *Server) RegisterWorker(ctx context.Context, req *ateapipb.RegisterWorke if errs := apivalidation.ValidateRegisterWorkerRequest(ctx, req); len(errs) > 0 { return nil, resources.ToAPIError(errs) } - reported := req.GetCapacity() + name := req.GetWorker().GetName() + worker, err := s.workerOnCallerNode(ctx, name, caller) + if err != nil { + return nil, err + } + + epoch := req.GetEpoch() + status := worker.GetStatus() + switch registered := status.GetRegisteredEpoch(); { + case epoch > status.GetIssuedEpoch(): + return nil, apierror.FailedPrecondition("epoch %d was never handed out for Worker %s", epoch, name) + case epoch < registered: + return nil, apierror.FailedPrecondition("epoch %d of Worker %s is superseded by epoch %d", epoch, name, registered) + case epoch == registered: + // A worker process registers once, so the registered epoch's registration + // does not change: an identical one is a retry of the registration + // that landed, and any other is refused. + if slices.Equal(status.GetIps(), req.GetIps()) && proto.Equal(status.GetCapacity(), req.GetCapacity()) { + return &ateapipb.RegisterWorkerResponse{Worker: worker}, nil + } + return nil, apierror.FailedPrecondition("epoch %d of Worker %s is already registered with other IPs or capacity", epoch, name) + } + + // The precondition holds this write to the Worker read above, so epoch is + // above its registered epoch. + updated, err := s.store.UpdateWorker(ctx, name, store.PreconditionFrom(worker), func(toUpdate *ateapipb.Worker) error { + toUpdate.Status.RegisteredEpoch = epoch + // The sandbox classes were the previous worker process's. This one + // reports its own, and the Worker takes no Actors until it does. + toUpdate.Status.SandboxClasses = nil + toUpdate.Status.Ips = slices.Clone(req.GetIps()) + // Replaces rather than merges: a Worker reports everything it has, so a + // dimension this report leaves out is one it no longer supplies. + toUpdate.Status.Capacity = req.GetCapacity() + return nil + }) + if err != nil { + return nil, workerWriteError(name, "registering", err) + } + slog.InfoContext(ctx, "Worker registered", + slog.String("worker", name), + slog.Int64("epoch", epoch), + slog.Int64("previous_epoch", status.GetRegisteredEpoch()), + slog.Any("ips", req.GetIps()), + slog.String("capacity", req.GetCapacity().String())) + return &ateapipb.RegisterWorkerResponse{Worker: updated}, nil +} + +// SetWorkerSandboxRuntimes records the sandbox classes and runtimes of the +// registered worker process. As with MintCert, the caller must be an atelet +// running on the Worker's node. +func (s *Server) SetWorkerSandboxRuntimes(ctx context.Context, req *ateapipb.SetWorkerSandboxRuntimesRequest) (*ateapipb.SetWorkerSandboxRuntimesResponse, error) { + caller, err := ateletauth.Authenticate(ctx, s.ateletSPIFFEID) + if err != nil { + return nil, err + } + if errs := apivalidation.ValidateSetWorkerSandboxRuntimesRequest(ctx, req); len(errs) > 0 { + return nil, resources.ToAPIError(errs) + } classes := req.GetSandboxClasses() name := req.GetWorker().GetName() for _, c := range classes { @@ -54,7 +143,39 @@ func (s *Server) RegisterWorker(ctx context.Context, req *ateapipb.RegisterWorke } } - // Use authoritative state to authorize the write. + worker, err := s.workerOnCallerNode(ctx, name, caller) + if err != nil { + return nil, err + } + status := worker.GetStatus() + if registered := status.GetRegisteredEpoch(); req.GetEpoch() != registered { + return nil, apierror.FailedPrecondition("epoch %d is not the registered epoch %d of Worker %s", req.GetEpoch(), registered, name) + } + if err := validateSandboxClasses(hardware.SandboxClass(status), classes); err != nil { + return nil, err + } + if sameClasses(status.GetSandboxClasses(), classes) { + return &ateapipb.SetWorkerSandboxRuntimesResponse{Worker: worker}, nil + } + + updated, err := s.store.UpdateWorker(ctx, name, store.PreconditionFrom(worker), func(toUpdate *ateapipb.Worker) error { + // Replaces rather than merges: a Worker reports every runtime it has. + toUpdate.Status.SandboxClasses = classes + return nil + }) + if err != nil { + return nil, workerWriteError(name, "recording sandbox runtimes", err) + } + slog.InfoContext(ctx, "Worker reported its sandbox runtimes", + slog.String("worker", name), + slog.Int64("epoch", req.GetEpoch()), + slog.String("sandbox_classes", fmt.Sprint(updated.GetStatus().GetSandboxClasses()))) + return &ateapipb.SetWorkerSandboxRuntimesResponse{Worker: updated}, nil +} + +// workerOnCallerNode reads the named Worker, authorized by authoritative state: +// a Worker on another node than the calling atelet's is reported as not found. +func (s *Server) workerOnCallerNode(ctx context.Context, name string, caller *ateletauth.Caller) (*ateapipb.Worker, error) { worker, err := s.store.GetWorker(ctx, name) if err != nil { if errors.Is(err, store.ErrNotFound) { @@ -64,52 +185,39 @@ func (s *Server) RegisterWorker(ctx context.Context, req *ateapipb.RegisterWorke } if worker.GetNodeName() != caller.NodeName { // Do not disclose Workers on other nodes. - slog.WarnContext(ctx, "Refusing a capacity report for a worker on another node", + slog.WarnContext(ctx, "Refusing a report for a worker on another node", slog.String("worker", name), slog.String("worker_node", worker.GetNodeName()), slog.String("caller_node", caller.NodeName), slog.String("caller_pod", caller.PodName)) return nil, apierror.NotFound("Worker %s not found", name) } - status := worker.GetStatus() - if err := validateSandboxClasses(hardware.SandboxClass(status), classes); err != nil { - return nil, err - } - - if proto.Equal(status.GetCapacity(), reported) && sameClasses(status.GetSandboxClasses(), classes) { - return &ateapipb.RegisterWorkerResponse{Worker: worker}, nil + if worker.GetStatus() == nil { + worker.Status = &ateapipb.WorkerStatus{} } + return worker, nil +} - updated, err := s.store.UpdateWorker(ctx, name, store.PreconditionFrom(worker), func(toUpdate *ateapipb.Worker) error { - // Replaces rather than merges: a Worker reports everything it has, so a - // dimension or runtime this report leaves out is one it no longer - // supplies. - toUpdate.Status.Capacity = reported - toUpdate.Status.SandboxClasses = classes - return nil - }) +// workerWriteError maps a failed write to a Worker to the error its caller +// retries on: a concurrent write is Aborted, which the caller resends after +// reading the Worker again. +func workerWriteError(name, doing string, err error) error { switch { - case err == nil: case errors.Is(err, store.ErrNotFound): - return nil, apierror.NotFound("Worker %s not found", name) + return apierror.NotFound("Worker %s not found", name) case errors.Is(err, store.ErrUIDConflict), errors.Is(err, store.ErrVersionConflict): - return nil, apierror.Aborted("concurrent update conflict, please retry") + return apierror.Aborted("concurrent update conflict, please retry") default: - return nil, fmt.Errorf("while recording capacity for worker %s: %w", name, err) + return fmt.Errorf("while %s for worker %s: %w", doing, name, err) } - slog.InfoContext(ctx, "Worker registered its capacity and sandbox classes", - slog.String("worker", name), - slog.String("was", worker.GetStatus().GetCapacity().String()), - slog.String("now", updated.GetStatus().GetCapacity().String()), - slog.String("sandbox_classes", fmt.Sprint(updated.GetStatus().GetSandboxClasses()))) - return &ateapipb.RegisterWorkerResponse{Worker: updated}, nil } // validateSandboxClasses rejects an entry of a class other than the one // already recorded on the Worker. A worker pod runs one sandbox class for its -// lifetime, fixed by its first registration, so a later report of another -// class contradicts the recorded state. A Worker that has not registered yet -// has nothing for a report to contradict. +// lifetime, and the first report of each worker process records it, so a +// later report of another class contradicts the recorded state. A Worker +// whose worker process has not reported yet has nothing for a report to +// contradict. func validateSandboxClasses(recorded string, classes []*ateapipb.WorkerSandboxClass) error { if recorded == "" { return nil diff --git a/cmd/ateapi/internal/workerservice/register_test.go b/cmd/ateapi/internal/workerservice/register_test.go index 153cda1e7d..1fff45aa07 100644 --- a/cmd/ateapi/internal/workerservice/register_test.go +++ b/cmd/ateapi/internal/workerservice/register_test.go @@ -52,42 +52,358 @@ func testClass(name string, def *ateapipb.VersionedSandboxCompat, restorable ... // the class it gives the Worker. var testSandboxClasses = []*ateapipb.WorkerSandboxClass{testClass("gvisor", testCompat("amd64"))} -func setRequest(actors int32) *ateapipb.RegisterWorkerRequest { +func registerRequest(epoch int64, actors int32) *ateapipb.RegisterWorkerRequest { return &ateapipb.RegisterWorkerRequest{ + Worker: &ateapipb.ObjectRef{Name: testWorkerName}, + Epoch: epoch, + Ips: testIPs, + Capacity: &ateapipb.WorkerResources{Actors: actors}, + } +} + +func runtimesRequest(epoch int64) *ateapipb.SetWorkerSandboxRuntimesRequest { + return &ateapipb.SetWorkerSandboxRuntimesRequest{ Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - Capacity: &ateapipb.WorkerResources{Actors: actors}, + Epoch: epoch, SandboxClasses: testSandboxClasses, } } -// The point of the whole path: a Worker moves from what it reported before to -// what its worker reports now. +func mustGetWorker(t *testing.T, s *Server, name string) *ateapipb.Worker { + t.Helper() + w, err := s.store.GetWorker(context.Background(), name) + if err != nil { + t.Fatalf("GetWorker: %v", err) + } + return w +} + +// Each acquire hands out the next epoch, and none of them becomes active: a +// worker process that never registers changes nothing the control plane acts +// on. +func TestAcquireWorkerEpoch(t *testing.T) { + st, cleanup := storetest.SetupTestStore(t) + defer cleanup() + s := New(st, &fakeControl{}, testAteletSPIFFEID, nil, "", nil) + seedReportedWorker(t, st, testNode, &ateapipb.WorkerResources{Actors: 1}) + authed := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)) + + for want := int64(testEpoch + 1); want <= testEpoch+2; want++ { + got, err := s.AcquireWorkerEpoch(authed, &ateapipb.AcquireWorkerEpochRequest{Worker: &ateapipb.ObjectRef{Name: testWorkerName}}) + if err != nil { + t.Fatalf("AcquireWorkerEpoch() failed: %v", err) + } + if got.GetEpoch() != want { + t.Errorf("epoch = %d, want %d", got.GetEpoch(), want) + } + } + after := mustGetWorker(t, s, testWorkerName) + if got, want := after.GetStatus().GetIssuedEpoch(), int64(testEpoch+2); got != want { + t.Errorf("issued_epoch = %d, want %d", got, want) + } + if got := after.GetStatus().GetRegisteredEpoch(); got != testEpoch { + t.Errorf("registered_epoch = %d, want %d unchanged", got, testEpoch) + } +} + +func TestAcquireWorkerEpoch_NewWorker(t *testing.T) { + st, cleanup := storetest.SetupTestStore(t) + defer cleanup() + s := New(st, &fakeControl{}, testAteletSPIFFEID, nil, "", nil) + seedWorker(t, st, testWorkerName, testNode, &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE}) + authed := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)) + + got, err := s.AcquireWorkerEpoch(authed, &ateapipb.AcquireWorkerEpochRequest{Worker: &ateapipb.ObjectRef{Name: testWorkerName}}) + if err != nil { + t.Fatalf("AcquireWorkerEpoch() failed: %v", err) + } + if got.GetEpoch() != 1 { + t.Errorf("epoch = %d, want 1", got.GetEpoch()) + } +} + +func TestAcquireWorkerEpoch_Errors(t *testing.T) { + st, cleanup := storetest.SetupTestStore(t) + defer cleanup() + s := New(st, &fakeControl{}, testAteletSPIFFEID, nil, "", nil) + seedReportedWorker(t, st, testNode, &ateapipb.WorkerResources{Actors: 1}) + authed := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)) + ref := &ateapipb.ObjectRef{Name: testWorkerName} + + tests := []struct { + name string + ctx context.Context + req *ateapipb.AcquireWorkerEpochRequest + want codes.Code + }{ + {"unauthenticated", ateletauthtest.ContextWith(nil), &ateapipb.AcquireWorkerEpochRequest{Worker: ref}, codes.Unauthenticated}, + {"no worker ref", authed, &ateapipb.AcquireWorkerEpochRequest{}, codes.InvalidArgument}, + {"other node", ateletauthtest.ContextWith(ateletauthtest.CertOn(t, "some-other-node")), &ateapipb.AcquireWorkerEpochRequest{Worker: ref}, codes.NotFound}, + {"absent worker", authed, &ateapipb.AcquireWorkerEpochRequest{Worker: &ateapipb.ObjectRef{Name: "3b9f1e77-2c4d-4a80-91be-6d5c8f0a7e21"}}, codes.NotFound}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := s.AcquireWorkerEpoch(tc.ctx, tc.req) + if got := apierror.Code(err); got != tc.want { + t.Errorf("code = %v (err %v), want %v", got, err, tc.want) + } + }) + } + if got := mustGetWorker(t, s, testWorkerName).GetStatus().GetIssuedEpoch(); got != testEpoch { + t.Errorf("issued_epoch = %d after refused acquires, want %d unchanged", got, testEpoch) + } +} + +// The point of the whole path: a new worker process acquires an epoch and +// registers with it, which activates the epoch and replaces what the previous +// process reported. func TestRegisterWorker(t *testing.T) { st, cleanup := storetest.SetupTestStore(t) defer cleanup() s := New(st, &fakeControl{}, testAteletSPIFFEID, nil, "", nil) seedReportedWorker(t, st, testNode, &ateapipb.WorkerResources{Actors: 1, CpuMillicores: 2000, MemoryBytes: 0}) + authed := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)) - got, err := s.RegisterWorker(ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)), setRequest(4094)) + acquired, err := s.AcquireWorkerEpoch(authed, &ateapipb.AcquireWorkerEpochRequest{Worker: &ateapipb.ObjectRef{Name: testWorkerName}}) + if err != nil { + t.Fatalf("AcquireWorkerEpoch() failed: %v", err) + } + req := registerRequest(acquired.GetEpoch(), 4094) + req.Ips = []string{"10.9.8.7", "fd00::7"} + got, err := s.RegisterWorker(authed, req) if err != nil { t.Fatalf("RegisterWorker() failed: %v", err) } - if want := int32(4094); got.GetWorker().GetStatus().GetCapacity().GetActors() != want { - t.Errorf("capacity.actors = %d, want %d", got.GetWorker().GetStatus().GetCapacity().GetActors(), want) + status := got.GetWorker().GetStatus() + if status.GetRegisteredEpoch() != acquired.GetEpoch() { + t.Errorf("registered_epoch = %d, want %d", status.GetRegisteredEpoch(), acquired.GetEpoch()) + } + if diff := cmp.Diff(req.GetIps(), status.GetIps()); diff != "" { + t.Errorf("ips mismatch (-want +got):\n%s", diff) + } + if want := int32(4094); status.GetCapacity().GetActors() != want { + t.Errorf("capacity.actors = %d, want %d", status.GetCapacity().GetActors(), want) } // A report replaces what is recorded. The Worker reports everything it has, // so a dimension this one leaves out is one it no longer supplies -- keeping // the old value would advertise compute nothing claims to have. - if c := got.GetWorker().GetStatus().GetCapacity(); c.GetCpuMillicores() != 0 || c.GetMemoryBytes() != 0 { + if c := status.GetCapacity(); c.GetCpuMillicores() != 0 || c.GetMemoryBytes() != 0 { t.Errorf("capacity = %v, want the report's own (no cpu or memory)", c) } + // The sandbox classes were the previous process's; this one reports its own. + if got := status.GetSandboxClasses(); got != nil { + t.Errorf("sandbox_classes = %v, want cleared until the new process reports", got) + } + if diff := cmp.Diff(mustGetWorker(t, s, testWorkerName), got.GetWorker(), protocmp.Transform()); diff != "" { + t.Errorf("response is not the stored Worker (-want +got):\n%s", diff) + } +} + +// Only an epoch that was handed out, and is not older than the registered one, +// registers. Every Worker here starts with testEpoch registered and two more +// acquired. +func TestRegisterWorker_Epochs(t *testing.T) { + st, cleanup := storetest.SetupTestStore(t) + defer cleanup() + s := New(st, &fakeControl{}, testAteletSPIFFEID, nil, "", nil) + authed := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)) + seeded := func() *ateapipb.WorkerStatus { + return &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + IssuedEpoch: testEpoch + 2, + RegisteredEpoch: testEpoch, + Ips: testIPs, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + SandboxClasses: testSandboxClasses, + } + } + + tests := []struct { + name string + epoch int64 + ips []string + wantCode codes.Code + // wantWrite is whether the registration changes the record. + wantWrite bool + // wantRegistered is the registered_epoch afterwards. + wantRegistered int64 + // wantRuntimes is whether the seeded runtimes survive. + wantRuntimes bool + }{{ + name: "retry of the registration writes nothing", + epoch: testEpoch, + ips: testIPs, + wantRegistered: testEpoch, + wantRuntimes: true, + }, { + // A worker process registers once: the registered epoch's IPs and + // capacity do not change. + name: "registered epoch with other IPs is refused", + epoch: testEpoch, + ips: []string{"10.9.8.7"}, + wantCode: codes.FailedPrecondition, + wantRegistered: testEpoch, + wantRuntimes: true, + }, { + // A later acquire may be activated before an earlier one: only the + // highest registered epoch counts, and acquires never registered stay + // inactive. + name: "highest acquired epoch activates", + epoch: testEpoch + 2, + ips: testIPs, + wantWrite: true, + wantRegistered: testEpoch + 2, + }, { + name: "older epoch is refused", + epoch: testEpoch - 1, + ips: testIPs, + wantCode: codes.FailedPrecondition, + wantRegistered: testEpoch, + // Not written, so the runtimes stay. + wantRuntimes: true, + }, { + name: "epoch never handed out is refused", + epoch: testEpoch + 3, + ips: testIPs, + wantCode: codes.FailedPrecondition, + wantRegistered: testEpoch, + wantRuntimes: true, + }} + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + name := fmt.Sprintf("1a7e4c83-6d20-4f95-b3c8-%012d", i) + before := seedWorker(t, st, name, testNode, seeded()) + + _, err := s.RegisterWorker(authed, &ateapipb.RegisterWorkerRequest{ + Worker: &ateapipb.ObjectRef{Name: name}, + Epoch: tt.epoch, + Ips: tt.ips, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + }) + if code := apierror.Code(err); code != tt.wantCode { + t.Fatalf("code = %v (err %v), want %v", code, err, tt.wantCode) + } + after := mustGetWorker(t, s, name) + if wrote := after.GetMetadata().GetVersion() != before.GetMetadata().GetVersion(); wrote != tt.wantWrite { + t.Errorf("version %d -> %d, want a write: %v", before.GetMetadata().GetVersion(), after.GetMetadata().GetVersion(), tt.wantWrite) + } + if got := after.GetStatus().GetRegisteredEpoch(); got != tt.wantRegistered { + t.Errorf("registered_epoch = %d, want %d", got, tt.wantRegistered) + } + if got := len(after.GetStatus().GetSandboxClasses()) > 0; got != tt.wantRuntimes { + t.Errorf("sandbox_classes kept = %v, want %v", got, tt.wantRuntimes) + } + if tt.wantCode == codes.OK { + if diff := cmp.Diff(tt.ips, after.GetStatus().GetIps()); diff != "" { + t.Errorf("ips mismatch (-want +got):\n%s", diff) + } + } + }) + } +} + +// An atelet speaks for the Workers it herds and no others. A Worker on another +// node is reported as absent rather than forbidden, so a caller learns nothing +// about what runs elsewhere. +func TestRegisterWorker_OtherNodeIsNotFound(t *testing.T) { + st, cleanup := storetest.SetupTestStore(t) + defer cleanup() + s := New(st, &fakeControl{}, testAteletSPIFFEID, nil, "", nil) + seedReportedWorker(t, st, testNode, &ateapipb.WorkerResources{Actors: 1}) + + _, err := s.RegisterWorker(ateletauthtest.ContextWith(ateletauthtest.CertOn(t, "some-other-node")), registerRequest(testEpoch, 4094)) + if got := apierror.Code(err); got != codes.NotFound { + t.Fatalf("code = %v (err %v), want NotFound", got, err) + } + + // And the report must not have landed. + after, err := st.GetWorker(context.Background(), testWorkerName) + if err != nil { + t.Fatalf("GetWorker: %v", err) + } + if got := after.GetStatus().GetCapacity().GetActors(); got != 1 { + t.Errorf("capacity.actors = %d, want 1 unchanged", got) + } +} + +func TestRegisterWorker_Errors(t *testing.T) { + st, cleanup := storetest.SetupTestStore(t) + defer cleanup() + s := New(st, &fakeControl{}, testAteletSPIFFEID, nil, "", nil) + seedReportedWorker(t, st, testNode, &ateapipb.WorkerResources{Actors: 1}) + authed := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)) + without := func(mutate func(*ateapipb.RegisterWorkerRequest)) *ateapipb.RegisterWorkerRequest { + req := registerRequest(testEpoch, 2) + mutate(req) + return req + } + + tests := []struct { + name string + ctx context.Context + req *ateapipb.RegisterWorkerRequest + want codes.Code + }{ + {"unauthenticated", ateletauthtest.ContextWith(nil), registerRequest(testEpoch, 2), codes.Unauthenticated}, + {"no worker ref", authed, without(func(r *ateapipb.RegisterWorkerRequest) { r.Worker = nil }), codes.InvalidArgument}, + {"no epoch", authed, without(func(r *ateapipb.RegisterWorkerRequest) { r.Epoch = 0 }), codes.InvalidArgument}, + {"no ips", authed, without(func(r *ateapipb.RegisterWorkerRequest) { r.Ips = nil }), codes.InvalidArgument}, + {"no capacity", authed, without(func(r *ateapipb.RegisterWorkerRequest) { r.Capacity = nil }), codes.InvalidArgument}, + {"absent worker", authed, without(func(r *ateapipb.RegisterWorkerRequest) { + r.Worker = &ateapipb.ObjectRef{Name: "3b9f1e77-2c4d-4a80-91be-6d5c8f0a7e21"} + }), codes.NotFound}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := s.RegisterWorker(tc.ctx, tc.req) + if got := apierror.Code(err); got != tc.want { + t.Errorf("code = %v (err %v), want %v", got, err, tc.want) + } + }) + } +} + +// A report goes straight to the store, so nothing else checks it. A negative +// ceiling is the case that matters: placement asks whether allocated is below +// capacity, so the Worker would take no Actor ever again. +func TestRegisterWorker_RejectsNonsense(t *testing.T) { + st, cleanup := storetest.SetupTestStore(t) + defer cleanup() + s := New(st, &fakeControl{}, testAteletSPIFFEID, nil, "", nil) + seeded := seedReportedWorker(t, st, testNode, &ateapipb.WorkerResources{Actors: 4094}) + authed := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)) + + for _, tc := range []struct { + name string + capacity *ateapipb.WorkerResources + }{ + {"negative ceiling", &ateapipb.WorkerResources{Actors: -1}}, + {"int32 underflow", &ateapipb.WorkerResources{Actors: -2147483648}}, + {"negative quantity", &ateapipb.WorkerResources{CpuMillicores: -1, MemoryBytes: 0}}, + } { + t.Run(tc.name, func(t *testing.T) { + req := registerRequest(testEpoch, 0) + req.Capacity = tc.capacity + _, err := s.RegisterWorker(authed, req) + if got := apierror.Code(err); got != codes.InvalidArgument { + t.Fatalf("code = %v (err %v), want %v", got, err, codes.InvalidArgument) + } + }) + } + + after, err := st.GetWorker(context.Background(), testWorkerName) + if err != nil { + t.Fatalf("GetWorker: %v", err) + } + if diff := cmp.Diff(seeded.GetStatus().GetCapacity(), after.GetStatus().GetCapacity(), protocmp.Transform()); diff != "" { + t.Errorf("capacity changed despite every report being refused (-want +got):\n%s", diff) + } } -// A report's classes are checked against the class the Worker recorded at its -// first registration and recorded only when they differ from what is stored. -// Capacity is held at what was seeded, so a write here is one the classes -// alone caused. -func TestRegisterWorker_SandboxClasses(t *testing.T) { +// A report's classes are checked against the class the registered worker +// process reported first and recorded only when they differ from what is +// stored. +func TestSetWorkerSandboxRuntimes(t *testing.T) { st, cleanup := storetest.SetupTestStore(t) defer cleanup() s := New(st, &fakeControl{}, testAteletSPIFFEID, nil, "", nil) @@ -95,23 +411,27 @@ func TestRegisterWorker_SandboxClasses(t *testing.T) { amd64, arm64 := testCompat("amd64"), testCompat("arm64") v8, v9 := testCompat("arm64-v8"), testCompat("arm64-v9") - // Every Worker here has already reported gvisor with amd64 as its default - // and v8 and v9 restorable, in that order, at a capacity of one actor. + // Every Worker here has registered testEpoch and reported gvisor with + // amd64 as its default and v8 and v9 restorable, in that order. seeded := func() *ateapipb.WorkerStatus { return &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - Capacity: &ateapipb.WorkerResources{Actors: 1}, - SandboxClasses: []*ateapipb.WorkerSandboxClass{testClass("gvisor", amd64, v8, v9)}, + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + IssuedEpoch: testEpoch + 1, + RegisteredEpoch: testEpoch, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + SandboxClasses: []*ateapipb.WorkerSandboxClass{testClass("gvisor", amd64, v8, v9)}, } } tests := []struct { name string - // unreported seeds a Worker that has not registered yet, with no - // sandbox classes, instead of seeded(). + // unreported seeds a Worker whose worker process has registered + // testEpoch but reported no sandbox classes yet, instead of seeded(). unreported bool - classes []*ateapipb.WorkerSandboxClass - wantCode codes.Code + // epoch defaults to testEpoch. + epoch int64 + classes []*ateapipb.WorkerSandboxClass + wantCode codes.Code // wantWrite is whether the report changes the record. wantWrite bool }{{ @@ -140,8 +460,20 @@ func TestRegisterWorker_SandboxClasses(t *testing.T) { classes: []*ateapipb.WorkerSandboxClass{testClass("gvisor", amd64)}, wantWrite: true, }, { - // Once a Worker has registered a sandbox class, an entry of another - // class contradicts its recorded state and is refused. + // A report from a worker process that has since been replaced, or + // one that has not registered yet, describes no running process. + name: "older epoch is refused", + epoch: testEpoch - 1, + classes: []*ateapipb.WorkerSandboxClass{testClass("gvisor", arm64)}, + wantCode: codes.FailedPrecondition, + }, { + name: "acquired but unregistered epoch is refused", + epoch: testEpoch + 1, + classes: []*ateapipb.WorkerSandboxClass{testClass("gvisor", arm64)}, + wantCode: codes.FailedPrecondition, + }, { + // Once the worker process has reported a sandbox class, an entry of + // another class contradicts its recorded state and is refused. name: "another class is refused", classes: []*ateapipb.WorkerSandboxClass{testClass("microvm", amd64, v8, v9)}, wantCode: codes.FailedPrecondition, @@ -152,8 +484,8 @@ func TestRegisterWorker_SandboxClasses(t *testing.T) { classes: []*ateapipb.WorkerSandboxClass{testClass("gvisor", amd64, v8, v9), testClass("microvm", amd64)}, wantCode: codes.InvalidArgument, }, { - // A Worker that has not registered yet accepts any class on its first - // report. + // A worker process that has not reported yet sets the class on its + // first report. name: "unreported Worker takes a class of any name", unreported: true, classes: []*ateapipb.WorkerSandboxClass{testClass("microvm", amd64)}, @@ -167,22 +499,23 @@ func TestRegisterWorker_SandboxClasses(t *testing.T) { name := fmt.Sprintf("1a7e4c83-6d20-4f95-b3c8-%012d", i) initialStatus := seeded() if tt.unreported { - initialStatus = &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE} + initialStatus.SandboxClasses = nil } before := seedWorker(t, st, name, testNode, initialStatus) + epoch := tt.epoch + if epoch == 0 { + epoch = testEpoch + } - got, err := s.RegisterWorker(authed, &ateapipb.RegisterWorkerRequest{ + got, err := s.SetWorkerSandboxRuntimes(authed, &ateapipb.SetWorkerSandboxRuntimesRequest{ Worker: &ateapipb.ObjectRef{Name: name}, - Capacity: &ateapipb.WorkerResources{Actors: 1}, + Epoch: epoch, SandboxClasses: tt.classes, }) if code := apierror.Code(err); code != tt.wantCode { t.Fatalf("code = %v (err %v), want %v", code, err, tt.wantCode) } - after, err := st.GetWorker(context.Background(), name) - if err != nil { - t.Fatalf("GetWorker: %v", err) - } + after := mustGetWorker(t, s, name) if wrote := after.GetMetadata().GetVersion() != before.GetMetadata().GetVersion(); wrote != tt.wantWrite { t.Fatalf("version %d -> %d, want a write: %v", before.GetMetadata().GetVersion(), after.GetMetadata().GetVersion(), tt.wantWrite) } @@ -202,57 +535,10 @@ func TestRegisterWorker_SandboxClasses(t *testing.T) { } } -// An atelet speaks for the Workers it herds and no others. A Worker on another -// node is reported as absent rather than forbidden, so a caller learns nothing -// about what runs elsewhere. -func TestRegisterWorker_OtherNodeIsNotFound(t *testing.T) { - st, cleanup := storetest.SetupTestStore(t) - defer cleanup() - s := New(st, &fakeControl{}, testAteletSPIFFEID, nil, "", nil) - seedReportedWorker(t, st, testNode, &ateapipb.WorkerResources{Actors: 1}) - - _, err := s.RegisterWorker(ateletauthtest.ContextWith(ateletauthtest.CertOn(t, "some-other-node")), setRequest(4094)) - if got := apierror.Code(err); got != codes.NotFound { - t.Fatalf("code = %v (err %v), want NotFound", got, err) - } - - // And the report must not have landed. - after, err := st.GetWorker(context.Background(), testWorkerName) - if err != nil { - t.Fatalf("GetWorker: %v", err) - } - if got := after.GetStatus().GetCapacity().GetActors(); got != 1 { - t.Errorf("capacity.actors = %d, want 1 unchanged", got) - } -} - -// Re-sending the same capacity is not an update. A worker reports once, but it -// retries until accepted and reports again if it restarts, so a repeat must not -// churn the Worker's version. -func TestRegisterWorker_UnchangedDoesNotWrite(t *testing.T) { - st, cleanup := storetest.SetupTestStore(t) - defer cleanup() - s := New(st, &fakeControl{}, testAteletSPIFFEID, nil, "", nil) - seeded := seedReportedWorker(t, st, testNode, &ateapipb.WorkerResources{Actors: 4094}) - - for range 3 { - if _, err := s.RegisterWorker(ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)), setRequest(4094)); err != nil { - t.Fatalf("RegisterWorker() failed: %v", err) - } - } - after, err := st.GetWorker(context.Background(), testWorkerName) - if err != nil { - t.Fatalf("GetWorker: %v", err) - } - if got, want := after.GetMetadata().GetVersion(), seeded.GetMetadata().GetVersion(); got != want { - t.Errorf("version = %d after three identical reports, want %d unchanged", got, want) - } -} - // Attribute order is the producer's business, not the record's: the same // attributes in another order are the same runtime, so they are stored sorted // by key and a reordered repeat writes nothing. -func TestRegisterWorker_SortsAttributes(t *testing.T) { +func TestSetWorkerSandboxRuntimes_SortsAttributes(t *testing.T) { st, cleanup := storetest.SetupTestStore(t) defer cleanup() s := New(st, &fakeControl{}, testAteletSPIFFEID, nil, "", nil) @@ -266,118 +552,68 @@ func TestRegisterWorker_SortsAttributes(t *testing.T) { } return c } - register := func(def, restorable *ateapipb.VersionedSandboxCompat) *ateapipb.Worker { + report := func(def, restorable *ateapipb.VersionedSandboxCompat) *ateapipb.Worker { t.Helper() - got, err := s.RegisterWorker(authed, &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - Capacity: &ateapipb.WorkerResources{Actors: 1}, - SandboxClasses: []*ateapipb.WorkerSandboxClass{testClass("gvisor", def, restorable)}, - }) + req := runtimesRequest(testEpoch) + req.SandboxClasses = []*ateapipb.WorkerSandboxClass{testClass("gvisor", def, restorable)} + got, err := s.SetWorkerSandboxRuntimes(authed, req) if err != nil { - t.Fatalf("RegisterWorker() failed: %v", err) + t.Fatalf("SetWorkerSandboxRuntimes() failed: %v", err) } return got.GetWorker() } - first := register(compatWith("cpu_features", "architecture"), compatWith("sandbox_bundle_hash", "architecture")) + first := report(compatWith("cpu_features", "architecture"), compatWith("sandbox_bundle_hash", "architecture")) want := []*ateapipb.WorkerSandboxClass{testClass("gvisor", compatWith("architecture", "cpu_features"), compatWith("architecture", "sandbox_bundle_hash"))} if diff := cmp.Diff(want, first.GetStatus().GetSandboxClasses(), protocmp.Transform()); diff != "" { t.Errorf("sandbox_classes not stored sorted by key (-want +got):\n%s", diff) } - again := register(compatWith("architecture", "cpu_features"), compatWith("architecture", "sandbox_bundle_hash")) + again := report(compatWith("architecture", "cpu_features"), compatWith("architecture", "sandbox_bundle_hash")) if got, want := again.GetMetadata().GetVersion(), first.GetMetadata().GetVersion(); got != want { t.Errorf("version = %d after a reordered repeat, want %d unchanged", got, want) } } -func TestRegisterWorker_Errors(t *testing.T) { +func TestSetWorkerSandboxRuntimes_Errors(t *testing.T) { st, cleanup := storetest.SetupTestStore(t) defer cleanup() s := New(st, &fakeControl{}, testAteletSPIFFEID, nil, "", nil) seedReportedWorker(t, st, testNode, &ateapipb.WorkerResources{Actors: 1}) authed := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)) + without := func(mutate func(*ateapipb.SetWorkerSandboxRuntimesRequest)) *ateapipb.SetWorkerSandboxRuntimesRequest { + req := runtimesRequest(testEpoch) + mutate(req) + return req + } tests := []struct { name string ctx context.Context - req *ateapipb.RegisterWorkerRequest + req *ateapipb.SetWorkerSandboxRuntimesRequest want codes.Code }{ - {"unauthenticated", ateletauthtest.ContextWith(nil), setRequest(2), codes.Unauthenticated}, - {"no worker ref", authed, &ateapipb.RegisterWorkerRequest{ - Capacity: &ateapipb.WorkerResources{Actors: 2}, - SandboxClasses: testSandboxClasses, - }, codes.InvalidArgument}, - {"no capacity", authed, &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - SandboxClasses: testSandboxClasses, - }, codes.InvalidArgument}, - {"no sandbox classes", authed, &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - Capacity: &ateapipb.WorkerResources{Actors: 2}, - }, codes.InvalidArgument}, - {"class without a default runtime", authed, &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - Capacity: &ateapipb.WorkerResources{Actors: 2}, - SandboxClasses: []*ateapipb.WorkerSandboxClass{{SandboxClass: "gvisor"}}, - }, codes.InvalidArgument}, - {"class that is not a sandbox class", authed, &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - Capacity: &ateapipb.WorkerResources{Actors: 2}, - SandboxClasses: []*ateapipb.WorkerSandboxClass{testClass("plain", testCompat("amd64"))}, - }, codes.InvalidArgument}, - {"absent worker", authed, &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: "3b9f1e77-2c4d-4a80-91be-6d5c8f0a7e21"}, - Capacity: &ateapipb.WorkerResources{Actors: 2}, - SandboxClasses: testSandboxClasses, - }, codes.NotFound}, + {"unauthenticated", ateletauthtest.ContextWith(nil), runtimesRequest(testEpoch), codes.Unauthenticated}, + {"no worker ref", authed, without(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.Worker = nil }), codes.InvalidArgument}, + {"no epoch", authed, without(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.Epoch = 0 }), codes.InvalidArgument}, + {"no sandbox classes", authed, without(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { r.SandboxClasses = nil }), codes.InvalidArgument}, + {"class without a default runtime", authed, without(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { + r.SandboxClasses = []*ateapipb.WorkerSandboxClass{{SandboxClass: "gvisor"}} + }), codes.InvalidArgument}, + {"class that is not a sandbox class", authed, without(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { + r.SandboxClasses = []*ateapipb.WorkerSandboxClass{testClass("plain", testCompat("amd64"))} + }), codes.InvalidArgument}, + {"other node", ateletauthtest.ContextWith(ateletauthtest.CertOn(t, "some-other-node")), runtimesRequest(testEpoch), codes.NotFound}, + {"absent worker", authed, without(func(r *ateapipb.SetWorkerSandboxRuntimesRequest) { + r.Worker = &ateapipb.ObjectRef{Name: "3b9f1e77-2c4d-4a80-91be-6d5c8f0a7e21"} + }), codes.NotFound}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - _, err := s.RegisterWorker(tc.ctx, tc.req) + _, err := s.SetWorkerSandboxRuntimes(tc.ctx, tc.req) if got := apierror.Code(err); got != tc.want { t.Errorf("code = %v (err %v), want %v", got, err, tc.want) } }) } } - -// A report goes straight to the store, so nothing else checks it. A negative -// ceiling is the case that matters: placement asks whether allocated is below -// capacity, so the Worker would take no Actor ever again. -func TestRegisterWorker_RejectsNonsense(t *testing.T) { - st, cleanup := storetest.SetupTestStore(t) - defer cleanup() - s := New(st, &fakeControl{}, testAteletSPIFFEID, nil, "", nil) - seeded := seedReportedWorker(t, st, testNode, &ateapipb.WorkerResources{Actors: 4094}) - authed := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)) - - for _, tc := range []struct { - name string - capacity *ateapipb.WorkerResources - }{ - {"negative ceiling", &ateapipb.WorkerResources{Actors: -1}}, - {"int32 underflow", &ateapipb.WorkerResources{Actors: -2147483648}}, - {"negative quantity", &ateapipb.WorkerResources{CpuMillicores: -1, MemoryBytes: 0}}, - } { - t.Run(tc.name, func(t *testing.T) { - _, err := s.RegisterWorker(authed, &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - Capacity: tc.capacity, - SandboxClasses: testSandboxClasses, - }) - if got := apierror.Code(err); got != codes.InvalidArgument { - t.Fatalf("code = %v (err %v), want %v", got, err, codes.InvalidArgument) - } - }) - } - - after, err := st.GetWorker(context.Background(), testWorkerName) - if err != nil { - t.Fatalf("GetWorker: %v", err) - } - if diff := cmp.Diff(seeded.GetStatus().GetCapacity(), after.GetStatus().GetCapacity(), protocmp.Transform()); diff != "" { - t.Errorf("capacity changed despite every report being refused (-want +got):\n%s", diff) - } -} diff --git a/cmd/ateapi/internal/workerservice/service_test.go b/cmd/ateapi/internal/workerservice/service_test.go index 5ad949601d..915dc30962 100644 --- a/cmd/ateapi/internal/workerservice/service_test.go +++ b/cmd/ateapi/internal/workerservice/service_test.go @@ -78,15 +78,25 @@ func (f *fakeControl) CrashActorReportedByWorker(_ context.Context, actorRef res return f.crashResp, f.err } -// seedReportedWorker registers a Worker on nodeName that has already reported -// capacity and its default runtime: identity from the pod, the way the syncer -// writes it, plus the result of an earlier registration. A Worker that has -// never registered carries none, so what a fresh registration replaces is what -// this seeds. +// testEpoch is the epoch seedReportedWorker records as acquired and active. +const testEpoch = 3 + +// testIPs are the pod IPs seedReportedWorker records as registered. +var testIPs = []string{"10.1.2.3"} + +// seedReportedWorker registers a Worker on nodeName whose worker process has +// already registered with testEpoch, testIPs and capacity, and reported its +// default runtime: identity from the pod, the way the syncer writes it, plus +// the result of an earlier registration. func seedReportedWorker(t *testing.T, st store.Interface, nodeName string, capacity *ateapipb.WorkerResources) *ateapipb.Worker { t.Helper() return seedWorker(t, st, testWorkerName, nodeName, &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: capacity, SandboxClasses: testSandboxClasses, + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + IssuedEpoch: testEpoch, + RegisteredEpoch: testEpoch, + Ips: testIPs, + Capacity: capacity, + SandboxClasses: testSandboxClasses, }) } @@ -101,7 +111,6 @@ func seedWorker(t *testing.T, st store.Interface, name, nodeName string, status WorkerPod: "worker-pod-1", WorkerPodUid: name, NodeName: nodeName, - Ips: []string{"10.1.2.3"}, Status: status, }) if err != nil { diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply.go b/cmd/atecontroller/internal/controllers/workerpool_apply.go index b2945a7d90..8be68e8c28 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply.go @@ -123,6 +123,7 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel workerOTelSetti args := []string{ "--pod-uid=$(POD_UID)", + "--pod-ips=$(POD_IPS)", "--atunnel-listen-address=:443", "--atunnel-connect-listen-address=:8443", "--atunnel-credential-bundle=" + atunnelIdentityMountPath + "/credential-bundle.pem", @@ -301,6 +302,9 @@ func workerContainerEnv(otel workerOTelSettings) []*corev1ac.EnvVarApplyConfigur fieldRefEnv("WORKER_POOL_NAME", "metadata.labels['"+workerPoolLabel+"']"), fieldRefEnv("POD_NAME", "metadata.name"), fieldRefEnv("NODE_NAME", "spec.nodeName"), + // Resolved when the container starts, so a worker process started in a + // recreated pod sandbox registers the sandbox's new IPs. + fieldRefEnv("POD_IPS", "status.podIPs"), } if otel.Endpoint != "" { envs = append(envs, corev1ac.EnvVar().WithName("OTEL_EXPORTER_OTLP_ENDPOINT").WithValue(otel.Endpoint)) diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go index 8143c695bb..af9780948d 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go @@ -874,6 +874,7 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf WithImage(wp.Spec.WorkerImage). WithArgs( "--pod-uid=$(POD_UID)", + "--pod-ips=$(POD_IPS)", "--atunnel-listen-address=:443", "--atunnel-connect-listen-address=:8443", "--atunnel-credential-bundle="+atunnelIdentityMountPath+"/credential-bundle.pem", @@ -937,6 +938,11 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf WithValueFrom(corev1ac.EnvVarSource(). WithFieldRef(corev1ac.ObjectFieldSelector(). WithFieldPath("spec.nodeName"))), + corev1ac.EnvVar(). + WithName("POD_IPS"). + WithValueFrom(corev1ac.EnvVarSource(). + WithFieldRef(corev1ac.ObjectFieldSelector(). + WithFieldPath("status.podIPs"))), corev1ac.EnvVar(). WithName("OTEL_RESOURCE_ATTRIBUTES"). WithValue(workerOTelResourceAttributes), diff --git a/cmd/atecontroller/internal/workersync/fakecontrol_test.go b/cmd/atecontroller/internal/workersync/fakecontrol_test.go index a225348d83..7c5e3132d9 100644 --- a/cmd/atecontroller/internal/workersync/fakecontrol_test.go +++ b/cmd/atecontroller/internal/workersync/fakecontrol_test.go @@ -49,9 +49,6 @@ type fakeControl struct { mu sync.Mutex workers map[string]*ateapipb.Worker uidSeq int - // history holds every Worker written, in order, so a test can check the - // states in between the ones it reads back. - history []*ateapipb.Worker // createHook, when set, decides CreateWorker's outcome: a non-nil error is // returned and nothing is registered. @@ -111,20 +108,9 @@ func (f *fakeControl) putLocked(w *ateapipb.Worker) *ateapipb.Worker { return proto.Clone(stored).(*ateapipb.Worker) } -// storeLocked writes w and records it in the write history. +// storeLocked writes w. func (f *fakeControl) storeLocked(w *ateapipb.Worker) { f.workers[w.GetMetadata().GetName()] = w - f.history = append(f.history, proto.Clone(w).(*ateapipb.Worker)) -} - -// writes returns every Worker written since the history was last taken, in -// order, and clears it. -func (f *fakeControl) writes() []*ateapipb.Worker { - f.mu.Lock() - defer f.mu.Unlock() - history := f.history - f.history = nil - return history } func (f *fakeControl) setCreateHook(hook func(*ateapipb.Worker) error) { @@ -214,16 +200,11 @@ func (f *fakeControl) UpdateWorker(_ context.Context, in *ateapipb.UpdateWorkerR updated.Metadata = proto.Clone(stored.GetMetadata()).(*ateapipb.ResourceMetadata) updated.Status = proto.Clone(stored.GetStatus()).(*ateapipb.WorkerStatus) - // labels and epoch are the only fields an update may change, so pinning - // them to what is stored leaves any remaining difference on a field that is - // immutable after create — including one the request cleared by omitting - // it. - if updated.GetEpoch() < stored.GetEpoch() { - return nil, status.Error(codes.InvalidArgument, "update decreased epoch") - } + // labels are the only field an update may change, so pinning them to what + // is stored leaves any remaining difference on a field that is immutable + // after create, including one the request cleared by omitting it. probe := proto.Clone(updated).(*ateapipb.Worker) probe.Labels = stored.GetLabels() - probe.Epoch = stored.GetEpoch() if !proto.Equal(probe, stored) { return nil, status.Error(codes.InvalidArgument, "update changed a field that is immutable after create") } diff --git a/cmd/atecontroller/internal/workersync/syncer.go b/cmd/atecontroller/internal/workersync/syncer.go index d6ecb1661e..0bba57763b 100644 --- a/cmd/atecontroller/internal/workersync/syncer.go +++ b/cmd/atecontroller/internal/workersync/syncer.go @@ -21,7 +21,6 @@ import ( "fmt" "log/slog" "maps" - "slices" "time" "go.ate.dev/substrate/internal/installdefaults" @@ -292,25 +291,10 @@ func (s *WorkerPoolSyncer) reconcile(ctx context.Context, key workerKey) error { return s.createOrUpdateWorker(ctx, key, pod) } -// workerContainer is the name of the worker pod's worker container. -const workerContainer = "worker" - -// podEpoch counts the runs of the pod's worker container: 1 for its first run -// and one more for each restart. 0 until kubelet reports the container. -func podEpoch(pod *corev1.Pod) int64 { - for _, cs := range pod.Status.ContainerStatuses { - if cs.Name == workerContainer { - return int64(cs.RestartCount) + 1 - } - } - return 0 -} - // createOrUpdateWorker registers the pod's Worker, or brings the registered // one in line with the pod. A Worker is registered UNAVAILABLE and marked -// available only while its pod is Ready, and only once its epoch matches the -// worker running, so nothing is placed on it under the epoch of a worker that -// has since restarted. +// available only while its pod is Ready. Its epochs, IPs, capacity and runtimes +// are its worker process's to report, through WorkerService. func (s *WorkerPoolSyncer) createOrUpdateWorker(ctx context.Context, key workerKey, pod *corev1.Pod) error { poolName := pod.Labels[workerPodLabel] poolObject, exists, err := s.workerPoolInformer.GetIndexer().GetByKey(key.namespace + "/" + poolName) @@ -329,9 +313,9 @@ func (s *WorkerPoolSyncer) createOrUpdateWorker(ctx context.Context, key workerK w, err := s.client.GetWorker(ctx, &ateapipb.GetWorkerRequest{Worker: key.workerRef()}) if status.Code(err) == codes.NotFound { - // ips and node_name are required and immutable, so the pod is registered - // once it has an IP; the update that reports it re-enqueues the pod. - if len(pod.Status.PodIPs) == 0 { + // node_name is required and immutable, so the pod is registered once it + // is scheduled; the update that binds it re-enqueues the pod. + if pod.Spec.NodeName == "" { return nil } slog.InfoContext(ctx, "Syncer: registering worker", key.logAttrs()...) @@ -342,15 +326,10 @@ func (s *WorkerPoolSyncer) createOrUpdateWorker(ctx context.Context, key workerK WorkerNamespace: pod.Namespace, WorkerPool: poolName, WorkerPod: pod.Name, - Ips: podIPs(pod), WorkerPodUid: string(pod.UID), NodeName: pod.Spec.NodeName, WorkerSpiffeId: workerSPIFFEID(pod), Labels: pool.GetLabels(), - Epoch: podEpoch(pod), - // Capacity and runtimes are the Worker's to report, not the - // syncer's to infer from the pod: they are what the worker can - // actually supply. } // status is output-only: CreateWorker registers the Worker UNAVAILABLE. // @@ -384,25 +363,11 @@ func (s *WorkerPoolSyncer) createOrUpdateWorker(ctx context.Context, key workerK // after create, so drift there cannot be repaired by an update; it takes a // new pod, which arrives under a new key. var changed bool - if epoch := podEpoch(pod); epoch > w.GetEpoch() { - slog.InfoContext(ctx, "Syncer: updating worker (worker restarted)", - append(key.logAttrs(), slog.Int64("epoch", epoch))...) - w.Epoch = epoch - changed = true - } if !maps.Equal(w.GetLabels(), pool.GetLabels()) { slog.InfoContext(ctx, "Syncer: updating worker (labels changed)", key.logAttrs()...) w.Labels = pool.GetLabels() changed = true } - if ips := podIPs(pod); len(ips) > 0 && !slices.Equal(w.GetIps(), ips) { - // TODO: I don't think this is possible, but handling this case so we can - // log it just in case we can reproduce it. It is logged rather than - // repaired because ips is immutable on a registered Worker: writing the - // pod's value back would be rejected rather than applied. - slog.WarnContext(ctx, "Syncer: registered worker IPs disagree with its pod", - append(key.logAttrs(), slog.Any("registered", w.GetIps()), slog.Any("pod_ips", ips))...) - } if changed { w, err = s.client.UpdateWorker(ctx, &ateapipb.UpdateWorkerRequest{Worker: w}) if err != nil { @@ -410,8 +375,7 @@ func (s *WorkerPoolSyncer) createOrUpdateWorker(ctx context.Context, key workerK } } - // Last, so the Worker takes Actors only at the epoch written above. A - // DRAINING Worker is left alone: its pod is on its way out. + // A DRAINING Worker is left alone: its pod is on its way out. if available && w.GetStatus().GetState() == ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE { slog.InfoContext(ctx, "Syncer: marking worker available (pod Ready)", key.logAttrs()...) _, err = s.client.MarkWorkerAvailable(ctx, &ateapipb.MarkWorkerAvailableRequest{Worker: key.workerRef()}) @@ -541,16 +505,6 @@ func (s *WorkerPoolSyncer) getOwner(ctx context.Context, namespace string, ref * return owner, nil } -// podIPs returns the pod's IP addresses, one per IP family, in the order -// Kubernetes reports them. -func podIPs(pod *corev1.Pod) []string { - ips := make([]string, 0, len(pod.Status.PodIPs)) - for _, ip := range pod.Status.PodIPs { - ips = append(ips, ip.IP) - } - return ips -} - // workerSPIFFEID returns the SPIFFE ID the podidentity signer issues to pod. func workerSPIFFEID(pod *corev1.Pod) string { return installdefaults.SPIFFEID(pod.Namespace, pod.Spec.ServiceAccountName) diff --git a/cmd/atecontroller/internal/workersync/syncer_restart_test.go b/cmd/atecontroller/internal/workersync/syncer_restart_test.go deleted file mode 100644 index 6a788cc33d..0000000000 --- a/cmd/atecontroller/internal/workersync/syncer_restart_test.go +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package workersync - -import ( - "context" - "testing" - - "go.ate.dev/substrate/pkg/proto/ateapipb" - corev1 "k8s.io/api/core/v1" -) - -// withWorkerRestarts reports the pod's worker container as having restarted n -// times. -func withWorkerRestarts(pod *corev1.Pod, n int32) *corev1.Pod { - pod.Status.ContainerStatuses = []corev1.ContainerStatus{ - {Name: "sidecar", RestartCount: 7}, - {Name: workerContainer, RestartCount: n}, - } - return pod -} - -func TestPodEpoch(t *testing.T) { - pod := workerPod("ns", "worker", "pool", testPodUID, "10.0.0.1") - if got := podEpoch(pod); got != 0 { - t.Errorf("podEpoch with no worker status = %d, want 0", got) - } - if got := podEpoch(withWorkerRestarts(pod, 0)); got != 1 { - t.Errorf("podEpoch on the first run = %d, want 1", got) - } - if got := podEpoch(withWorkerRestarts(pod, 4)); got != 5 { - t.Errorf("podEpoch after 4 restarts = %d, want 5", got) - } -} - -func TestSyncer_RegistersEpoch(t *testing.T) { - ctx := context.Background() - ns, podName, poolName := "ns-syncer-epoch", "worker-epoch-1", "pool1" - - api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) - key := seedPod(t, pods, withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 2)) - - mustReconcile(t, ctx, s, key) - if got := api.get(testPodUID).GetEpoch(); got != 3 { - t.Errorf("registered epoch = %d, want 3", got) - } -} - -func TestSyncer_RaisesEpochOnWorkerRestart(t *testing.T) { - ctx := context.Background() - ns, podName, poolName := "ns-syncer-restart", "worker-restart-1", "pool1" - - for _, tc := range []struct { - name string - ready bool - }{ - {name: "Ready", ready: true}, - // A restarting worker is not Ready, but its Actors are already lost. - {name: "not Ready", ready: false}, - } { - t.Run(tc.name, func(t *testing.T) { - api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) - pod := withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 0) - key := seedPod(t, pods, pod) - mustReconcile(t, ctx, s, key) - if got := api.get(testPodUID).GetEpoch(); got != 1 { - t.Fatalf("registered epoch = %d, want 1", got) - } - - restarted := withWorkerRestarts(pod.DeepCopy(), 1) - if !tc.ready { - restarted.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionFalse}} - } - if err := pods.Update(restarted); err != nil { - t.Fatalf("updating pod: %v", err) - } - mustReconcile(t, ctx, s, key) - got := api.get(testPodUID) - if got.GetEpoch() != 2 { - t.Errorf("epoch after a restart = %d, want 2", got.GetEpoch()) - } - - // Unchanged on the next pass: no further write. - version := got.GetMetadata().GetVersion() - mustReconcile(t, ctx, s, key) - if v := api.get(testPodUID).GetMetadata().GetVersion(); v != version { - t.Errorf("worker version = %d after an unchanged pass, want %d", v, version) - } - }) - } -} - -// TestSyncer_NoPlacementUnderAnEarlierEpoch walks a worker restart through -// the pod states kubelet reports, and pins that no write leaves the Worker -// ACTIVE while its epoch trails the worker that is running. An Actor placed -// then would be stamped with the earlier epoch and crashed with the Actors the -// restart lost. -func TestSyncer_NoPlacementUnderAnEarlierEpoch(t *testing.T) { - ctx := context.Background() - ns, podName, poolName := "ns-syncer-placement", "worker-placement-1", "pool1" - notReady := func(pod *corev1.Pod) *corev1.Pod { - pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionFalse}} - return pod - } - - for _, tc := range []struct { - name string - steps []*corev1.Pod - want []ateapipb.WorkerState - }{ - { - name: "every transition observed", - steps: []*corev1.Pod{ - // worker exited; kubelet has not started it again. - notReady(withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 0)), - // worker running again, readiness probe not yet passed. - notReady(withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 1)), - withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 1), - }, - want: []ateapipb.WorkerState{ - ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE, - ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE, - ateapipb.WorkerState_WORKER_STATE_ACTIVE, - }, - }, - { - // The informer coalesces updates, so the syncer may only see the - // pod Ready again after the restart. - name: "not Ready never observed", - steps: []*corev1.Pod{ - withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 1), - }, - want: []ateapipb.WorkerState{ateapipb.WorkerState_WORKER_STATE_ACTIVE}, - }, - { - // The epoch has to be raised before the Worker is marked available. - name: "Ready again under a new epoch", - steps: []*corev1.Pod{ - notReady(withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 0)), - withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 1), - }, - want: []ateapipb.WorkerState{ - ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE, - ateapipb.WorkerState_WORKER_STATE_ACTIVE, - }, - }, - { - // The restart is first seen with worker running again but not Ready. - name: "not Ready under a new epoch", - steps: []*corev1.Pod{ - notReady(withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 1)), - withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 1), - }, - want: []ateapipb.WorkerState{ - ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE, - ateapipb.WorkerState_WORKER_STATE_ACTIVE, - }, - }, - } { - t.Run(tc.name, func(t *testing.T) { - api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) - key := seedPod(t, pods, withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 0)) - mustReconcile(t, ctx, s, key) - if state := api.get(testPodUID).GetStatus().GetState(); state != ateapipb.WorkerState_WORKER_STATE_ACTIVE { - t.Fatalf("state after registering = %v, want %v", state, ateapipb.WorkerState_WORKER_STATE_ACTIVE) - } - api.writes() - - for i, pod := range tc.steps { - if err := pods.Update(pod); err != nil { - t.Fatalf("step %d: updating pod: %v", i, err) - } - mustReconcile(t, ctx, s, key) - if state := api.get(testPodUID).GetStatus().GetState(); state != tc.want[i] { - t.Errorf("step %d: state = %v, want %v", i, state, tc.want[i]) - } - for j, w := range api.writes() { - if w.GetStatus().GetState() == ateapipb.WorkerState_WORKER_STATE_ACTIVE && w.GetEpoch() < podEpoch(pod) { - t.Errorf("step %d, write %d: worker ACTIVE at epoch %d while the pod reports epoch %d", i, j, w.GetEpoch(), podEpoch(pod)) - } - } - } - if got := api.get(testPodUID).GetEpoch(); got != 2 { - t.Errorf("epoch after the restart = %d, want 2", got) - } - }) - } -} - -// A pod first seen not Ready after worker restarts registers unavailable at -// the epoch of the worker it is running. -func TestSyncer_NotReadyPodRegistersAtItsEpoch(t *testing.T) { - ctx := context.Background() - ns, poolName := "ns-syncer-notready-restart", "pool1" - - api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) - pod := withWorkerRestarts(workerPod(ns, "worker-1", poolName, testPodUID, "10.0.0.1"), 3) - pod.Status.Conditions = nil - key := seedPod(t, pods, pod) - - mustReconcile(t, ctx, s, key) - got := api.get(testPodUID) - if got.GetStatus().GetState() != ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE { - t.Errorf("state = %v, want %v", got.GetStatus().GetState(), ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE) - } - if got.GetEpoch() != 4 { - t.Errorf("epoch = %d, want 4", got.GetEpoch()) - } -} - -func TestSyncer_EpochNeverLowered(t *testing.T) { - ctx := context.Background() - ns, podName, poolName := "ns-syncer-lower", "worker-lower-1", "pool1" - - api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) - registered := registeredWorker(ns, poolName, podName, testPodUID, "10.0.0.1") - registered.Epoch = 5 - api.put(registered) - key := seedPod(t, pods, withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 0)) - - mustReconcile(t, ctx, s, key) - got := api.get(testPodUID) - if got.GetEpoch() != 5 { - t.Errorf("epoch = %d, want 5 left alone", got.GetEpoch()) - } - if got.GetMetadata().GetVersion() != 1 { - t.Errorf("worker version = %d, want 1: nothing to write", got.GetMetadata().GetVersion()) - } -} diff --git a/cmd/atecontroller/internal/workersync/syncer_test.go b/cmd/atecontroller/internal/workersync/syncer_test.go index 437e37ae69..4502d59ce8 100644 --- a/cmd/atecontroller/internal/workersync/syncer_test.go +++ b/cmd/atecontroller/internal/workersync/syncer_test.go @@ -19,7 +19,6 @@ import ( "errors" "fmt" "maps" - "slices" "testing" "time" @@ -51,8 +50,9 @@ const ( otherPodUID = "22222222-2222-2222-2222-222222222222" ) -// workerPod builds a pod the informer's label selector matches. An empty ip -// leaves the pod unregistrable, the shape a pod has before its sandbox is up. +// workerPod builds a pod the informer's label selector matches, scheduled to a +// node. An empty ip leaves the pod with no status, so not Ready, the shape a +// pod has before its sandbox is up. func workerPod(ns, name, poolName, uid, ip string) *corev1.Pod { pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -92,14 +92,13 @@ func workerPool(ns, name string, labels map[string]string) *atev1alpha1.WorkerPo // registeredWorker is a Worker in the shape the syncer would have registered // for the given pod, for seeding the fake API before a test runs. -func registeredWorker(ns, poolName, podName, uid, ip string) *ateapipb.Worker { +func registeredWorker(ns, poolName, podName, uid string) *ateapipb.Worker { return &ateapipb.Worker{ Metadata: &ateapipb.ResourceMetadata{Name: uid}, WorkerNamespace: ns, WorkerPool: poolName, WorkerPod: podName, WorkerPodUid: uid, - Ips: []string{ip}, NodeName: "node1", } } @@ -210,8 +209,9 @@ func TestSyncer_Lifecycle(t *testing.T) { api := newFakeControl() fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, map[string]string{"foo": "bar"})) - // A pod with no IP is not registered: there is nothing to route to yet. + // A pod not yet scheduled is not registered: node_name is required. pod := workerPod(ns, podName, poolName, testPodUID, "") + pod.Spec.NodeName = "" if _, err := fakeK8s.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil { t.Fatalf("create pod: %v", err) } @@ -219,24 +219,21 @@ func TestSyncer_Lifecycle(t *testing.T) { // wrong in: the poll only ends early if a worker shows up. err := wait.PollUntilContextTimeout(ctx, 20*time.Millisecond, 300*time.Millisecond, true, func(context.Context) (bool, error) { if got := api.names(); len(got) != 0 { - return false, fmt.Errorf("registry holds %v for a pod with no IP, want it empty", got) + return false, fmt.Errorf("registry holds %v for a pod with no node, want it empty", got) } return false, nil }) if !errors.Is(err, context.DeadlineExceeded) { - t.Fatalf("while checking a pod with no IP stays unregistered: %v", err) + t.Fatalf("while checking a pod with no node stays unregistered: %v", err) } - // Once the pod reports an IP it is registered, with the fields the syncer - // copies off its pool. + // Once the pod is scheduled and Ready it is registered, with the fields the + // syncer copies off its pool. if _, err := fakeK8s.CoreV1().Pods(ns).Update(ctx, workerPod(ns, podName, poolName, testPodUID, "127.0.0.1"), metav1.UpdateOptions{}); err != nil { t.Fatalf("update pod: %v", err) } got := waitForWorker(t, ctx, api, testPodUID, workerActive) - if want := []string{"127.0.0.1"}; !slices.Equal(got.GetIps(), want) { - t.Errorf("worker ips = %q, want %q", got.GetIps(), want) - } if !maps.Equal(got.GetLabels(), map[string]string{"foo": "bar"}) { t.Errorf("worker labels = %v, want map[foo:bar]", got.GetLabels()) } @@ -312,7 +309,7 @@ func TestSyncer_SoftDelete_MarksDraining(t *testing.T) { ns, poolName, podName, ip := "ns-drain", "pool1", "worker-drain", "10.0.0.2" api := newFakeControl() - api.put(registeredWorker(ns, poolName, podName, testPodUID, ip)) + api.put(registeredWorker(ns, poolName, podName, testPodUID)) s, pods, _ := setupReconcileTest(t, api) pod := workerPod(ns, podName, poolName, testPodUID, ip) @@ -335,7 +332,7 @@ func TestSyncer_SoftDelete_NoPodIP(t *testing.T) { ns, poolName, podName := "ns-drain-noip", "pool1", "worker-drain-noip" api := newFakeControl() - api.put(registeredWorker(ns, poolName, podName, testPodUID, "10.0.0.3")) + api.put(registeredWorker(ns, poolName, podName, testPodUID)) s, pods, _ := setupReconcileTest(t, api) // Same pod as the draining test above, minus the pod IP. @@ -365,7 +362,7 @@ func TestMarkWorkerDraining(t *testing.T) { t.Run("already draining returns nil without a write", func(t *testing.T) { api := newFakeControl() - w := registeredWorker(ns, poolName, podName, testPodUID, "10.0.0.4") + w := registeredWorker(ns, poolName, podName, testPodUID) w.Status = &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_DRAINING} seeded := api.put(w) @@ -382,7 +379,7 @@ func TestMarkWorkerDraining(t *testing.T) { t.Run("active worker marked draining", func(t *testing.T) { api := newFakeControl() - api.put(registeredWorker(ns, poolName, podName, testPodUID, "10.0.0.4")) + api.put(registeredWorker(ns, poolName, podName, testPodUID)) s := &WorkerPoolSyncer{client: api} if err := s.markWorkerDraining(ctx, key); err != nil { @@ -404,7 +401,7 @@ func TestReconcileDeadWorker(t *testing.T) { t.Run("registered worker is removed", func(t *testing.T) { api := newFakeControl() - api.put(registeredWorker(ns, poolName, podName, testPodUID, "10.0.0.5")) + api.put(registeredWorker(ns, poolName, podName, testPodUID)) s := &WorkerPoolSyncer{client: api} if err := s.reconcileDeadWorker(ctx, key); err != nil { @@ -433,8 +430,8 @@ func TestSyncer_ReconcileOrphanedWorkers(t *testing.T) { ns, poolName := "ns-recon", "pool1" api := newFakeControl() - api.put(registeredWorker(ns, poolName, "worker-live", testPodUID, "10.0.0.9")) - api.put(registeredWorker(ns, poolName, "worker-orphan", otherPodUID, "10.0.0.10")) + api.put(registeredWorker(ns, poolName, "worker-live", testPodUID)) + api.put(registeredWorker(ns, poolName, "worker-orphan", otherPodUID)) s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) // The cache holds only the live pod, so it is an authoritative snapshot. @@ -464,7 +461,7 @@ func TestSyncer_EnqueueRegisteredWorkers_RetriesTransientListError(t *testing.T) defer cancel() api := newFakeControl() - api.put(registeredWorker("ns-enq-retry", "pool1", "worker-1", testPodUID, "10.0.0.10")) + api.put(registeredWorker("ns-enq-retry", "pool1", "worker-1", testPodUID)) failsLeft := 1 api.setListHook(func(*ateapipb.ListWorkersRequest) error { @@ -520,7 +517,7 @@ func TestSyncer_EnqueueRegisteredWorkers_StreamsPagesAndRetriesLatePage(t *testi "bbbbbbbb-0000-0000-0000-000000000000", "cccccccc-0000-0000-0000-000000000000", } { - api.put(registeredWorker("ns", "p", fmt.Sprintf("w%d", i), uid, "10.0.0.1")) + api.put(registeredWorker("ns", "p", fmt.Sprintf("w%d", i), uid)) } // The second page fails once before serving. @@ -545,21 +542,31 @@ func TestSyncer_EnqueueRegisteredWorkers_StreamsPagesAndRetriesLatePage(t *testi } } -// A dual-stack pod registers every IP Kubernetes reports, in order. -func TestSyncer_CreateWorker_DualStack(t *testing.T) { +// A scheduled pod is registered before it has an IP, and unavailable: its IPs +// are the worker process's to report when it registers, not the syncer's to +// copy off the pod. +func TestSyncer_CreateWorker_BeforePodIP(t *testing.T) { ctx := context.Background() - ns, podName, poolName := "ns-syncer-dual-stack", "worker-unit-dual-stack", "pool-dual-stack" + ns, podName, poolName := "ns-syncer-no-ip", "worker-unit-no-ip", "pool-no-ip" api := newFakeControl() s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) - pod := workerPod(ns, podName, poolName, testPodUID, "fd00::1") - pod.Status.PodIPs = append(pod.Status.PodIPs, corev1.PodIP{IP: "10.0.0.1"}) - key := seedPod(t, pods, pod) + key := seedPod(t, pods, workerPod(ns, podName, poolName, testPodUID, "")) mustReconcile(t, ctx, s, key) - if got, want := api.get(testPodUID).GetIps(), []string{"fd00::1", "10.0.0.1"}; !slices.Equal(got, want) { - t.Fatalf("worker ips = %q, want %q", got, want) + got := api.get(testPodUID) + if got == nil { + t.Fatal("no worker registered for a scheduled pod with no IP") + } + if got.GetNodeName() != "node1" { + t.Errorf("worker node = %q, want node1", got.GetNodeName()) + } + if state := got.GetStatus().GetState(); state != ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE { + t.Errorf("worker state = %v, want %v", state, ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE) + } + if ips := got.GetStatus().GetIps(); len(ips) != 0 { + t.Errorf("worker ips = %q, want none until its worker process registers", ips) } } @@ -595,8 +602,8 @@ func TestSyncer_UpdateWorker_RetryOnVersionConflict(t *testing.T) { key := seedPod(t, pods, workerPod(ns, podName, poolName, testPodUID, "10.0.0.1")) mustReconcile(t, ctx, s, key) - if got, want := api.get(testPodUID).GetIps(), []string{"10.0.0.1"}; !slices.Equal(got, want) { - t.Fatalf("worker ips = %q, want %q", got, want) + if api.get(testPodUID) == nil { + t.Fatal("worker not registered") } // Change a mutable worker field on the pool, so the next reconcile has an @@ -772,7 +779,7 @@ func TestSyncer_DeletedWorkerPool(t *testing.T) { t.Run(tc.name, func(t *testing.T) { ctx := context.Background() api := newFakeControl() - api.put(registeredWorker(ns, poolName, podName, testPodUID, ip)) + api.put(registeredWorker(ns, poolName, podName, testPodUID)) s, pods, _ := setupReconcileTest(t, api) pod := workerPod(ns, podName, poolName, testPodUID, ip) @@ -809,7 +816,7 @@ func TestSyncer_DeletedWorkerPool_NotReadyPod(t *testing.T) { ns, poolName, podName, ip := "ns-deleted-pool-notready", "pool1", "worker-1", "10.0.0.5" api := newFakeControl() - api.put(registeredWorker(ns, poolName, podName, testPodUID, ip)) + api.put(registeredWorker(ns, poolName, podName, testPodUID)) s, pods, _ := setupReconcileTest(t, api) pod := workerPod(ns, podName, poolName, testPodUID, ip) @@ -835,7 +842,7 @@ func TestSyncer_DeletedWorkerPool_Errors(t *testing.T) { ns, poolName, podName, ip := "ns-deleted-pool-err", "pool1", "worker-1", "10.0.0.5" api := newFakeControl() - api.put(registeredWorker(ns, poolName, podName, testPodUID, ip)) + api.put(registeredWorker(ns, poolName, podName, testPodUID)) s, pods, _ := setupReconcileTest(t, api) pod := workerPod(ns, podName, poolName, testPodUID, ip) @@ -924,7 +931,7 @@ func TestSyncer_StartWaitsForWorkerPoolCache(t *testing.T) { poolLabels := map[string]string{"foo": "bar"} api := newFakeControl() - api.put(registeredWorker(ns, poolName, podName, testPodUID, "10.0.0.8")) + api.put(registeredWorker(ns, poolName, podName, testPodUID)) //nolint:staticcheck // NewSimpleClientset is what the informer machinery takes. fakeK8s := fake.NewSimpleClientset(workerPod(ns, podName, poolName, testPodUID, "10.0.0.8")) @@ -1050,7 +1057,7 @@ func TestSyncer_TerminalPod_DeletesPod(t *testing.T) { t.Run(string(phase), func(t *testing.T) { ctx := context.Background() api := newFakeControl() - api.put(registeredWorker(ns, poolName, podName, testPodUID, ip)) + api.put(registeredWorker(ns, poolName, podName, testPodUID)) s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, ip) @@ -1145,8 +1152,8 @@ func TestSyncer_TerminalPod_ViaInformer(t *testing.T) { } // TestSyncer_PodRecreatedWithNewUID verifies that when a pod is deleted and -// recreated under the same name, the registry converges to the new pod's UID -// and IP. A coalesced Delete+Create surfaces as a single update event, which +// recreated under the same name, the registry converges to the new pod's UID. +// A coalesced Delete+Create surfaces as a single update event, which // enqueues both incarnations' keys, so both are reconciled here. func TestSyncer_PodRecreatedWithNewUID(t *testing.T) { ctx := context.Background() @@ -1177,9 +1184,6 @@ func TestSyncer_PodRecreatedWithNewUID(t *testing.T) { if got == nil { t.Fatal("the recreated pod's worker was not registered") } - if want := []string{"10.0.0.8"}; !slices.Equal(got.GetIps(), want) { - t.Errorf("worker ips = %q, want %q", got.GetIps(), want) - } if got.GetWorkerPodUid() != otherPodUID { t.Errorf("worker pod uid = %q, want %q", got.GetWorkerPodUid(), otherPodUID) } @@ -1189,26 +1193,30 @@ func TestSyncer_PodRecreatedWithNewUID(t *testing.T) { } } -// TestSyncer_DeletePodWithoutIP verifies that deleting a pod that never got an -// IP (and so was never registered) is a no-op rather than an error, which is -// what keeps it from error-looping. -func TestSyncer_DeletePodWithoutIP(t *testing.T) { +// TestSyncer_DeleteUnscheduledPod verifies that deleting a pod that was never +// scheduled (and so was never registered) is a no-op rather than an error, +// which is what keeps it from error-looping. +func TestSyncer_DeleteUnscheduledPod(t *testing.T) { ctx := context.Background() - ns, podName, poolName := "ns-syncer-neverip", "worker-noip-1", "pool1" + ns, podName, poolName := "ns-syncer-unscheduled", "worker-unscheduled-1", "pool1" api := newFakeControl() s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "") + pod.Spec.NodeName = "" key := seedPod(t, pods, pod) mustReconcile(t, ctx, s, key) + if got := api.names(); len(got) != 0 { + t.Fatalf("registry holds %v for an unscheduled pod, want it empty", got) + } if err := pods.Delete(pod); err != nil { t.Fatalf("deleting pod: %v", err) } - // The pod never had an IP so nothing was registered, and the delete path is - // an idempotent no-op: no error, so nothing requeues. + // The pod was never scheduled so nothing was registered, and the delete + // path is an idempotent no-op: no error, so nothing requeues. mustReconcile(t, ctx, s, key) if got := api.names(); len(got) != 0 { t.Errorf("registry holds %v, want it empty", got) @@ -1287,10 +1295,10 @@ func TestSyncer_RegisteredPodWithoutIPIsUnavailable(t *testing.T) { api := newFakeControl() s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) - key := seedPod(t, pods, withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.9"), 0)) + key := seedPod(t, pods, workerPod(ns, podName, poolName, testPodUID, "10.0.0.9")) mustReconcile(t, ctx, s, key) - if err := pods.Update(withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, ""), 1)); err != nil { + if err := pods.Update(workerPod(ns, podName, poolName, testPodUID, "")); err != nil { t.Fatalf("updating pod: %v", err) } mustReconcile(t, ctx, s, key) @@ -1298,12 +1306,6 @@ func TestSyncer_RegisteredPodWithoutIPIsUnavailable(t *testing.T) { if got.GetStatus().GetState() != ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE { t.Errorf("state = %v, want %v", got.GetStatus().GetState(), ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE) } - if got.GetEpoch() != 2 { - t.Errorf("epoch = %d, want 2", got.GetEpoch()) - } - if want := []string{"10.0.0.9"}; !slices.Equal(got.GetIps(), want) { - t.Errorf("ips = %v, want the registered %v kept", got.GetIps(), want) - } } // TestSyncer_ReadinessFlapDoesNotDeregister pins that once a worker exists, a diff --git a/cmd/atelet/internal/apivalidation/validate.go b/cmd/atelet/internal/apivalidation/validate.go index 19ec1e6877..6c5d8f7624 100644 --- a/cmd/atelet/internal/apivalidation/validate.go +++ b/cmd/atelet/internal/apivalidation/validate.go @@ -74,6 +74,11 @@ func ValidateDetachActorRequest(ctx context.Context, req *ateletpb.DetachActorRe return Validate_DetachActorRequest(ctx, operation.Operation{Type: operation.Create}, nil, req, nil) } +// ValidateSetWorkerSandboxRuntimesRequest runs the generated validation for req. +func ValidateSetWorkerSandboxRuntimesRequest(ctx context.Context, req *ateletpb.SetWorkerSandboxRuntimesRequest) field.ErrorList { + return Validate_SetWorkerSandboxRuntimesRequest(ctx, operation.Operation{Type: operation.Create}, nil, req, nil) +} + // ValidateRegisterWorkerRequest runs the generated validation for req. func ValidateRegisterWorkerRequest(ctx context.Context, req *ateletpb.RegisterWorkerRequest) field.ErrorList { return Validate_RegisterWorkerRequest(ctx, operation.Operation{Type: operation.Create}, nil, req, nil) diff --git a/cmd/atelet/internal/apivalidation/validation_test.go b/cmd/atelet/internal/apivalidation/validation_test.go index 6276cf600f..1795e48ab1 100644 --- a/cmd/atelet/internal/apivalidation/validation_test.go +++ b/cmd/atelet/internal/apivalidation/validation_test.go @@ -1014,7 +1014,7 @@ func TestValidateDetachActorRequest(t *testing.T) { } } -func TestValidateRegisterWorkerRequest(t *testing.T) { +func TestValidateSetWorkerSandboxRuntimesRequest(t *testing.T) { testCompat := func() *ateletpb.VersionedSandboxCompat { return &ateletpb.VersionedSandboxCompat{ SchemaVersion: "v1", @@ -1031,20 +1031,20 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { return &ateletpb.WorkerSandboxClass{SandboxClass: "gvisor", DefaultRuntime: testRuntime(testCompat())} } // withClass returns a request reporting c as its only sandbox class. - withClass := func(c *ateletpb.WorkerSandboxClass) *ateletpb.RegisterWorkerRequest { - return &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, + withClass := func(c *ateletpb.WorkerSandboxClass) *ateletpb.SetWorkerSandboxRuntimesRequest { + return &ateletpb.SetWorkerSandboxRuntimesRequest{ + Epoch: 1, SandboxClasses: []*ateletpb.WorkerSandboxClass{c}, } } // withDefaultRuntime returns a request whose gVisor class cold-boots // under compat. - withDefaultRuntime := func(compat *ateletpb.VersionedSandboxCompat) *ateletpb.RegisterWorkerRequest { + withDefaultRuntime := func(compat *ateletpb.VersionedSandboxCompat) *ateletpb.SetWorkerSandboxRuntimesRequest { return withClass(&ateletpb.WorkerSandboxClass{SandboxClass: "gvisor", DefaultRuntime: testRuntime(compat)}) } // withRestorable returns a request whose gVisor class can also restore // compats. - withRestorable := func(compats ...*ateletpb.VersionedSandboxCompat) *ateletpb.RegisterWorkerRequest { + withRestorable := func(compats ...*ateletpb.VersionedSandboxCompat) *ateletpb.SetWorkerSandboxRuntimesRequest { c := testClass() for _, compat := range compats { var rt *ateletpb.WorkerSandboxRuntime @@ -1055,13 +1055,6 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { } return withClass(c) } - withCapacity := func(capacity *ateletpb.WorkerResources) *ateletpb.RegisterWorkerRequest { - return &ateletpb.RegisterWorkerRequest{ - Capacity: capacity, - SandboxClasses: []*ateletpb.WorkerSandboxClass{testClass()}, - } - } - capacityPath := field.NewPath("capacity") classesPath := field.NewPath("sandbox_classes") classPath := classesPath.Index(0) defaultRuntimePath := classPath.Child("default_runtime") @@ -1070,7 +1063,7 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { tests := []struct { name string - obj *ateletpb.RegisterWorkerRequest + obj *ateletpb.SetWorkerSandboxRuntimesRequest want field.ErrorList }{{ name: "valid", @@ -1079,24 +1072,24 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { name: "valid with restorable runtimes", obj: withRestorable(testCompat()), }, { - name: "missing capacity", - obj: withCapacity(nil), - want: field.ErrorList{field.Required(field.NewPath("capacity"), "")}, + name: "missing epoch", + obj: &ateletpb.SetWorkerSandboxRuntimesRequest{SandboxClasses: []*ateletpb.WorkerSandboxClass{testClass()}}, + want: field.ErrorList{field.Required(field.NewPath("epoch"), "")}, }, { name: "missing sandbox_classes", - obj: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{}}, + obj: &ateletpb.SetWorkerSandboxRuntimesRequest{Epoch: 1}, want: field.ErrorList{field.Required(classesPath, "")}, }, { name: "empty sandbox_classes", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, + obj: &ateletpb.SetWorkerSandboxRuntimesRequest{ + Epoch: 1, SandboxClasses: []*ateletpb.WorkerSandboxClass{}, }, want: field.ErrorList{field.Required(classesPath, "")}, }, { name: "too many sandbox_classes", - obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{}, + obj: &ateletpb.SetWorkerSandboxRuntimesRequest{ + Epoch: 1, SandboxClasses: []*ateletpb.WorkerSandboxClass{ testClass(), {SandboxClass: "microvm", DefaultRuntime: testRuntime(testCompat())}, @@ -1205,15 +1198,61 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { name: "too many restorable_runtimes", obj: withRestorable(sandboxCompats(33, testCompat)...), want: field.ErrorList{field.TooMany(restorablePath, 33, 32).WithOrigin("maxItems")}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assertValidateErr(t, ValidateSetWorkerSandboxRuntimesRequest(context.Background(), tt.obj), tt.want) + }) + } +} + +func TestValidateRegisterWorkerRequest(t *testing.T) { + ips := []string{"10.0.0.1"} + withCapacity := func(c *ateletpb.WorkerResources) *ateletpb.RegisterWorkerRequest { + return &ateletpb.RegisterWorkerRequest{Epoch: 1, Ips: ips, Capacity: c} + } + capacityPath := field.NewPath("capacity") + + tests := []struct { + name string + obj *ateletpb.RegisterWorkerRequest + want field.ErrorList + }{{ + name: "valid", + obj: &ateletpb.RegisterWorkerRequest{Epoch: 1, Ips: ips, Capacity: &ateletpb.WorkerResources{}}, + }, { + name: "dual-stack IPs", + obj: &ateletpb.RegisterWorkerRequest{Epoch: 1, Ips: []string{"10.0.0.1", "fd00::1"}, Capacity: &ateletpb.WorkerResources{}}, + }, { + name: "missing capacity", + obj: &ateletpb.RegisterWorkerRequest{Epoch: 1, Ips: ips}, + want: field.ErrorList{field.Required(field.NewPath("capacity"), "")}, + }, { + name: "missing epoch", + obj: &ateletpb.RegisterWorkerRequest{Ips: ips, Capacity: &ateletpb.WorkerResources{}}, + want: field.ErrorList{field.Required(field.NewPath("epoch"), "")}, + }, { + name: "negative epoch", + obj: &ateletpb.RegisterWorkerRequest{Epoch: -1, Ips: ips, Capacity: &ateletpb.WorkerResources{}}, + want: field.ErrorList{field.Invalid(field.NewPath("epoch"), nil, "").WithOrigin("minimum")}, + }, { + name: "missing ips", + obj: &ateletpb.RegisterWorkerRequest{Epoch: 1, Capacity: &ateletpb.WorkerResources{}}, + want: field.ErrorList{field.Required(field.NewPath("ips"), "")}, + }, { + name: "too many ips", + obj: &ateletpb.RegisterWorkerRequest{Epoch: 1, Ips: []string{"10.0.0.1", "fd00::1", "10.0.0.2"}, Capacity: &ateletpb.WorkerResources{}}, + want: field.ErrorList{field.TooMany(field.NewPath("ips"), 3, 2).WithOrigin("maxItems")}, }, { name: "full capacity", obj: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{Actors: 4, CpuMillicores: 4000, MemoryBytes: 8 << 30}, - SandboxClasses: []*ateletpb.WorkerSandboxClass{testClass()}, + Epoch: 1, + Ips: ips, + Capacity: &ateletpb.WorkerResources{Actors: 4, CpuMillicores: 4000, MemoryBytes: 8 << 30}, }, }, { name: "negative actors", - obj: withCapacity(&ateletpb.WorkerResources{Actors: -1}), + obj: &ateletpb.RegisterWorkerRequest{Epoch: 1, Ips: ips, Capacity: &ateletpb.WorkerResources{Actors: -1}}, want: field.ErrorList{field.Invalid(field.NewPath("capacity", "actors"), nil, "").WithOrigin("minimum")}, }, { name: "negative cpu_millicores", diff --git a/cmd/atelet/internal/apivalidation/zz_generated.validation.go b/cmd/atelet/internal/apivalidation/zz_generated.validation.go index 62e02b5a9a..a312fc66a6 100644 --- a/cmd/atelet/internal/apivalidation/zz_generated.validation.go +++ b/cmd/atelet/internal/apivalidation/zz_generated.validation.go @@ -2025,41 +2025,42 @@ func Validate_RegisterWorkerRequest( ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *ateletpb.RegisterWorkerRequest) (errs field.ErrorList) { - { // field ateletpb.RegisterWorkerRequest.Capacity + { // field ateletpb.RegisterWorkerRequest.Epoch fn := func( fldPath *field.Path, - obj, oldObj *ateletpb.WorkerResources, + obj, oldObj *int64, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { return nil } } // call field-attached validations earlyReturn := false - if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } if earlyReturn { return // do not proceed } - // call the type's validation function - errs = append(errs, Validate_WorkerResources(ctx, op, fldPath, obj, oldObj)...) + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } return } oldVal := safe.Field(oldObj, - func(oldObj *ateletpb.RegisterWorkerRequest) *ateletpb.WorkerResources { - return oldObj.Capacity + func(oldObj *ateletpb.RegisterWorkerRequest) *int64 { + return &oldObj.Epoch }) - errs = append(errs, fn(fldPath.Child("capacity"), obj.Capacity, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("epoch"), &obj.Epoch, oldVal, oldObj != nil)...) } - { // field ateletpb.RegisterWorkerRequest.SandboxClasses + { // field ateletpb.RegisterWorkerRequest.Ips fn := func( fldPath *field.Path, - obj, oldObj []*ateletpb.WorkerSandboxClass, + obj, oldObj []string, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -2069,11 +2070,7 @@ func Validate_RegisterWorkerRequest( } // call field-attached validations earlyReturn := false - if e := validate.PtrSliceNoNils[ateletpb.WorkerSandboxClass](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 1).MarkShortCircuit(); len(e) != 0 { + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 2).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } @@ -2084,30 +2081,44 @@ func Validate_RegisterWorkerRequest( if earlyReturn { return // do not proceed } - if e := validate.MinItems(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { - errs = append(errs, e...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.RegisterWorkerRequest) []string { + return oldObj.Ips + }) + errs = append(errs, fn(fldPath.Child("ips"), obj.Ips, oldVal, oldObj != nil)...) + } + + { // field ateletpb.RegisterWorkerRequest.Capacity + fn := func( + fldPath *field.Path, + obj, oldObj *ateletpb.WorkerResources, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } } - // lists with map semantics require unique keys - if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, - func(a *ateletpb.WorkerSandboxClass, b *ateletpb.WorkerSandboxClass) bool { - return a.SandboxClass == b.SandboxClass - }); len(e) != 0 { + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) + earlyReturn = true } - // iterate the list and call the type's validation function - if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, - func(a *ateletpb.WorkerSandboxClass, b *ateletpb.WorkerSandboxClass) bool { - return a.SandboxClass == b.SandboxClass - }, ateDeepEqual, Validate_WorkerSandboxClass); len(e) != 0 { - errs = append(errs, e...) + if earlyReturn { + return // do not proceed } + // call the type's validation function + errs = append(errs, Validate_WorkerResources(ctx, op, fldPath, obj, oldObj)...) return } oldVal := safe.Field(oldObj, - func(oldObj *ateletpb.RegisterWorkerRequest) []*ateletpb.WorkerSandboxClass { - return oldObj.SandboxClasses + func(oldObj *ateletpb.RegisterWorkerRequest) *ateletpb.WorkerResources { + return oldObj.Capacity }) - errs = append(errs, fn(fldPath.Child("sandbox_classes"), obj.SandboxClasses, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("capacity"), obj.Capacity, oldVal, oldObj != nil)...) } return errs @@ -3180,6 +3191,101 @@ func Validate_SecurityContext( return errs } +// Validate_SetWorkerSandboxRuntimesRequest validates an instance of SetWorkerSandboxRuntimesRequest according +// to declarative validation rules in the API schema. +func Validate_SetWorkerSandboxRuntimesRequest( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateletpb.SetWorkerSandboxRuntimesRequest) (errs field.ErrorList) { + + { // field ateletpb.SetWorkerSandboxRuntimesRequest.Epoch + fn := func( + fldPath *field.Path, + obj, oldObj *int64, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.SetWorkerSandboxRuntimesRequest) *int64 { + return &oldObj.Epoch + }) + errs = append(errs, fn(fldPath.Child("epoch"), &obj.Epoch, oldVal, oldObj != nil)...) + } + + { // field ateletpb.SetWorkerSandboxRuntimesRequest.SandboxClasses + fn := func( + fldPath *field.Path, + obj, oldObj []*ateletpb.WorkerSandboxClass, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateletpb.WorkerSandboxClass](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 1).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.RequiredSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MinItems(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + // lists with map semantics require unique keys + if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, + func(a *ateletpb.WorkerSandboxClass, b *ateletpb.WorkerSandboxClass) bool { + return a.SandboxClass == b.SandboxClass + }); len(e) != 0 { + errs = append(errs, e...) + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, + func(a *ateletpb.WorkerSandboxClass, b *ateletpb.WorkerSandboxClass) bool { + return a.SandboxClass == b.SandboxClass + }, ateDeepEqual, Validate_WorkerSandboxClass); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.SetWorkerSandboxRuntimesRequest) []*ateletpb.WorkerSandboxClass { + return oldObj.SandboxClasses + }) + errs = append(errs, fn(fldPath.Child("sandbox_classes"), obj.SandboxClasses, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_Snapshot validates an instance of Snapshot according // to declarative validation rules in the API schema. func Validate_Snapshot( diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index 2b5c55e8ed..7bc276da1e 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -90,6 +90,8 @@ type fakeWorker struct { // preserveRestoreDir records the PreserveRestoreDir flag from the most // recent RestoreWorkload request. preserveRestoreDir bool + // workerEpoch is the epoch RunWorkload and RestoreWorkload report. + workerEpoch int64 // fidelities records the fidelity each checkpoint or restore arrived // with, by RPC name. fidelities map[string]ateworkerpb.SnapshotFidelity @@ -126,7 +128,7 @@ func (f *fakeWorker) recordActorDirs(rpc string, actorDirs *ateworkerpb.ActorDir func (f *fakeWorker) RunWorkload(_ context.Context, req *ateworkerpb.RunWorkloadRequest) (*ateworkerpb.RunWorkloadResponse, error) { f.recordActorDirs("RunWorkload", req.GetActorDirs()) - return &ateworkerpb.RunWorkloadResponse{}, nil + return &ateworkerpb.RunWorkloadResponse{WorkerEpoch: f.workerEpoch}, nil } func (f *fakeWorker) CheckpointWorkload(_ context.Context, req *ateworkerpb.CheckpointWorkloadRequest) (*ateworkerpb.CheckpointWorkloadResponse, error) { @@ -165,7 +167,7 @@ func (f *fakeWorker) RestoreWorkload(_ context.Context, req *ateworkerpb.Restore } f.restored[name] = string(body) } - return &ateworkerpb.RestoreWorkloadResponse{}, nil + return &ateworkerpb.RestoreWorkloadResponse{WorkerEpoch: f.workerEpoch}, nil } func (f *fakeWorker) TerminateWorkload(_ context.Context, req *ateworkerpb.TerminateWorkloadRequest) (*ateworkerpb.TerminateWorkloadResponse, error) { @@ -247,7 +249,8 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity, useDir snapshotName2 = "pause-snap-2" ) - worker := &fakeWorker{snapshotFiles: map[string]string{"checkpoint.img": "guest-memory"}} + const workerEpoch = 7 + worker := &fakeWorker{snapshotFiles: map[string]string{"checkpoint.img": "guest-memory"}, workerEpoch: workerEpoch} serveFakeWorker(t, worker) host := imageVolumeTestRegistry(t) @@ -269,7 +272,7 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity, useDir } advertiseRuntime(t, s, sandboxRec) - if _, err := s.Run(ctx, &ateletpb.RunRequest{ + runResp, err := s.Run(ctx, &ateletpb.RunRequest{ Atespace: atespace, ActorName: actorName, ActorUid: actorUID, @@ -278,9 +281,15 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity, useDir WorkerPodUid: workerPodUID, SandboxRuntime: runtimeFor(t, sandboxRec), Spec: spec, - }); err != nil { + }) + if err != nil { t.Fatalf("Run: %v", err) } + // The control plane stamps the Actor with the epoch of the worker process + // that started it. + if got := runResp.GetWorkerEpoch(); got != workerEpoch { + t.Errorf("Run worker_epoch = %d, want %d", got, workerEpoch) + } // Async suspend: a deferred checkpoint, which leaves the snapshot on this node. resp, err := s.Checkpoint(ctx, &ateletpb.CheckpointRequest{ @@ -316,7 +325,7 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity, useDir // Resume: atelet must restore from that local snapshot, not from // Snapshot.Object. - if _, err := s.Restore(ctx, &ateletpb.RestoreRequest{ + restoreResp, err := s.Restore(ctx, &ateletpb.RestoreRequest{ Atespace: atespace, ActorName: actorName, ActorUid: actorUID, @@ -326,9 +335,13 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity, useDir SandboxRuntime: runtimeFor(t, sandboxRec), Spec: spec, Snapshot: lifecycleSnapshot(actorUID, snapshotName, 1, fidelity), - }); err != nil { + }) + if err != nil { t.Fatalf("Restore: %v", err) } + if got := restoreResp.GetWorkerEpoch(); got != workerEpoch { + t.Errorf("Restore worker_epoch = %d, want %d", got, workerEpoch) + } if got := worker.restored["checkpoint.img"]; got != "guest-memory" { t.Fatalf("restore staged %q for worker, want the pause snapshot's %q", got, "guest-memory") } diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index f638e938b7..d935c6df09 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -886,7 +886,7 @@ func (s *Atelet) Run(ctx context.Context, req *ateletpb.RunRequest) (resp *atele // Tell worker to start the workload. gVisor uses RunscPath; the micro-VM // runtime uses the full RuntimeAssetPaths set. - if _, err := client.RunWorkload(ctx, &ateworkerpb.RunWorkloadRequest{ + workerResp, err := client.RunWorkload(ctx, &ateworkerpb.RunWorkloadRequest{ Atespace: actorRef.Atespace, ActorName: actorRef.Name, ActorTemplateAtespace: req.GetActorTemplateAtespace(), @@ -898,11 +898,12 @@ func (s *Atelet) Run(ctx context.Context, req *ateletpb.RunRequest) (resp *atele ActorDirs: ateletpath.ActorDirs(actorUID), EgressGateway: toWorkerEgressGateway(req.GetEgressGateway()), Resources: toAteworkerResources(req.GetResources()), - }); err != nil { - return nil, fmt.Errorf("while calling worker RunWorkload: %w", err) + }) + if err != nil { + return nil, workerCallError("RunWorkload", err) } - return &ateletpb.RunResponse{}, nil + return &ateletpb.RunResponse{WorkerEpoch: workerResp.GetWorkerEpoch()}, nil } var snapshotSizeBytes metric.Int64Histogram @@ -1808,7 +1809,7 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res // The worker_restore phase is opaque from here; worker logs its own breakdown of // this call as "Actor restore phases". tWorker := time.Now() - _, err = client.RestoreWorkload(ctx, &ateworkerpb.RestoreWorkloadRequest{ + workerResp, err := client.RestoreWorkload(ctx, &ateworkerpb.RestoreWorkloadRequest{ Atespace: actorRef.Atespace, ActorName: actorRef.Name, ActorTemplateAtespace: req.GetActorTemplateAtespace(), @@ -1828,7 +1829,18 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res return nil, s.boundRetryableRestoreError(actorUID, req.GetSnapshot().GetSnapshotUuid(), workerRestoreError(ctx, err)) } - return &ateletpb.RestoreResponse{}, nil + return &ateletpb.RestoreResponse{WorkerEpoch: workerResp.GetWorkerEpoch()}, nil +} + +// workerCallError is atelet's error for a failed call to the worker's rpc. A +// worker that is unavailable, refusing workloads until it registers or gone +// with a restart, is reported as Unavailable, so the caller retries once the +// worker process has registered. Anything else reaches the caller as Internal. +func workerCallError(rpc string, err error) error { + if status.Code(err) == codes.Unavailable { + return apierror.Unavailable("worker %s: the worker is unavailable: %s", rpc, status.Convert(err).Message()) + } + return fmt.Errorf("while calling worker %s: %w", rpc, err) } // DetachActor makes the actor's latest local snapshot durable, then deletes the diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index c028be60b3..4da514ea30 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -1654,3 +1654,25 @@ func TestServingCheck(t *testing.T) { t.Errorf("servingCheck after the port opened = %v, want nil", err) } } + +// A worker that is unavailable, whether refusing workloads until it registers +// or unreachable after a restart, reaches atelet's caller as Unavailable so the +// resume is retried. Any other worker failure is Internal. +func TestWorkerCallError(t *testing.T) { + for _, tc := range []struct { + name string + err error + want codes.Code + }{ + {name: "worker refuses", err: status.Error(codes.Unavailable, "worker has not registered yet"), want: codes.Unavailable}, + {name: "worker unreachable", err: fmt.Errorf("dial: %w", status.Error(codes.Unavailable, "connection refused")), want: codes.Unavailable}, + {name: "worker fails", err: status.Error(codes.Internal, "sandbox would not start"), want: codes.Internal}, + {name: "plain error", err: errors.New("boom"), want: codes.Internal}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := apierror.Code(workerCallError("RestoreWorkload", tc.err)); got != tc.want { + t.Errorf("workerCallError() code = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/cmd/atelet/workersupport.go b/cmd/atelet/workersupport.go index 85cdc2e724..e852af7dd7 100644 --- a/cmd/atelet/workersupport.go +++ b/cmd/atelet/workersupport.go @@ -102,17 +102,36 @@ func verifyClientOnSameNode(node *substratex509.PodIdentity) func(tls.Connection } } -// RegisterWorker registers the calling worker with the control plane: what the -// worker says its capacity and sandbox runtimes are, in one -// WorkerService.RegisterWorker call so capacity and runtimes land atomically. +// AcquireWorkerEpoch hands the calling worker process a new epoch from the +// control plane's WorkerService.AcquireWorkerEpoch. // -// It returns the control plane's error unwrapped so the caller retries: an -// accepted call is the only thing that puts capacity and the current runtimes -// on the Worker, and a Worker record the syncer has not created yet is the -// ordinary reason for a first attempt to fail. +// Each of the registration calls below returns the control plane's error +// unwrapped, so the caller can tell a refusal from a failure to retry: a Worker +// record the syncer has not created yet is the ordinary reason for a first +// attempt to fail. +func (s *workerSupportServer) AcquireWorkerEpoch(ctx context.Context, _ *ateletpb.AcquireWorkerEpochRequest) (*ateletpb.AcquireWorkerEpochResponse, error) { + // Identity comes only from the mTLS certificate, never from the request: + // a worker can acquire an epoch for itself and no one else. + workerIdentity, err := authenticatedWorkerIdentity(ctx) + if err != nil { + return nil, err + } + resp, err := s.workers.AcquireWorkerEpoch(ctx, &ateapipb.AcquireWorkerEpochRequest{ + // Workers are global-scoped and named by their pod UID. + Worker: &ateapipb.ObjectRef{Name: workerIdentity.PodUID}, + }) + if err != nil { + return nil, err + } + return &ateletpb.AcquireWorkerEpochResponse{Epoch: resp.GetEpoch()}, nil +} + +// RegisterWorker registers the calling worker process with the control plane: +// it activates the epoch the process acquired and records its pod IPs and +// capacity, in one WorkerService.RegisterWorker call. func (s *workerSupportServer) RegisterWorker(ctx context.Context, req *ateletpb.RegisterWorkerRequest) (*ateletpb.RegisterWorkerResponse, error) { // Identity comes only from the mTLS certificate, never from the request: - // a worker can report its own capacity and runtimes and no one else's. + // a worker can register itself and no one else. workerIdentity, err := authenticatedWorkerIdentity(ctx) if err != nil { return nil, err @@ -125,17 +144,45 @@ func (s *workerSupportServer) RegisterWorker(ctx context.Context, req *ateletpb. } if _, err := s.workers.RegisterWorker(ctx, &ateapipb.RegisterWorkerRequest{ // Workers are global-scoped and named by their pod UID. + Worker: &ateapipb.ObjectRef{Name: workerIdentity.PodUID}, + Epoch: req.GetEpoch(), + Ips: req.GetIps(), + Capacity: toWorkerResources(req.GetCapacity()), + }); err != nil { + return nil, err + } + slog.InfoContext(ctx, "Registered worker", + slog.String("pod_uid", workerIdentity.PodUID), + slog.Int64("epoch", req.GetEpoch()), + slog.Any("ips", req.GetIps()), + slog.Any("capacity", req.GetCapacity())) + return &ateletpb.RegisterWorkerResponse{}, nil +} + +// SetWorkerSandboxRuntimes records the sandbox runtimes of the calling worker +// process through WorkerService.SetWorkerSandboxRuntimes. +func (s *workerSupportServer) SetWorkerSandboxRuntimes(ctx context.Context, req *ateletpb.SetWorkerSandboxRuntimesRequest) (*ateletpb.SetWorkerSandboxRuntimesResponse, error) { + // Identity comes only from the mTLS certificate, never from the request: + // a worker can report its own runtimes and no one else's. + workerIdentity, err := authenticatedWorkerIdentity(ctx) + if err != nil { + return nil, err + } + if errs := apivalidation.ValidateSetWorkerSandboxRuntimesRequest(ctx, req); len(errs) > 0 { + return nil, resources.ToGRPCStatusError(errs) + } + if _, err := s.workers.SetWorkerSandboxRuntimes(ctx, &ateapipb.SetWorkerSandboxRuntimesRequest{ Worker: &ateapipb.ObjectRef{Name: workerIdentity.PodUID}, - Capacity: toWorkerResources(req.GetCapacity()), + Epoch: req.GetEpoch(), SandboxClasses: toAteAPISandboxClasses(req.GetSandboxClasses()), }); err != nil { return nil, err } - slog.InfoContext(ctx, "Registered worker capacity and sandbox classes", + slog.InfoContext(ctx, "Recorded worker sandbox runtimes", slog.String("pod_uid", workerIdentity.PodUID), - slog.Any("capacity", req.GetCapacity()), + slog.Int64("epoch", req.GetEpoch()), slog.Any("sandbox_classes", req.GetSandboxClasses())) - return &ateletpb.RegisterWorkerResponse{}, nil + return &ateletpb.SetWorkerSandboxRuntimesResponse{}, nil } // toWorkerResources converts atelet's WorkerResources to the control plane's, diff --git a/cmd/atelet/workersupport_test.go b/cmd/atelet/workersupport_test.go index f649c9e899..7913ae973b 100644 --- a/cmd/atelet/workersupport_test.go +++ b/cmd/atelet/workersupport_test.go @@ -88,10 +88,21 @@ func workerCertificate(t *testing.T, podUID, nodeName string) *x509.Certificate type fakeWorkerService struct { ateapipb.WorkerServiceClient - got []*ateapipb.RegisterWorkerRequest - mintGot []*ateapipb.MintWorkerActorCertificateRequest - mintResp *ateapipb.MintWorkerActorCertificateResponse - err error + got []*ateapipb.RegisterWorkerRequest + acquireGot []*ateapipb.AcquireWorkerEpochRequest + runtimesGot []*ateapipb.SetWorkerSandboxRuntimesRequest + epoch int64 + mintGot []*ateapipb.MintWorkerActorCertificateRequest + mintResp *ateapipb.MintWorkerActorCertificateResponse + err error +} + +func (s *fakeWorkerService) AcquireWorkerEpoch(_ context.Context, in *ateapipb.AcquireWorkerEpochRequest, _ ...grpc.CallOption) (*ateapipb.AcquireWorkerEpochResponse, error) { + if s.err != nil { + return nil, s.err + } + s.acquireGot = append(s.acquireGot, in) + return &ateapipb.AcquireWorkerEpochResponse{Epoch: s.epoch}, nil } func (s *fakeWorkerService) RegisterWorker(_ context.Context, in *ateapipb.RegisterWorkerRequest, _ ...grpc.CallOption) (*ateapipb.RegisterWorkerResponse, error) { @@ -102,6 +113,14 @@ func (s *fakeWorkerService) RegisterWorker(_ context.Context, in *ateapipb.Regis return &ateapipb.RegisterWorkerResponse{}, nil } +func (s *fakeWorkerService) SetWorkerSandboxRuntimes(_ context.Context, in *ateapipb.SetWorkerSandboxRuntimesRequest, _ ...grpc.CallOption) (*ateapipb.SetWorkerSandboxRuntimesResponse, error) { + if s.err != nil { + return nil, s.err + } + s.runtimesGot = append(s.runtimesGot, in) + return &ateapipb.SetWorkerSandboxRuntimesResponse{}, nil +} + func (s *fakeWorkerService) MintWorkerActorCertificate(_ context.Context, in *ateapipb.MintWorkerActorCertificateRequest, _ ...grpc.CallOption) (*ateapipb.MintWorkerActorCertificateResponse, error) { if s.err != nil { return nil, s.err @@ -113,7 +132,137 @@ func (s *fakeWorkerService) MintWorkerActorCertificate(_ context.Context, in *at return &ateapipb.MintWorkerActorCertificateResponse{}, nil } +func TestAcquireWorkerEpoch(t *testing.T) { + tests := []struct { + name string + unauthenticated bool + serviceErr error + wantCode codes.Code + wantEpoch int64 + wantForwarded []*ateapipb.AcquireWorkerEpochRequest + }{{ + // The Worker is named after the worker pod UID from the certificate. + name: "acquires for the calling worker", + wantEpoch: 7, + wantForwarded: []*ateapipb.AcquireWorkerEpochRequest{{Worker: &ateapipb.ObjectRef{Name: "pod-a"}}}, + }, { + name: "requires a certificate", + unauthenticated: true, + wantCode: codes.Unauthenticated, + }, { + name: "surfaces the control plane's rejection", + serviceErr: status.Error(codes.NotFound, "no such worker"), + wantCode: codes.NotFound, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + workers := &fakeWorkerService{err: tt.serviceErr, epoch: 7} + svc := &workerSupportServer{workers: workers} + ctx := workerContext(t, "pod-a") + if tt.unauthenticated { + ctx = context.Background() + } + + resp, err := svc.AcquireWorkerEpoch(ctx, &ateletpb.AcquireWorkerEpochRequest{}) + if got := status.Code(err); got != tt.wantCode { + t.Errorf("AcquireWorkerEpoch() code = %v (%v), want %v", got, err, tt.wantCode) + } + if got := resp.GetEpoch(); got != tt.wantEpoch { + t.Errorf("AcquireWorkerEpoch() epoch = %d, want %d", got, tt.wantEpoch) + } + if diff := cmp.Diff(tt.wantForwarded, workers.acquireGot, protocmp.Transform()); diff != "" { + t.Errorf("forwarded requests mismatch (-want +got):\n%s", diff) + } + }) + } +} + func TestRegisterWorker(t *testing.T) { + ips := []string{"10.0.0.1"} + forwarded := func(capacity *ateapipb.WorkerResources) []*ateapipb.RegisterWorkerRequest { + // The Worker is named after the worker pod UID, taken from the + // certificate rather than the request; epoch, IPs and capacity come + // from what the worker reported. + return []*ateapipb.RegisterWorkerRequest{{ + Worker: &ateapipb.ObjectRef{Name: "pod-a"}, + Epoch: 3, + Ips: ips, + Capacity: capacity, + }} + } + + tests := []struct { + name string + // unauthenticated drops the peer certificate from the context. + unauthenticated bool + // serviceErr is what the control plane answers with. + serviceErr error + req *ateletpb.RegisterWorkerRequest + wantCode codes.Code + wantForwarded []*ateapipb.RegisterWorkerRequest + }{{ + name: "records what the worker says", + req: &ateletpb.RegisterWorkerRequest{ + Epoch: 3, + Ips: ips, + Capacity: &ateletpb.WorkerResources{Actors: 4, CpuMillicores: 2000, MemoryBytes: 4 << 30}, + }, + wantForwarded: forwarded(&ateapipb.WorkerResources{Actors: 4, CpuMillicores: 2000, MemoryBytes: 4294967296}), + }, { + name: "omits undetermined compute", + req: &ateletpb.RegisterWorkerRequest{Epoch: 3, Ips: ips, Capacity: &ateletpb.WorkerResources{Actors: 1}}, + wantForwarded: forwarded(&ateapipb.WorkerResources{Actors: 1}), + }, { + name: "forwards empty capacity", + req: &ateletpb.RegisterWorkerRequest{Epoch: 3, Ips: ips, Capacity: &ateletpb.WorkerResources{}}, + wantForwarded: forwarded(&ateapipb.WorkerResources{}), + }, { + name: "rejects invalid capacity without forwarding", + req: &ateletpb.RegisterWorkerRequest{ + Epoch: 3, + Ips: ips, + Capacity: &ateletpb.WorkerResources{CpuMillicores: -1}, + }, + wantCode: codes.InvalidArgument, + }, { + name: "rejects missing capacity without forwarding", + req: &ateletpb.RegisterWorkerRequest{Epoch: 3, Ips: ips}, + wantCode: codes.InvalidArgument, + }, { + // A worker may report only what its certificate proves it is. + name: "requires a certificate", + unauthenticated: true, + req: &ateletpb.RegisterWorkerRequest{Epoch: 3, Ips: ips, Capacity: &ateletpb.WorkerResources{Actors: 1}}, + wantCode: codes.Unauthenticated, + }, { + // The Worker record may not exist yet. The error must reach the + // worker so it retries. + name: "surfaces the control plane's rejection", + serviceErr: errors.New("no such worker"), + req: &ateletpb.RegisterWorkerRequest{Epoch: 3, Ips: ips, Capacity: &ateletpb.WorkerResources{Actors: 1}}, + wantCode: codes.Unknown, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + workers := &fakeWorkerService{err: tt.serviceErr} + svc := &workerSupportServer{workers: workers} + ctx := workerContext(t, "pod-a") + if tt.unauthenticated { + ctx = context.Background() + } + + _, err := svc.RegisterWorker(ctx, tt.req) + if got := status.Code(err); got != tt.wantCode { + t.Errorf("RegisterWorker() code = %v (%v), want %v", got, err, tt.wantCode) + } + if diff := cmp.Diff(tt.wantForwarded, workers.got, protocmp.Transform()); diff != "" { + t.Errorf("forwarded requests mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestSetWorkerSandboxRuntimes(t *testing.T) { reqCompat := func(hash string) *ateletpb.VersionedSandboxCompat { return &ateletpb.VersionedSandboxCompat{ SchemaVersion: "v1", @@ -145,10 +294,10 @@ func TestRegisterWorker(t *testing.T) { } return []*ateletpb.WorkerSandboxClass{c} } - forwarded := func(capacity *ateapipb.WorkerResources, restorable ...*ateapipb.VersionedSandboxCompat) []*ateapipb.RegisterWorkerRequest { + forwarded := func(restorable ...*ateapipb.VersionedSandboxCompat) []*ateapipb.SetWorkerSandboxRuntimesRequest { // The Worker is named after the worker pod UID, taken from the - // certificate rather than the request; capacity and sandbox classes - // come from what the worker reported. + // certificate rather than the request; epoch and sandbox classes come + // from what the worker reported. c := &ateapipb.WorkerSandboxClass{ SandboxClass: "gvisor", DefaultRuntime: &ateapipb.WorkerSandboxRuntime{Version: wantDefault}, @@ -156,9 +305,9 @@ func TestRegisterWorker(t *testing.T) { for _, r := range restorable { c.RestorableRuntimes = append(c.RestorableRuntimes, &ateapipb.WorkerSandboxRuntime{Version: r}) } - return []*ateapipb.RegisterWorkerRequest{{ + return []*ateapipb.SetWorkerSandboxRuntimesRequest{{ Worker: &ateapipb.ObjectRef{Name: "pod-a"}, - Capacity: capacity, + Epoch: 3, SandboxClasses: []*ateapipb.WorkerSandboxClass{c}, }} } @@ -169,50 +318,29 @@ func TestRegisterWorker(t *testing.T) { unauthenticated bool // serviceErr is what the control plane answers with. serviceErr error - req *ateletpb.RegisterWorkerRequest + req *ateletpb.SetWorkerSandboxRuntimesRequest wantCode codes.Code - wantForwarded []*ateapipb.RegisterWorkerRequest + wantForwarded []*ateapipb.SetWorkerSandboxRuntimesRequest }{{ - name: "records what the worker says", - req: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{Actors: 4, CpuMillicores: 2000, MemoryBytes: 4 << 30}, - SandboxClasses: reqClasses(), - }, - wantForwarded: forwarded(&ateapipb.WorkerResources{Actors: 4, CpuMillicores: 2000, MemoryBytes: 4294967296}), + name: "records what the worker says", + req: &ateletpb.SetWorkerSandboxRuntimesRequest{Epoch: 3, SandboxClasses: reqClasses()}, + wantForwarded: forwarded(), }, { - name: "forwards restorable runtimes", - req: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{Actors: 1}, - SandboxClasses: reqClasses(reqCompat("d547")), - }, - wantForwarded: forwarded(&ateapipb.WorkerResources{Actors: 1}, wantCompat("d547")), - }, { - name: "omits undetermined compute", - req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, SandboxClasses: reqClasses()}, - wantForwarded: forwarded(&ateapipb.WorkerResources{Actors: 1}), + name: "forwards restorable runtimes", + req: &ateletpb.SetWorkerSandboxRuntimesRequest{Epoch: 3, SandboxClasses: reqClasses(reqCompat("d547"))}, + wantForwarded: forwarded(wantCompat("d547")), }, { - name: "forwards empty capacity", - req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{}, SandboxClasses: reqClasses()}, - wantForwarded: forwarded(&ateapipb.WorkerResources{}), - }, { - name: "rejects invalid capacity without forwarding", - req: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{CpuMillicores: -1}, - SandboxClasses: reqClasses(), - }, - wantCode: codes.InvalidArgument, - }, { - name: "rejects missing capacity without forwarding", - req: &ateletpb.RegisterWorkerRequest{SandboxClasses: reqClasses()}, + name: "rejects missing sandbox classes without forwarding", + req: &ateletpb.SetWorkerSandboxRuntimesRequest{Epoch: 3}, wantCode: codes.InvalidArgument, }, { - name: "rejects missing sandbox classes without forwarding", - req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}}, + name: "rejects missing epoch without forwarding", + req: &ateletpb.SetWorkerSandboxRuntimesRequest{SandboxClasses: reqClasses()}, wantCode: codes.InvalidArgument, }, { name: "rejects missing default runtime without forwarding", - req: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{Actors: 1}, + req: &ateletpb.SetWorkerSandboxRuntimesRequest{ + Epoch: 3, SandboxClasses: []*ateletpb.WorkerSandboxClass{{SandboxClass: "gvisor"}}, }, wantCode: codes.InvalidArgument, @@ -220,8 +348,8 @@ func TestRegisterWorker(t *testing.T) { // A worker runs one class until WorkerPoolSpec.sandboxClasses // allows more. name: "rejects two sandbox classes without forwarding", - req: &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{Actors: 1}, + req: &ateletpb.SetWorkerSandboxRuntimesRequest{ + Epoch: 3, SandboxClasses: []*ateletpb.WorkerSandboxClass{ {SandboxClass: "gvisor", DefaultRuntime: &ateletpb.WorkerSandboxRuntime{Version: reqDefault}}, {SandboxClass: "microvm", DefaultRuntime: &ateletpb.WorkerSandboxRuntime{Version: reqDefault}}, @@ -232,16 +360,15 @@ func TestRegisterWorker(t *testing.T) { // A worker may report only what its certificate proves it is. name: "requires a certificate", unauthenticated: true, - req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, SandboxClasses: reqClasses()}, + req: &ateletpb.SetWorkerSandboxRuntimesRequest{Epoch: 3, SandboxClasses: reqClasses()}, wantCode: codes.Unauthenticated, }, { - // The Worker record may not exist yet. The error must reach the - // worker so it retries: it reports once, so a swallowed failure - // leaves the Worker with no capacity forever. - name: "surfaces the control plane's rejection", - serviceErr: errors.New("no such worker"), - req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, SandboxClasses: reqClasses()}, - wantCode: codes.Unknown, + // A refused epoch must reach the worker unchanged: it is what tells + // the worker process to stop serving. + name: "surfaces the control plane's refusal", + serviceErr: status.Error(codes.FailedPrecondition, "epoch superseded"), + req: &ateletpb.SetWorkerSandboxRuntimesRequest{Epoch: 3, SandboxClasses: reqClasses()}, + wantCode: codes.FailedPrecondition, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -252,11 +379,11 @@ func TestRegisterWorker(t *testing.T) { ctx = context.Background() } - _, err := svc.RegisterWorker(ctx, tt.req) + _, err := svc.SetWorkerSandboxRuntimes(ctx, tt.req) if got := status.Code(err); got != tt.wantCode { - t.Errorf("RegisterWorker() code = %v (%v), want %v", got, err, tt.wantCode) + t.Errorf("SetWorkerSandboxRuntimes() code = %v (%v), want %v", got, err, tt.wantCode) } - if diff := cmp.Diff(tt.wantForwarded, workers.got, protocmp.Transform()); diff != "" { + if diff := cmp.Diff(tt.wantForwarded, workers.runtimesGot, protocmp.Transform()); diff != "" { t.Errorf("forwarded requests mismatch (-want +got):\n%s", diff) } }) diff --git a/cmd/ateworker-gvisor/main.go b/cmd/ateworker-gvisor/main.go index 8481a61ff5..edb36400b9 100644 --- a/cmd/ateworker-gvisor/main.go +++ b/cmd/ateworker-gvisor/main.go @@ -75,6 +75,7 @@ func flagsConfigPath() string { var ( podUID = pflag.String("pod-uid", "", "The UID of the current pod") + podIPs = pflag.String("pod-ips", "", "The comma-separated IPs of the current pod") tunnelConfig = atunnel.RegisterFlags(pflag.CommandLine) @@ -257,6 +258,7 @@ func do(ctx context.Context) error { ateworkerpb.RegisterWorkerServer(svr, ateworkerService) reflection.Register(svr) readiness := &serverboot.Readiness{} + readiness.AddCheck("registered", ateworkerService.epoch.Registered) // Trap SIGTERM (sent by the kubelet at the start of the pod's termination // grace period) and propagate it into the sandbox so the actor can save its @@ -286,6 +288,7 @@ func do(ctx context.Context) error { TrustBundlePath: tunnelConfig.TrustBundle, AteletSPIFFEID: tunnelConfig.BrokerIdentity, Actors: *maxActors, + PodIPs: register.ParsePodIPs(*podIPs), SandboxClass: string(atev1alpha1.SandboxClassGvisor), RuntimesFile: nodepath.WorkerSandboxRuntimesPath(*podUID), } @@ -295,7 +298,7 @@ func do(ctx context.Context) error { ateworkerService.crashReporter = reporter } go func() { - err := register.Report(ctx, reportConfig) + err := register.Report(ctx, reportConfig, &ateworkerService.epoch) if err != nil && ctx.Err() == nil { serverboot.Fatal(ctx, "Failed to register worker", err) } @@ -353,6 +356,10 @@ type AteWorkerService struct { // workload RPCs are rejected with codes.Unavailable. shuttingDown atomic.Bool + // epoch is the epoch this process registered with. It starts no workload + // before registering. + epoch register.Epoch + // cgroupRoot is where the sandbox's cgroup v2 leaves live: the worker pod's // own cgroup scope, which cgroup.Delegate prepares. A field rather // than a constant so tests can point GetWorkloadStats at a fixture tree. @@ -399,7 +406,7 @@ func (s *AteWorkerService) beginRPC(actorUID, name string, cancel context.Cancel } // rejectIfDraining returns a codes.Unavailable error if ateworker has begun graceful -// shutdown, so the control plane reschedules the actor onto a live worker. +// shutdown. func (s *AteWorkerService) rejectIfDraining() error { if s.shuttingDown.Load() { return apierror.Unavailable("worker draining: not accepting new workloads") @@ -601,6 +608,10 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run if err := ateworkervalidation.ValidateRunWorkloadRequest(ctx, req); err != nil { return nil, err } + epoch, err := s.epoch.Require() + if err != nil { + return nil, err + } if err := validateRunscPath(req.GetActorUid(), req.GetActorDirs(), req.GetRunscPath()); err != nil { return nil, err } @@ -723,7 +734,7 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run s.recordInitial(ctx, hosted) s.watchSandbox(ctx, hosted, req.GetRunscPath(), req.GetActorDirs(), req.GetSpec().GetContainers()) - return &ateworkerpb.RunWorkloadResponse{}, nil + return &ateworkerpb.RunWorkloadResponse{WorkerEpoch: epoch}, nil } // Allow checkpointing even if the pod is shutting down. This will allow actors @@ -992,6 +1003,10 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb if err := ateworkervalidation.ValidateRestoreWorkloadRequest(ctx, req); err != nil { return nil, err } + epoch, err := s.epoch.Require() + if err != nil { + return nil, err + } // Taken before the lock, so prep covers the wait for it. tStart := time.Now() if err := validateFidelity(req.GetFidelity()); err != nil { @@ -1196,7 +1211,7 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb s.recordInitial(ctx, hosted) s.watchSandbox(ctx, hosted, req.GetRunscPath(), req.GetActorDirs(), containers) - return &ateworkerpb.RestoreWorkloadResponse{}, nil + return &ateworkerpb.RestoreWorkloadResponse{WorkerEpoch: epoch}, nil } func (s *AteWorkerService) TerminateWorkload(ctx context.Context, req *ateworkerpb.TerminateWorkloadRequest) (*ateworkerpb.TerminateWorkloadResponse, error) { diff --git a/cmd/ateworker-gvisor/main_test.go b/cmd/ateworker-gvisor/main_test.go index 71b7500d9a..bbe6d61de0 100644 --- a/cmd/ateworker-gvisor/main_test.go +++ b/cmd/ateworker-gvisor/main_test.go @@ -72,8 +72,30 @@ func TestRPCsRejectInvalidRequests(t *testing.T) { } } +// A worker process starts no workload before it registers: the control plane +// could not tell which process ran it. Unavailable so the caller retries. +func TestStartupRPCsRejectedBeforeRegistration(t *testing.T) { + s := &AteWorkerService{} + ctx := context.Background() + for name, call := range map[string]func() error{ + "RunWorkload": func() error { + _, err := s.RunWorkload(ctx, validRunRequest()) + return err + }, + "RestoreWorkload": func() error { + _, err := s.RestoreWorkload(ctx, validRestoreRequest()) + return err + }, + } { + if got := apierror.Code(call()); got != codes.Unavailable { + t.Errorf("%s() before registration code = %v, want %v", name, got, codes.Unavailable) + } + } +} + func TestRPCsRejectUntrustedRunscPath(t *testing.T) { s := &AteWorkerService{} + s.epoch.Store(1) ctx := context.Background() for name, call := range map[string]func() error{ "RunWorkload": func() error { diff --git a/cmd/ateworker-microvm/main.go b/cmd/ateworker-microvm/main.go index b1cd7a3d74..4ca28c83a5 100644 --- a/cmd/ateworker-microvm/main.go +++ b/cmd/ateworker-microvm/main.go @@ -65,6 +65,7 @@ import ( var ( podUID = pflag.String("pod-uid", "", "The UID of the current pod") + podIPs = pflag.String("pod-ips", "", "The comma-separated IPs of the current pod") chBinary = pflag.String("cloud-hypervisor-binary", "cloud-hypervisor", "Path to the cloud-hypervisor binary (used to relaunch on restore).") guestDebug = pflag.Bool("guest-debug", false, "enable guest debugging (agent debug logs, debug console, and early serial console)") vmmMemReserve = pflag.Int("vmm-mem-reserve-mib", vmmMemReserveMiB, "Guest RAM (MiB) held back from the pod's memory limit for the cloud-hypervisor VMM + virtiofsd, which run as host processes in the pod cgroup alongside the guest RAM. Prevents the pod OOMing when the VM is sized to the pod's memory limit.") @@ -252,6 +253,7 @@ func do(ctx context.Context) error { ateworkerpb.RegisterWorkerServer(svr, ateworkerService) reflection.Register(svr) readiness := &serverboot.Readiness{} + readiness.AddCheck("registered", ateworkerService.epoch.Registered) // Trap SIGTERM (sent by the kubelet at the start of the pod's termination grace // period) and propagate it into the guest so the actor can save its state and @@ -285,6 +287,7 @@ func do(ctx context.Context) error { TrustBundlePath: tunnelConfig.TrustBundle, AteletSPIFFEID: tunnelConfig.BrokerIdentity, Actors: *maxActors, + PodIPs: register.ParsePodIPs(*podIPs), SandboxClass: string(atev1alpha1.SandboxClassMicroVM), RuntimesFile: nodepath.WorkerSandboxRuntimesPath(*podUID), } @@ -294,7 +297,7 @@ func do(ctx context.Context) error { ateworkerService.crashReporter = reporter } go func() { - err := register.Report(ctx, reportConfig) + err := register.Report(ctx, reportConfig, &ateworkerService.epoch) if err != nil && ctx.Err() == nil { serverboot.Fatal(ctx, "Failed to register worker", err) } @@ -363,6 +366,10 @@ type AteWorkerService struct { // turned away instead of queueing behind it. shuttingDown atomic.Bool + // epoch is the epoch this process registered with. It starts no workload + // before registering. + epoch register.Epoch + podUID string chBinary string guestDebug bool @@ -457,7 +464,7 @@ func validateRuntimeAssetPaths(actorUID string, actorDirs *ateworkerpb.ActorDirs } // rejectIfDraining returns a codes.Unavailable error if ateworker has begun graceful -// shutdown, so the control plane reschedules the actor onto a live worker. +// shutdown. func (s *AteWorkerService) rejectIfDraining() error { if s.shuttingDown.Load() { return apierror.Unavailable("worker draining: not accepting new workloads") diff --git a/cmd/ateworker-microvm/main_test.go b/cmd/ateworker-microvm/main_test.go index 35620cad37..801f36a9c7 100644 --- a/cmd/ateworker-microvm/main_test.go +++ b/cmd/ateworker-microvm/main_test.go @@ -82,6 +82,27 @@ func testSpec() *ateworkerpb.WorkloadSpec { return &ateworkerpb.WorkloadSpec{Containers: []*ateworkerpb.Container{{Name: "app"}}} } +// A worker process starts no workload before it registers: the control plane +// could not tell which process ran it. Unavailable so the caller retries. +func TestStartupRPCsRejectedBeforeRegistration(t *testing.T) { + s := &AteWorkerService{} + ctx := context.Background() + for name, call := range map[string]func() error{ + "RunWorkload": func() error { + _, err := s.RunWorkload(ctx, validRunRequest()) + return err + }, + "RestoreWorkload": func() error { + _, err := s.RestoreWorkload(ctx, validRestoreRequest()) + return err + }, + } { + if got := apierror.Code(call()); got != codes.Unavailable { + t.Errorf("%s() before registration code = %v, want %v", name, got, codes.Unavailable) + } + } +} + func validRunRequest() *ateworkerpb.RunWorkloadRequest { return &ateworkerpb.RunWorkloadRequest{Atespace: testActor.Ref.Atespace, ActorName: testActor.Ref.Name, ActorUid: testActor.UID, Spec: testSpec(), ActorDirs: testDirs()} } diff --git a/cmd/ateworker-microvm/restore.go b/cmd/ateworker-microvm/restore.go index 61a8bbc086..b65bb74f63 100644 --- a/cmd/ateworker-microvm/restore.go +++ b/cmd/ateworker-microvm/restore.go @@ -120,6 +120,10 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb if err := ateworkervalidation.ValidateRestoreWorkloadRequest(ctx, req); err != nil { return nil, err } + epoch, err := s.epoch.Require() + if err != nil { + return nil, err + } if err := validateFidelity(req.GetFidelity()); err != nil { return nil, err } @@ -229,7 +233,7 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb } s.actorLogger.EmitLifecycleLog(ctx, "Actor restored", attribution) - return &ateworkerpb.RestoreWorkloadResponse{}, nil + return &ateworkerpb.RestoreWorkloadResponse{WorkerEpoch: epoch}, nil } // restoreMemoryFidelity restores a whole-guest snapshot: relaunch cloud-hypervisor diff --git a/cmd/ateworker-microvm/restore_test.go b/cmd/ateworker-microvm/restore_test.go index d2641e2b74..edc7998dd7 100644 --- a/cmd/ateworker-microvm/restore_test.go +++ b/cmd/ateworker-microvm/restore_test.go @@ -590,6 +590,7 @@ func TestRestoreGuestReadyError(t *testing.T) { func TestRPCsRejectUntrustedRuntimeAssetPaths(t *testing.T) { s := &AteWorkerService{} + s.epoch.Store(1) ctx := context.Background() assets := map[string]string{assetCH: "/bin/sh"} for name, call := range map[string]func() error{ diff --git a/cmd/ateworker-microvm/run.go b/cmd/ateworker-microvm/run.go index 33f5bd913e..4b23bbf7f4 100644 --- a/cmd/ateworker-microvm/run.go +++ b/cmd/ateworker-microvm/run.go @@ -239,6 +239,10 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run if err := ateworkervalidation.ValidateRunWorkloadRequest(ctx, req); err != nil { return nil, err } + epoch, err := s.epoch.Require() + if err != nil { + return nil, err + } if err := validateRuntimeAssetPaths(req.GetActorUid(), req.GetActorDirs(), req.GetRuntimeAssetPaths()); err != nil { return nil, err } @@ -295,7 +299,7 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run } s.actorLogger.EmitLifecycleLog(ctx, "Actor started", attribution) slog.InfoContext(ctx, "Actor started (overlay rootfs)", slog.String("id", p.actorUID)) - return &ateworkerpb.RunWorkloadResponse{}, nil + return &ateworkerpb.RunWorkloadResponse{WorkerEpoch: epoch}, nil } // actorBootParams is what a cold boot needs about an actor. It comes from a Run diff --git a/cmd/benchmarking/isolate/fake-atelet/epochs.go b/cmd/benchmarking/isolate/fake-atelet/epochs.go new file mode 100644 index 0000000000..6a7059e304 --- /dev/null +++ b/cmd/benchmarking/isolate/fake-atelet/epochs.go @@ -0,0 +1,58 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "sync" + +// workerEpochs records the epoch each Worker on this node registered with +// through the relay, for the herder to report from Run and Restore as the +// worker process would. ate-api-server commits a resumed Actor as RUNNING +// only when that epoch is the Worker's active one. +// +// It is kept in memory only. A fake-atelet that restarts after its Workers +// registered reports epoch 0 for them, and fake-workersync never registers a +// Worker again, so every resume on those Workers fails with Unavailable until +// they are replaced. +// +// A nil workerEpochs records nothing and reports 0. +type workerEpochs struct { + mu sync.Mutex + epochs map[string]int64 +} + +func newWorkerEpochs() *workerEpochs { + return &workerEpochs{epochs: map[string]int64{}} +} + +// set records epoch for the Worker named name. +func (e *workerEpochs) set(name string, epoch int64) { + if e == nil { + return + } + e.mu.Lock() + defer e.mu.Unlock() + e.epochs[name] = epoch +} + +// get returns the epoch the Worker named name registered with, or 0 if it has +// not registered through this fake-atelet. +func (e *workerEpochs) get(name string) int64 { + if e == nil { + return 0 + } + e.mu.Lock() + defer e.mu.Unlock() + return e.epochs[name] +} diff --git a/cmd/benchmarking/isolate/fake-atelet/herder.go b/cmd/benchmarking/isolate/fake-atelet/herder.go index 7cb2b1ef29..3a2293589b 100644 --- a/cmd/benchmarking/isolate/fake-atelet/herder.go +++ b/cmd/benchmarking/isolate/fake-atelet/herder.go @@ -75,6 +75,9 @@ type herder struct { // leaves the mint out. csr is sent with every mint. minter actorCertMinter csr []byte + // epochs is where Run and Restore read the epoch they report; the relay + // records it when the Worker registers. Workers are named by their pod UID. + epochs *workerEpochs } // newActorCSR returns a CSR for a fresh key. ate-api-server takes only the @@ -159,7 +162,7 @@ func (h *herder) Run(ctx context.Context, req *ateletpb.RunRequest) (*ateletpb.R if err := wait(ctx, time.Now(), h.delays.run); err != nil { return nil, err } - return &ateletpb.RunResponse{}, nil + return &ateletpb.RunResponse{WorkerEpoch: h.epochs.get(req.GetWorkerPodUid())}, nil } func (h *herder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (*ateletpb.RestoreResponse, error) { @@ -169,7 +172,7 @@ func (h *herder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (*at if err := wait(ctx, time.Now(), h.delays.restore); err != nil { return nil, err } - return &ateletpb.RestoreResponse{}, nil + return &ateletpb.RestoreResponse{WorkerEpoch: h.epochs.get(req.GetWorkerPodUid())}, nil } func (h *herder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest) (*ateletpb.CheckpointResponse, error) { diff --git a/cmd/benchmarking/isolate/fake-atelet/herder_test.go b/cmd/benchmarking/isolate/fake-atelet/herder_test.go index 1715d67b20..740c957891 100644 --- a/cmd/benchmarking/isolate/fake-atelet/herder_test.go +++ b/cmd/benchmarking/isolate/fake-atelet/herder_test.go @@ -132,6 +132,38 @@ func TestHerderSucceedsAfterDelay(t *testing.T) { } } +// Run and Restore report the epoch the Worker registered with through the +// relay, as its worker process would, and 0 for a Worker that never did. +func TestHerderReportsTheRegisteredEpoch(t *testing.T) { + epochs := newWorkerEpochs() + epochs.set("pod-a", 4) + h := &herder{storage: &recordingStorage{}, epochs: epochs} + ctx := context.Background() + + for _, tc := range []struct { + pod string + want int64 + }{ + {pod: "pod-a", want: 4}, + {pod: "pod-unregistered", want: 0}, + } { + run, err := h.Run(ctx, &ateletpb.RunRequest{WorkerPodUid: tc.pod}) + if err != nil { + t.Fatalf("Run(%s): %v", tc.pod, err) + } + if got := run.GetWorkerEpoch(); got != tc.want { + t.Errorf("Run(%s) worker_epoch = %d, want %d", tc.pod, got, tc.want) + } + restore, err := h.Restore(ctx, &ateletpb.RestoreRequest{WorkerPodUid: tc.pod}) + if err != nil { + t.Fatalf("Restore(%s): %v", tc.pod, err) + } + if got := restore.GetWorkerEpoch(); got != tc.want { + t.Errorf("Restore(%s) worker_epoch = %d, want %d", tc.pod, got, tc.want) + } + } +} + func TestHerderZeroDelay(t *testing.T) { for name, call := range herderCalls(&herder{storage: &recordingStorage{}}) { if err := call(context.Background()); err != nil { diff --git a/cmd/benchmarking/isolate/fake-atelet/main.go b/cmd/benchmarking/isolate/fake-atelet/main.go index 5916095b4a..7199834ab6 100644 --- a/cmd/benchmarking/isolate/fake-atelet/main.go +++ b/cmd/benchmarking/isolate/fake-atelet/main.go @@ -133,7 +133,8 @@ func main() { serverboot.Fatal(ctx, "Failed to listen", err) } workers := ateapipb.NewWorkerServiceClient(ateapiConn) - h := &herder{delays: callDelays, storage: storage} + epochs := newWorkerEpochs() + h := &herder{delays: callDelays, storage: storage, epochs: epochs} if *mintActorCertificate { if h.csr, err = newActorCSR(); err != nil { serverboot.Fatal(ctx, "Failed to create the actor CSR", err) @@ -154,7 +155,7 @@ func main() { // Its own server and listener, so each port admits only its own caller: // ate-api-server on the herder's, fake-workersync on the relay's. relaySrv := grpc.NewServer(grpc.Creds(credentials.NewTLS(relayTLS))) - ateapipb.RegisterWorkerServiceServer(relaySrv, &relay{client: workers}) + ateapipb.RegisterWorkerServiceServer(relaySrv, &relay{client: workers, epochs: epochs}) health := &http.Server{Addr: *healthListenAddr, Handler: healthHandler(), ReadHeaderTimeout: 10 * time.Second} slog.WarnContext(ctx, "Serving the fake atelet: actor lifecycle calls succeed without running any workload", diff --git a/cmd/benchmarking/isolate/fake-atelet/relay.go b/cmd/benchmarking/isolate/fake-atelet/relay.go index eed6d28c00..b15ee2d785 100644 --- a/cmd/benchmarking/isolate/fake-atelet/relay.go +++ b/cmd/benchmarking/isolate/fake-atelet/relay.go @@ -26,7 +26,9 @@ import ( // capacityClient is the part of ateapipb.WorkerServiceClient the relay uses. type capacityClient interface { + AcquireWorkerEpoch(ctx context.Context, in *ateapipb.AcquireWorkerEpochRequest, opts ...grpc.CallOption) (*ateapipb.AcquireWorkerEpochResponse, error) RegisterWorker(ctx context.Context, in *ateapipb.RegisterWorkerRequest, opts ...grpc.CallOption) (*ateapipb.RegisterWorkerResponse, error) + SetWorkerSandboxRuntimes(ctx context.Context, in *ateapipb.SetWorkerSandboxRuntimesRequest, opts ...grpc.CallOption) (*ateapipb.SetWorkerSandboxRuntimesResponse, error) } // relay forwards fake-workersync's capacity reports to ate-api-server under @@ -42,6 +44,28 @@ type relay struct { ateapipb.UnimplementedWorkerServiceServer client capacityClient + // epochs records the epoch of each registration ate-api-server accepts. + epochs *workerEpochs +} + +func (r *relay) AcquireWorkerEpoch(ctx context.Context, in *ateapipb.AcquireWorkerEpochRequest) (*ateapipb.AcquireWorkerEpochResponse, error) { + resp, err := r.client.AcquireWorkerEpoch(ctx, in) + if err != nil { + slog.WarnContext(ctx, "Epoch acquire rejected", + slog.String("worker", in.GetWorker().GetName()), slog.Any("err", err)) + return nil, err + } + return resp, nil +} + +func (r *relay) SetWorkerSandboxRuntimes(ctx context.Context, in *ateapipb.SetWorkerSandboxRuntimesRequest) (*ateapipb.SetWorkerSandboxRuntimesResponse, error) { + resp, err := r.client.SetWorkerSandboxRuntimes(ctx, in) + if err != nil { + slog.WarnContext(ctx, "Sandbox runtimes report rejected", + slog.String("worker", in.GetWorker().GetName()), slog.Any("err", err)) + return nil, err + } + return resp, nil } func (r *relay) RegisterWorker(ctx context.Context, in *ateapipb.RegisterWorkerRequest) (*ateapipb.RegisterWorkerResponse, error) { @@ -51,6 +75,7 @@ func (r *relay) RegisterWorker(ctx context.Context, in *ateapipb.RegisterWorkerR slog.String("worker", in.GetWorker().GetName()), slog.Any("err", err)) return nil, err } + r.epochs.set(in.GetWorker().GetName(), in.GetEpoch()) slog.DebugContext(ctx, "Relayed capacity report", slog.String("worker", in.GetWorker().GetName()), slog.Any("capacity", in.GetCapacity())) return resp, nil diff --git a/cmd/benchmarking/isolate/fake-atelet/relay_test.go b/cmd/benchmarking/isolate/fake-atelet/relay_test.go index 5e44dea2c2..b9c0850303 100644 --- a/cmd/benchmarking/isolate/fake-atelet/relay_test.go +++ b/cmd/benchmarking/isolate/fake-atelet/relay_test.go @@ -31,10 +31,21 @@ import ( "google.golang.org/protobuf/proto" ) -// recordingWorkerService records the last report and answers with err. +// recordingWorkerService records the last request of each kind and answers +// with err. type recordingWorkerService struct { - got *ateapipb.RegisterWorkerRequest - err error + got *ateapipb.RegisterWorkerRequest + acquireGot *ateapipb.AcquireWorkerEpochRequest + runtimesGot *ateapipb.SetWorkerSandboxRuntimesRequest + err error +} + +func (f *recordingWorkerService) AcquireWorkerEpoch(_ context.Context, in *ateapipb.AcquireWorkerEpochRequest, _ ...grpc.CallOption) (*ateapipb.AcquireWorkerEpochResponse, error) { + f.acquireGot = in + if f.err != nil { + return nil, f.err + } + return &ateapipb.AcquireWorkerEpochResponse{Epoch: 4}, nil } func (f *recordingWorkerService) RegisterWorker(_ context.Context, in *ateapipb.RegisterWorkerRequest, _ ...grpc.CallOption) (*ateapipb.RegisterWorkerResponse, error) { @@ -45,6 +56,53 @@ func (f *recordingWorkerService) RegisterWorker(_ context.Context, in *ateapipb. return &ateapipb.RegisterWorkerResponse{}, nil } +func (f *recordingWorkerService) SetWorkerSandboxRuntimes(_ context.Context, in *ateapipb.SetWorkerSandboxRuntimesRequest, _ ...grpc.CallOption) (*ateapipb.SetWorkerSandboxRuntimesResponse, error) { + f.runtimesGot = in + if f.err != nil { + return nil, f.err + } + return &ateapipb.SetWorkerSandboxRuntimesResponse{}, nil +} + +func runtimesReport() *ateapipb.SetWorkerSandboxRuntimesRequest { + return &ateapipb.SetWorkerSandboxRuntimesRequest{ + Worker: &ateapipb.ObjectRef{Name: "fake-r1-c52b9bd3-0"}, + Epoch: 4, + SandboxClasses: []*ateapipb.WorkerSandboxClass{{ + SandboxClass: "gvisor", + DefaultRuntime: &ateapipb.WorkerSandboxRuntime{Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{{Key: "architecture", Value: "amd64"}}, + }}, + }}, + } +} + +func TestRelayForwardsTheAcquireUnchanged(t *testing.T) { + svc := &recordingWorkerService{} + in := &ateapipb.AcquireWorkerEpochRequest{Worker: &ateapipb.ObjectRef{Name: "fake-r1-c52b9bd3-0"}} + resp, err := (&relay{client: svc}).AcquireWorkerEpoch(context.Background(), in) + if err != nil { + t.Fatalf("AcquireWorkerEpoch: %v", err) + } + if !proto.Equal(svc.acquireGot, in) { + t.Errorf("forwarded %v, want %v", svc.acquireGot, in) + } + if got := resp.GetEpoch(); got != 4 { + t.Errorf("epoch = %d, want ate-api-server's 4", got) + } +} + +func TestRelayForwardsTheRuntimesUnchanged(t *testing.T) { + svc := &recordingWorkerService{} + if _, err := (&relay{client: svc}).SetWorkerSandboxRuntimes(context.Background(), runtimesReport()); err != nil { + t.Fatalf("SetWorkerSandboxRuntimes: %v", err) + } + if !proto.Equal(svc.runtimesGot, runtimesReport()) { + t.Errorf("forwarded %v, want %v", svc.runtimesGot, runtimesReport()) + } +} + func report() *ateapipb.RegisterWorkerRequest { return &ateapipb.RegisterWorkerRequest{ Worker: &ateapipb.ObjectRef{Name: "fake-r1-c52b9bd3-0"}, @@ -65,17 +123,57 @@ func TestRelayForwardsTheReportUnchanged(t *testing.T) { } } +// The herder reports the epoch a Worker registered with, so the relay records +// it, and only once ate-api-server has accepted the registration. +func TestRelayRecordsTheRegisteredEpoch(t *testing.T) { + in := report() + in.Epoch = 4 + + refused := newWorkerEpochs() + svc := &recordingWorkerService{err: status.Error(codes.FailedPrecondition, "epoch superseded")} + if _, err := (&relay{client: svc, epochs: refused}).RegisterWorker(context.Background(), in); err == nil { + t.Fatal("RegisterWorker succeeded, want ate-api-server's refusal") + } + if got := refused.get(in.GetWorker().GetName()); got != 0 { + t.Errorf("epoch after a refused registration = %d, want 0", got) + } + + accepted := newWorkerEpochs() + if _, err := (&relay{client: &recordingWorkerService{}, epochs: accepted}).RegisterWorker(context.Background(), in); err != nil { + t.Fatalf("RegisterWorker: %v", err) + } + if got := accepted.get(in.GetWorker().GetName()); got != 4 { + t.Errorf("epoch after an accepted registration = %d, want 4", got) + } +} + // fake-workersync retries a Worker ate-api-server has not seen yet and gives up // on a rejected report, so the relay must keep ate-api-server's status. func TestRelayPassesThroughAteAPIServersStatus(t *testing.T) { - for _, code := range []codes.Code{codes.NotFound, codes.InvalidArgument, codes.PermissionDenied, codes.Unavailable, codes.Internal} { - svc := &recordingWorkerService{err: status.Error(code, "from ate-api-server")} - _, err := (&relay{client: svc}).RegisterWorker(context.Background(), report()) - if got := status.Code(err); got != code { - t.Errorf("%v: code = %v", code, got) - } - if !strings.Contains(status.Convert(err).Message(), "from ate-api-server") { - t.Errorf("%v: %v does not carry ate-api-server's message", code, err) + calls := map[string]func(*relay) error{ + "AcquireWorkerEpoch": func(r *relay) error { + _, err := r.AcquireWorkerEpoch(context.Background(), &ateapipb.AcquireWorkerEpochRequest{}) + return err + }, + "RegisterWorker": func(r *relay) error { + _, err := r.RegisterWorker(context.Background(), report()) + return err + }, + "SetWorkerSandboxRuntimes": func(r *relay) error { + _, err := r.SetWorkerSandboxRuntimes(context.Background(), runtimesReport()) + return err + }, + } + for name, call := range calls { + for _, code := range []codes.Code{codes.NotFound, codes.InvalidArgument, codes.PermissionDenied, codes.Unavailable, codes.Internal} { + svc := &recordingWorkerService{err: status.Error(code, "from ate-api-server")} + err := call(&relay{client: svc}) + if got := status.Code(err); got != code { + t.Errorf("%s %v: code = %v", name, code, got) + } + if !strings.Contains(status.Convert(err).Message(), "from ate-api-server") { + t.Errorf("%s %v: %v does not carry ate-api-server's message", name, code, err) + } } } } diff --git a/cmd/benchmarking/isolate/fake-workersync/controller.go b/cmd/benchmarking/isolate/fake-workersync/controller.go index 18532a61d0..7e81e2ecc4 100644 --- a/cmd/benchmarking/isolate/fake-workersync/controller.go +++ b/cmd/benchmarking/isolate/fake-workersync/controller.go @@ -54,9 +54,22 @@ type workerClient interface { ListWorkerActorAssignments(ctx context.Context, in *ateapipb.ListWorkerActorAssignmentsRequest, opts ...grpc.CallOption) (*ateapipb.ListWorkerActorAssignmentsResponse, error) } -// capacityRelay sends a capacity report to the fake-atelet relay at addr. +// workerReport is what a fake worker process registers and reports. +type workerReport struct { + Name string + // Epoch is the Worker's registered epoch to register again, or 0 to acquire a + // new one. Reusing it keeps a restarted fake-workersync from raising the + // epoch, which would make ate-api-server crash every Actor on the Worker. + Epoch int64 + IPs []string + Capacity *ateapipb.WorkerResources + SandboxClasses []*ateapipb.WorkerSandboxClass +} + +// capacityRelay registers a fake worker process through the fake-atelet relay +// at addr: it acquires an epoch if needed, registers, and reports runtimes. type capacityRelay interface { - Report(ctx context.Context, addr string, req *ateapipb.RegisterWorkerRequest) error + Report(ctx context.Context, addr string, report workerReport) error // Prune lets go of every relay not in live, the relay addresses by node. Prune(live map[string]string) } @@ -208,7 +221,9 @@ func (c *controller) adopt(ctx context.Context) error { node: w.GetNodeName(), labels: w.GetLabels(), capacity: w.GetStatus().GetCapacity(), - reported: w.GetStatus().GetCapacity() != nil, + // Its runtimes are reported last, so a Worker without them is + // registered again. + reported: len(w.GetStatus().GetSandboxClasses()) > 0, // One never marked available goes back through create, which // marks it. created: w.GetStatus().GetState() != ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE, @@ -414,8 +429,21 @@ func (c *controller) ensure(ctx context.Context, pod string, d desiredWorker, w if addr == "" { return fmt.Errorf("no fake-atelet on node %s yet for Worker %s", w.node, name) } - if err := c.relay.Report(ctx, addr, &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: name}, + current, err := c.client.GetWorker(ctx, &ateapipb.GetWorkerRequest{Worker: &ateapipb.ObjectRef{Name: name}}) + if err != nil { + return fmt.Errorf("while reading Worker %s: %w", name, err) + } + ips := []string{fakeworker.IP(w.index)} + // An epoch's registration never changes, so a Worker registered with other + // IPs or capacity takes a new epoch, as a restarted worker process does. + epoch := current.GetStatus().GetRegisteredEpoch() + if !slices.Equal(current.GetStatus().GetIps(), ips) || !proto.Equal(current.GetStatus().GetCapacity(), fixed) { + epoch = 0 + } + if err := c.relay.Report(ctx, addr, workerReport{ + Name: name, + Epoch: epoch, + IPs: ips, Capacity: fixed, SandboxClasses: []*ateapipb.WorkerSandboxClass{{ SandboxClass: string(d.pool.Spec.DefaultSandboxClass()), @@ -441,8 +469,7 @@ func hostCompat() *ateapipb.VersionedSandboxCompat { // create registers the Worker with the fields the worker syncer would copy // from the pool and its pod, then marks it available as the syncer does once -// the pod is Ready. Epoch stays 0: a rising epoch makes ate-api-server crash -// every Actor on the Worker. +// the pod is Ready. func (c *controller) create(ctx context.Context, w *fakeWorker, wp *atev1alpha1.WorkerPool) error { name := w.name _, err := c.client.CreateWorker(ctx, &ateapipb.CreateWorkerRequest{Worker: &ateapipb.Worker{ @@ -452,7 +479,6 @@ func (c *controller) create(ctx context.Context, w *fakeWorker, wp *atev1alpha1. WorkerPod: w.pod, WorkerPodUid: name, NodeName: w.node, - Ips: []string{fakeworker.IP(w.index)}, Labels: maps.Clone(wp.GetLabels()), }}) if err != nil && status.Code(err) != codes.AlreadyExists { diff --git a/cmd/benchmarking/isolate/fake-workersync/controller_test.go b/cmd/benchmarking/isolate/fake-workersync/controller_test.go index 326c3bdd45..e0008fe6fa 100644 --- a/cmd/benchmarking/isolate/fake-workersync/controller_test.go +++ b/cmd/benchmarking/isolate/fake-workersync/controller_test.go @@ -254,38 +254,59 @@ func (f *fakeControl) uid(pod string) string { } // fakeRelay records accepted reports, as ate-api-server's view of capacity, -// and refuses Workers listed in reject with their code. An accepted report is -// also written to control's Worker, as RegisterWorker records it. +// and refuses Workers listed in reject with their code. A report with no epoch +// acquires the Worker's next one, as AcquireWorkerEpoch does. An accepted +// report is also written to control's Worker, as RegisterWorker and +// SetWorkerSandboxRuntimes record it. type fakeRelay struct { control *fakeControl mu sync.Mutex reported map[string]*ateapipb.WorkerResources // by Worker name classes map[string][]*ateapipb.WorkerSandboxClass // sandbox classes, by Worker name via map[string]string // relay address, by Worker name + ips map[string][]string // by Worker name + epochs map[string]int64 // registered epoch, by Worker name + acquired map[string]int // epochs acquired, by Worker name reject map[string]codes.Code calls int live map[string]string // as last pruned to } func newFakeRelay() *fakeRelay { - return &fakeRelay{reported: map[string]*ateapipb.WorkerResources{}, classes: map[string][]*ateapipb.WorkerSandboxClass{}, via: map[string]string{}, reject: map[string]codes.Code{}} + return &fakeRelay{ + reported: map[string]*ateapipb.WorkerResources{}, + classes: map[string][]*ateapipb.WorkerSandboxClass{}, + via: map[string]string{}, + ips: map[string][]string{}, + epochs: map[string]int64{}, + acquired: map[string]int{}, + reject: map[string]codes.Code{}, + } } -func (f *fakeRelay) Report(_ context.Context, addr string, req *ateapipb.RegisterWorkerRequest) error { +func (f *fakeRelay) Report(_ context.Context, addr string, report workerReport) error { f.mu.Lock() defer f.mu.Unlock() f.calls++ - name := req.GetWorker().GetName() + name := report.Name if code, ok := f.reject[name]; ok { return status.Error(code, "rejected") } - f.reported[name] = req.GetCapacity() - f.classes[name] = req.GetSandboxClasses() + epoch := report.Epoch + if epoch == 0 { + f.acquired[name]++ + epoch = f.epochs[name] + 1 + } + f.epochs[name] = epoch + f.reported[name] = report.Capacity + f.classes[name] = report.SandboxClasses + f.ips[name] = report.IPs f.via[name] = addr if f.control != nil { f.control.mu.Lock() if w, ok := f.control.workers[name]; ok { - w.Status.Capacity, w.Status.SandboxClasses = req.GetCapacity(), req.GetSandboxClasses() + w.Status.RegisteredEpoch, w.Status.Ips = epoch, report.IPs + w.Status.Capacity, w.Status.SandboxClasses = report.Capacity, report.SandboxClasses } f.control.mu.Unlock() } @@ -440,13 +461,21 @@ func TestReconcileHonorsReplicasAndLimits(t *testing.T) { t.Errorf("sandbox class = %q", hardware.SandboxClass(w.GetStatus())) case w.GetLabels()["workload"] != "bench": t.Errorf("Labels = %v, want the pool's, which template selectors match", w.GetLabels()) - case w.GetWorkerPodUid() != w.GetMetadata().GetName() || w.GetEpoch() != 0: - t.Errorf("WorkerPodUid/Epoch = %q/%d", w.GetWorkerPodUid(), w.GetEpoch()) + case w.GetWorkerPodUid() != w.GetMetadata().GetName(): + t.Errorf("WorkerPodUid = %q, want the Worker's name", w.GetWorkerPodUid()) } if got := rel.via[ctl.uid(name)]; got != "10.0.0.2:8086" { t.Errorf("capacity for %s sent via %q, want node-b's fake-atelet", name, got) } + // A new Worker has no epoch, so its fake worker process acquires one and + // registers with its fake pod IP. + if got := rel.acquired[ctl.uid(name)]; got != 1 { + t.Errorf("acquired %d epochs for a new Worker, want 1", got) + } + if got, want := rel.ips[ctl.uid(name)], []string{fakeworker.IP(1)}; !slices.Equal(got, want) { + t.Errorf("registered IPs %v, want %v", got, want) + } if got, want := rel.reported[ctl.uid(name)], wantCapacity(1000, "1500m", "4Gi"); !proto.Equal(got, want) { t.Errorf("reported %v, want %v from the pool's limits", got, want) } @@ -950,7 +979,7 @@ func TestPoolLabelChangeUpdatesWorkers(t *testing.T) { } } -// A Worker's sandbox class is fixed on its first RegisterWorker report, so a +// A worker pod runs one sandbox class for its lifetime, so a // sandboxClass edit replaces the pool's Workers the way it replaces real // worker pods: each pod gets a new Worker, under a new name and pod UID, at // once; an idle old Worker is deleted at once, a held one once its Actors @@ -1194,6 +1223,51 @@ func TestAdoptReportsAWorkerNeverAccepted(t *testing.T) { } } +// A report for a Worker that already registered its epoch with the same +// IPs and capacity registers that epoch again instead of acquiring a new one: a +// raised epoch would make ate-api-server crash every Actor on the Worker. One +// registered with other values acquires a new epoch, since ate-api-server +// refuses to change an epoch's registration. +func TestReportReusesTheRegisteredEpoch(t *testing.T) { + tests := []struct { + name string + ip string + capacity *ateapipb.WorkerResources + wantAcquired int + wantEpoch int64 + }{ + {name: "same registration", ip: fakeworker.IP(0), capacity: wantCapacity(1000, "86", "160Gi"), wantEpoch: 5}, + {name: "other capacity", ip: fakeworker.IP(0), capacity: wantCapacity(10, "86", "160Gi"), wantAcquired: 1, wantEpoch: 6}, + {name: "other IPs", ip: fakeworker.IP(1), capacity: wantCapacity(1000, "86", "160Gi"), wantAcquired: 1, wantEpoch: 6}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + c, ctl, rel, _ := newTestController(pool("bench", 1, nil)) + const first = "worker-1" // the first name newTestController's controller mints + rel.reject[first] = codes.Unavailable + if err := reconcile(t, c); err == nil { + t.Fatal("reconcile succeeded with a rejected report") + } + delete(rel.reject, first) + rel.epochs[first] = 5 + ctl.mu.Lock() + status := ctl.workers[first].Status + status.RegisteredEpoch, status.Ips, status.Capacity = 5, []string{tc.ip}, tc.capacity + ctl.mu.Unlock() + + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile: %v", err) + } + if got := rel.acquired[first]; got != tc.wantAcquired { + t.Errorf("acquired %d epochs, want %d", got, tc.wantAcquired) + } + if got := rel.epochs[first]; got != tc.wantEpoch { + t.Errorf("registered epoch %d, want %d", got, tc.wantEpoch) + } + }) + } +} + func TestDeleteAll(t *testing.T) { c, ctl, _, _ := newTestController(pool("bench", 3, nil)) if err := reconcile(t, c); err != nil { diff --git a/cmd/benchmarking/isolate/fake-workersync/relayclient.go b/cmd/benchmarking/isolate/fake-workersync/relayclient.go index 7d4fe48cad..e666edc55e 100644 --- a/cmd/benchmarking/isolate/fake-workersync/relayclient.go +++ b/cmd/benchmarking/isolate/fake-workersync/relayclient.go @@ -86,15 +86,38 @@ func newGRPCRelay(clientBundlePath, trustBundlePath, ateletID string) (*grpcRela return &grpcRelay{creds: credentials.NewTLS(cfg), conns: map[string]*grpc.ClientConn{}}, nil } -// Report sends req to the relay at addr (host:port). -func (g *grpcRelay) Report(ctx context.Context, addr string, req *ateapipb.RegisterWorkerRequest) error { +// Report registers report's Worker through the relay at addr (host:port), as a +// worker process does at startup. +func (g *grpcRelay) Report(ctx context.Context, addr string, report workerReport) error { conn, err := g.conn(addr) if err != nil { return err } ctx, cancel := context.WithTimeout(ctx, reportTimeout) defer cancel() - _, err = ateapipb.NewWorkerServiceClient(conn).RegisterWorker(ctx, req) + client := ateapipb.NewWorkerServiceClient(conn) + ref := &ateapipb.ObjectRef{Name: report.Name} + epoch := report.Epoch + if epoch == 0 { + resp, err := client.AcquireWorkerEpoch(ctx, &ateapipb.AcquireWorkerEpochRequest{Worker: ref}) + if err != nil { + return err + } + epoch = resp.GetEpoch() + } + if _, err := client.RegisterWorker(ctx, &ateapipb.RegisterWorkerRequest{ + Worker: ref, + Epoch: epoch, + Ips: report.IPs, + Capacity: report.Capacity, + }); err != nil { + return err + } + _, err = client.SetWorkerSandboxRuntimes(ctx, &ateapipb.SetWorkerSandboxRuntimesRequest{ + Worker: ref, + Epoch: epoch, + SandboxClasses: report.SandboxClasses, + }) return err } diff --git a/cmd/benchmarking/isolate/fake-workersync/relayclient_test.go b/cmd/benchmarking/isolate/fake-workersync/relayclient_test.go index 0cc83ce898..961ec0f599 100644 --- a/cmd/benchmarking/isolate/fake-workersync/relayclient_test.go +++ b/cmd/benchmarking/isolate/fake-workersync/relayclient_test.go @@ -42,14 +42,29 @@ const ( testWorkerSyncID = "spiffe://cluster.local/ns/ate-system/sa/fake-workersync" ) -// stubRelay is a stand-in fake-atelet relay: it records each report and +// stubEpoch is the epoch stubRelay hands out. +const stubEpoch = 9 + +// stubRelay is a stand-in fake-atelet relay: it records each request and // answers with err. type stubRelay struct { ateapipb.UnimplementedWorkerServiceServer - mu sync.Mutex - got []*ateapipb.RegisterWorkerRequest - err error + mu sync.Mutex + acquireGot []*ateapipb.AcquireWorkerEpochRequest + got []*ateapipb.RegisterWorkerRequest + runtimesGot []*ateapipb.SetWorkerSandboxRuntimesRequest + err error +} + +func (s *stubRelay) AcquireWorkerEpoch(_ context.Context, in *ateapipb.AcquireWorkerEpochRequest) (*ateapipb.AcquireWorkerEpochResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.acquireGot = append(s.acquireGot, in) + if s.err != nil { + return nil, s.err + } + return &ateapipb.AcquireWorkerEpochResponse{Epoch: stubEpoch}, nil } func (s *stubRelay) RegisterWorker(_ context.Context, in *ateapipb.RegisterWorkerRequest) (*ateapipb.RegisterWorkerResponse, error) { @@ -62,6 +77,16 @@ func (s *stubRelay) RegisterWorker(_ context.Context, in *ateapipb.RegisterWorke return &ateapipb.RegisterWorkerResponse{}, nil } +func (s *stubRelay) SetWorkerSandboxRuntimes(_ context.Context, in *ateapipb.SetWorkerSandboxRuntimesRequest) (*ateapipb.SetWorkerSandboxRuntimesResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.runtimesGot = append(s.runtimesGot, in) + if s.err != nil { + return nil, s.err + } + return &ateapipb.SetWorkerSandboxRuntimesResponse{}, nil +} + // relayServer serves svc with a certificate for serverID from ca, requiring // fake-workersync's certificate from ca, and returns its address. func relayServer(t *testing.T, ca *fakeworkertest.CA, serverID string, svc ateapipb.WorkerServiceServer) string { @@ -107,13 +132,27 @@ func newTestRelay(t *testing.T, ca *fakeworkertest.CA) *grpcRelay { return relay } -func capacityReport() *ateapipb.RegisterWorkerRequest { - return &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: "fake-r1-c52b9bd3-0"}, - Capacity: &ateapipb.WorkerResources{Actors: 1000}, +func testClasses() []*ateapipb.WorkerSandboxClass { + return []*ateapipb.WorkerSandboxClass{{ + SandboxClass: "gvisor", + DefaultRuntime: &ateapipb.WorkerSandboxRuntime{Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{{Key: "architecture", Value: "amd64"}}, + }}, + }} +} + +func capacityReport() workerReport { + return workerReport{ + Name: "fake-r1-c52b9bd3-0", + IPs: []string{"10.0.0.7"}, + Capacity: &ateapipb.WorkerResources{Actors: 1000}, + SandboxClasses: testClasses(), } } +// A Worker with no epoch acquires one, registers with it, then reports its +// runtimes under it, as a worker process does at startup. func TestRelayClientSendsTheReport(t *testing.T) { ca := fakeworkertest.NewCA(t) svc := &stubRelay{} @@ -124,14 +163,51 @@ func TestRelayClientSendsTheReport(t *testing.T) { t.Fatalf("Report: %v", err) } } - if len(svc.got) != 2 || !proto.Equal(svc.got[0], capacityReport()) { - t.Errorf("relay received %v, want %v twice", svc.got, capacityReport()) + ref := &ateapipb.ObjectRef{Name: "fake-r1-c52b9bd3-0"} + wantAcquire := &ateapipb.AcquireWorkerEpochRequest{Worker: ref} + wantRegister := &ateapipb.RegisterWorkerRequest{ + Worker: ref, + Epoch: stubEpoch, + Ips: []string{"10.0.0.7"}, + Capacity: &ateapipb.WorkerResources{Actors: 1000}, + } + wantRuntimes := &ateapipb.SetWorkerSandboxRuntimesRequest{Worker: ref, Epoch: stubEpoch, SandboxClasses: testClasses()} + if len(svc.acquireGot) != 2 || !proto.Equal(svc.acquireGot[0], wantAcquire) { + t.Errorf("relay received acquires %v, want %v twice", svc.acquireGot, wantAcquire) + } + if len(svc.got) != 2 || !proto.Equal(svc.got[0], wantRegister) { + t.Errorf("relay received registrations %v, want %v twice", svc.got, wantRegister) + } + if len(svc.runtimesGot) != 2 || !proto.Equal(svc.runtimesGot[0], wantRuntimes) { + t.Errorf("relay received runtimes %v, want %v twice", svc.runtimesGot, wantRuntimes) } if len(relay.conns) != 1 { t.Errorf("%d connections to one relay, want 1", len(relay.conns)) } } +// A Worker that already has a registered epoch registers it again without +// acquiring a new one. +func TestRelayClientReusesTheGivenEpoch(t *testing.T) { + ca := fakeworkertest.NewCA(t) + svc := &stubRelay{} + addr := relayServer(t, ca, testAteletID, svc) + report := capacityReport() + report.Epoch = 3 + if err := newTestRelay(t, ca).Report(context.Background(), addr, report); err != nil { + t.Fatalf("Report: %v", err) + } + if len(svc.acquireGot) != 0 { + t.Errorf("relay received acquires %v, want none", svc.acquireGot) + } + if len(svc.got) != 1 || svc.got[0].GetEpoch() != 3 { + t.Errorf("relay received registrations %v, want one with epoch 3", svc.got) + } + if len(svc.runtimesGot) != 1 || svc.runtimesGot[0].GetEpoch() != 3 { + t.Errorf("relay received runtimes %v, want one with epoch 3", svc.runtimesGot) + } +} + // fake-workersync reports only to the atelet identity: a report sent anywhere // else would hand a Worker's capacity to a workload that is not its atelet. func TestRelayClientRefusesAServerThatIsNotAtelet(t *testing.T) { diff --git a/docs/authorization.md b/docs/authorization.md index 6745a9d7d1..8dbf7b1ffa 100644 --- a/docs/authorization.md +++ b/docs/authorization.md @@ -217,7 +217,9 @@ certificate. | RPC | The caller must be | |---|---| +| `AcquireWorkerEpoch` | `atelet` on the worker's node | | `RegisterWorker` | `atelet` on the worker's node | +| `SetWorkerSandboxRuntimes` | `atelet` on the worker's node | | `RequestActorSuspend` | `atelet` on the node of the worker hosting the actor | | `MintWorkerActorJwt` | `atelet` on the node hosting the actor | | `MintWorkerActorCertificate` | `atelet` on the node of the worker the actor is assigned to | diff --git a/internal/ateworker/register/register.go b/internal/ateworker/register/register.go index 7c5fd61a5e..36b6c03f10 100644 --- a/internal/ateworker/register/register.go +++ b/internal/ateworker/register/register.go @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package register registers a worker with the control plane through the -// node-local atelet, reporting its compute capacity and the sandbox runtimes -// of its pool, and provides file-watching helpers shared by worker binaries. +// Package register registers a worker process with the control plane through +// the node-local atelet: it acquires an epoch, registers once with that epoch, +// its pod IPs and its compute capacity, then reports the sandbox runtimes of +// its pool. It also provides file-watching helpers shared by worker binaries. package register import ( @@ -30,8 +31,10 @@ import ( "slices" "strconv" "strings" + "sync/atomic" "time" + "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/ateletdial" "go.ate.dev/substrate/internal/hardware" "go.ate.dev/substrate/internal/proto/ateletpb" @@ -49,6 +52,42 @@ const ( resyncInterval = 5 * time.Minute ) +// Epoch is the epoch this worker process registered with. A worker process +// takes no workloads until it has registered, and reports this epoch with each +// one it starts, so the control plane can tell which process ran it. +type Epoch struct { + v atomic.Int64 +} + +// Load returns the registered epoch, or 0 before registration. +func (e *Epoch) Load() int64 { + return e.v.Load() +} + +// Store records the epoch this worker process registered with. +func (e *Epoch) Store(epoch int64) { + e.v.Store(epoch) +} + +// Registered is a readiness check that fails until this worker process has +// registered. The pod is then not Ready, so the Worker takes no placements +// while a restarted worker process has yet to register. +func (e *Epoch) Registered(context.Context) error { + if e.Load() == 0 { + return errors.New("worker has not registered yet") + } + return nil +} + +// Require returns the registered epoch for a workload about to start, or +// Unavailable before registration, so the caller retries once it has. +func (e *Epoch) Require() (int64, error) { + if epoch := e.Load(); epoch != 0 { + return epoch, nil + } + return 0, apierror.Unavailable("worker has not registered yet: not accepting workloads") +} + // FromFiles combines the downward API compute limits with the worker's own // actor limit. // @@ -71,6 +110,18 @@ func fromDir(dir string, actors int) *ateletpb.WorkerResources { } } +// ParsePodIPs splits the comma-separated pod IPs the downward API exposes for +// status.podIPs. +func ParsePodIPs(value string) []string { + var ips []string + for ip := range strings.SplitSeq(value, ",") { + if ip = strings.TrimSpace(ip); ip != "" { + ips = append(ips, ip) + } + } + return ips +} + // buildCompat converts the compatibility attributes atelet wrote for one // SandboxConfig version into a VersionedSandboxCompat. func buildCompat(attrs sandboxruntimes.Attributes) *ateletpb.VersionedSandboxCompat { @@ -93,9 +144,10 @@ func buildRuntime(rt sandboxruntimes.Runtime) *ateletpb.WorkerSandboxRuntime { return &ateletpb.WorkerSandboxRuntime{Version: buildCompat(rt.Version)} } -// buildRequest builds the RegisterWorkerRequest for capacity and one class -// of the sandbox runtimes file: the one this worker implements. -func buildRequest(capacity *ateletpb.WorkerResources, class *sandboxruntimes.Class) *ateletpb.RegisterWorkerRequest { +// buildRuntimesRequest builds the SetWorkerSandboxRuntimesRequest for one +// class of the sandbox runtimes file, the one this worker implements, for the +// worker process registered with epoch. +func buildRuntimesRequest(epoch int64, class *sandboxruntimes.Class) *ateletpb.SetWorkerSandboxRuntimesRequest { c := &ateletpb.WorkerSandboxClass{ SandboxClass: class.SandboxClass, DefaultRuntime: buildRuntime(class.DefaultRuntime), @@ -103,8 +155,8 @@ func buildRequest(capacity *ateletpb.WorkerResources, class *sandboxruntimes.Cla for _, rt := range class.RestorableRuntimes { c.RestorableRuntimes = append(c.RestorableRuntimes, buildRuntime(rt)) } - return &ateletpb.RegisterWorkerRequest{ - Capacity: capacity, + return &ateletpb.SetWorkerSandboxRuntimesRequest{ + Epoch: epoch, SandboxClasses: []*ateletpb.WorkerSandboxClass{c}, } } @@ -146,6 +198,8 @@ type ReportConfig struct { AteletSPIFFEID string // Actors is how many actors this worker will host at once. Actors int + // PodIPs are the worker pod's IPs, from the downward API. + PodIPs []string // SandboxClass is the class this worker implements ("gvisor", "microvm"). SandboxClass string // RuntimesFile is the sandbox runtimes file atelet keeps for this worker @@ -153,16 +207,27 @@ type ReportConfig struct { RuntimesFile string } -// Report keeps the worker registered with the node-local atelet until ctx -// ends: its capacity, and the runtimes in its sandbox runtimes file, which it -// reads at startup and again whenever atelet rewrites it. It returns early -// only on a configuration error, which no retry would fix. +// supportClient is the part of atelet's WorkerSupport that registration uses. +type supportClient interface { + AcquireWorkerEpoch(ctx context.Context) (int64, error) + RegisterWorker(ctx context.Context, req *ateletpb.RegisterWorkerRequest) error + SetWorkerSandboxRuntimes(ctx context.Context, req *ateletpb.SetWorkerSandboxRuntimesRequest) error +} + +// Report registers this worker process with the node-local atelet, stores the +// epoch it registered with in epoch, then keeps its sandbox runtimes reported +// until ctx ends: it reads its sandbox runtimes file at startup and again +// whenever atelet rewrites it. // -// Retrying is what makes a report durable: atelet only accepts once the +// It returns early on a configuration error, which no retry would fix, and +// when the control plane refuses this process's epoch, which means another +// worker process has registered since: this one must stop serving. +// +// Retrying is what makes registration durable: atelet only accepts once the // control plane has recorded it, and the Worker record may not exist yet when // a worker first comes up. Nothing else reports this, so giving up would leave // the Worker holding no capacity and hosting nothing. -func Report(ctx context.Context, cfg ReportConfig) error { +func Report(ctx context.Context, cfg ReportConfig, epoch *Epoch) error { if cfg.SandboxClass == "" { // The control plane refuses a runtime without a class. return errors.New("register worker: SandboxClass is required") @@ -170,49 +235,118 @@ func Report(ctx context.Context, cfg ReportConfig) error { if cfg.RuntimesFile == "" { return errors.New("register worker: RuntimesFile is required") } + if len(cfg.PodIPs) == 0 { + return errors.New("register worker: PodIPs is required") + } tlsConfig, err := ateletdial.TLSConfig(cfg.CredentialBundlePath, cfg.TrustBundlePath, cfg.AteletSPIFFEID) if err != nil { return fmt.Errorf("register worker: %w", err) } + client := &dialingClient{socketPath: cfg.SocketPath, tlsConfig: tlsConfig} changes := WatchFilePath(ctx, cfg.RuntimesFile) + return run(ctx, cfg, FromFiles(cfg.Actors), client, epoch, changes) +} + +// run registers, then reports runtimes until ctx ends or the epoch is refused. +func run(ctx context.Context, cfg ReportConfig, capacity *ateletpb.WorkerResources, client supportClient, epoch *Epoch, changes <-chan struct{}) error { + registered, err := registerOnce(ctx, client, &ateletpb.RegisterWorkerRequest{ + Ips: cfg.PodIPs, + Capacity: capacity, + }, initialReportBackoff, maxReportBackoff, time.After) + if err != nil { + return err + } + epoch.Store(registered) r := newReconciler( cfg, - FromFiles(cfg.Actors), + registered, func() (*sandboxruntimes.File, error) { return sandboxruntimes.Read(cfg.RuntimesFile) }, - func(ctx context.Context, req *ateletpb.RegisterWorkerRequest) error { - return reportOnce(ctx, cfg.SocketPath, tlsConfig, req) - }, + client.SetWorkerSandboxRuntimes, ) return r.run(ctx, changes) } -// reconciler registers the worker with the runtimes in its sandbox runtimes -// file whenever they differ from the ones it last registered. +// registerOnce acquires an epoch and registers this worker process with it, +// retrying with backoff until one registration is accepted, and returns the +// epoch. req carries the IPs and capacity; its epoch is set here. +// +// A registration that times out may still have landed, so it is resent with the +// same epoch, which the control plane accepts again without a write. A refusal +// of the epoch is returned, as is an invalid request. +func registerOnce(ctx context.Context, client supportClient, req *ateletpb.RegisterWorkerRequest, initialBackoff, maxBackoff time.Duration, after func(time.Duration) <-chan time.Time) (int64, error) { + backoff := initialBackoff + wait := func(err error, msg string) error { + slog.WarnContext(ctx, msg, slog.Any("err", err)) + select { + case <-ctx.Done(): + return ctx.Err() + case <-after(backoff): + } + backoff = min(backoff*2, maxBackoff) + return nil + } + + for req.Epoch == 0 { + epoch, err := client.AcquireWorkerEpoch(ctx) + if err == nil { + req.Epoch = epoch + break + } + if status.Code(err) == codes.InvalidArgument { + return 0, fmt.Errorf("register worker: acquiring an epoch: %w", err) + } + if err := wait(err, "Could not acquire a worker epoch; will retry"); err != nil { + return 0, err + } + } + for { + err := client.RegisterWorker(ctx, req) + if err == nil { + slog.InfoContext(ctx, "Registered worker", + slog.Int64("epoch", req.GetEpoch()), + slog.Any("ips", req.GetIps()), + slog.Any("capacity", req.GetCapacity())) + return req.GetEpoch(), nil + } + switch status.Code(err) { + case codes.InvalidArgument: + return 0, fmt.Errorf("register worker: %w", err) + case codes.FailedPrecondition: + return 0, fmt.Errorf("register worker: epoch %d refused: %w", req.GetEpoch(), err) + } + if err := wait(err, "Could not register worker; will retry"); err != nil { + return 0, err + } + } +} + +// reconciler reports the runtimes in the sandbox runtimes file whenever they +// differ from the ones it last reported. type reconciler struct { cfg ReportConfig - capacity *ateletpb.WorkerResources + epoch int64 read func() (*sandboxruntimes.File, error) - register func(context.Context, *ateletpb.RegisterWorkerRequest) error + report func(context.Context, *ateletpb.SetWorkerSandboxRuntimesRequest) error initialBackoff time.Duration maxBackoff time.Duration resync time.Duration after func(time.Duration) <-chan time.Time - // registered is the last request accepted by atelet, nil until the first - // registration succeeds. - registered *ateletpb.RegisterWorkerRequest + // reported is the last request accepted by atelet, nil until the first + // report succeeds. + reported *ateletpb.SetWorkerSandboxRuntimesRequest } func newReconciler( cfg ReportConfig, - capacity *ateletpb.WorkerResources, + epoch int64, read func() (*sandboxruntimes.File, error), - register func(context.Context, *ateletpb.RegisterWorkerRequest) error, + report func(context.Context, *ateletpb.SetWorkerSandboxRuntimesRequest) error, ) *reconciler { return &reconciler{ cfg: cfg, - capacity: capacity, + epoch: epoch, read: read, - register: register, + report: report, initialBackoff: initialReportBackoff, maxBackoff: maxReportBackoff, resync: resyncInterval, @@ -221,15 +355,20 @@ func newReconciler( } // run syncs at startup, on every change to the file, with backoff after a -// transient failure, and periodically at resyncInterval, until ctx ends. -// Change signals are ignored while backing off so periodic polling or rapid -// writes cannot bypass the backoff delay when atelet or ateapi is down. +// transient failure, and periodically at resyncInterval, until ctx ends or the +// control plane refuses the epoch. Change signals are ignored while backing off +// so periodic polling or rapid writes cannot bypass the backoff delay when +// atelet or ateapi is down. func (r *reconciler) run(ctx context.Context, changes <-chan struct{}) error { backoff := r.initialBackoff for { wait := r.resync onChange := changes - if r.sync(ctx) { + retry, err := r.sync(ctx) + if err != nil { + return err + } + if retry { wait = backoff backoff = min(backoff*2, r.maxBackoff) onChange = nil @@ -248,59 +387,91 @@ func (r *reconciler) run(ctx context.Context, changes <-chan struct{}) error { } } -// sync reads the file and registers if needed. It reports whether the caller -// should retry with backoff. -func (r *reconciler) sync(ctx context.Context) (retry bool) { +// sync reads the file and reports it if needed. It reports whether the caller +// should retry with backoff, and returns an error when the control plane +// refuses the epoch. +func (r *reconciler) sync(ctx context.Context) (retry bool, err error) { desired, err := r.read() if errors.Is(err, fs.ErrNotExist) { - if r.registered == nil { + if r.reported == nil { slog.InfoContext(ctx, "Waiting for the sandbox runtimes file, which atelet writes; see its logs if this persists", slog.String("path", r.cfg.RuntimesFile)) } - return false + return false, nil } if err != nil { slog.WarnContext(ctx, "Ignoring an unreadable sandbox runtimes file", slog.String("path", r.cfg.RuntimesFile), slog.Any("err", err)) - return false + return false, nil } class := classOf(desired, r.cfg.SandboxClass) if class == nil { slog.WarnContext(ctx, "Ignoring a sandbox runtimes file without this worker's sandbox class", slog.String("path", r.cfg.RuntimesFile), slog.String("sandbox_class", r.cfg.SandboxClass)) - return false + return false, nil } if len(class.DefaultRuntime.Version) == 0 { slog.WarnContext(ctx, "Ignoring a sandbox runtimes file with no default runtime", slog.String("path", r.cfg.RuntimesFile)) - return false + return false, nil } - req := buildRequest(r.capacity, class) - if proto.Equal(req, r.registered) { - return false + req := buildRuntimesRequest(r.epoch, class) + if proto.Equal(req, r.reported) { + return false, nil } - if err := r.register(ctx, req); err != nil { - if status.Code(err) == codes.InvalidArgument { - slog.ErrorContext(ctx, "Worker registration rejected as invalid; waiting for sandbox runtimes file to change", slog.Any("err", err)) - return false + if err := r.report(ctx, req); err != nil { + switch status.Code(err) { + case codes.InvalidArgument: + slog.ErrorContext(ctx, "Sandbox runtimes rejected as invalid; waiting for sandbox runtimes file to change", slog.Any("err", err)) + return false, nil + case codes.FailedPrecondition: + return false, fmt.Errorf("report sandbox runtimes: epoch %d refused: %w", r.epoch, err) } - slog.WarnContext(ctx, "Could not register worker; will retry", slog.Any("err", err)) - return true + slog.WarnContext(ctx, "Could not report sandbox runtimes; will retry", slog.Any("err", err)) + return true, nil } - r.registered = req - slog.InfoContext(ctx, "Registered worker capacity and sandbox classes", - slog.Any("capacity", req.GetCapacity()), + r.reported = req + slog.InfoContext(ctx, "Reported worker sandbox runtimes", + slog.Int64("epoch", r.epoch), slog.Any("sandbox_classes", req.GetSandboxClasses())) - return false + return false, nil +} + +// dialingClient dials atelet for each call. +type dialingClient struct { + socketPath string + tlsConfig *tls.Config +} + +func (c *dialingClient) AcquireWorkerEpoch(ctx context.Context) (int64, error) { + var epoch int64 + err := c.call(ctx, func(ctx context.Context, client ateletpb.WorkerSupportClient) error { + resp, err := client.AcquireWorkerEpoch(ctx, &ateletpb.AcquireWorkerEpochRequest{}) + epoch = resp.GetEpoch() + return err + }) + return epoch, err +} + +func (c *dialingClient) RegisterWorker(ctx context.Context, req *ateletpb.RegisterWorkerRequest) error { + return c.call(ctx, func(ctx context.Context, client ateletpb.WorkerSupportClient) error { + _, err := client.RegisterWorker(ctx, req) + return err + }) +} + +func (c *dialingClient) SetWorkerSandboxRuntimes(ctx context.Context, req *ateletpb.SetWorkerSandboxRuntimesRequest) error { + return c.call(ctx, func(ctx context.Context, client ateletpb.WorkerSupportClient) error { + _, err := client.SetWorkerSandboxRuntimes(ctx, req) + return err + }) } -// reportOnce dials atelet and sends one RegisterWorker. -func reportOnce(ctx context.Context, socketPath string, tlsConfig *tls.Config, req *ateletpb.RegisterWorkerRequest) error { - conn, err := ateletdial.Dial(socketPath, tlsConfig) +func (c *dialingClient) call(ctx context.Context, f func(context.Context, ateletpb.WorkerSupportClient) error) error { + conn, err := ateletdial.Dial(c.socketPath, c.tlsConfig) if err != nil { return err } defer conn.Close() callCtx, cancel := context.WithTimeout(ctx, reportTimeout) defer cancel() - _, err = ateletpb.NewWorkerSupportClient(conn).RegisterWorker(callCtx, req) - return err + return f(callCtx, ateletpb.NewWorkerSupportClient(conn)) } diff --git a/internal/ateworker/register/register_test.go b/internal/ateworker/register/register_test.go index e427024bc4..97570e9e78 100644 --- a/internal/ateworker/register/register_test.go +++ b/internal/ateworker/register/register_test.go @@ -27,8 +27,10 @@ import ( "github.com/google/go-cmp/cmp" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/testing/protocmp" + "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/hardware" "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/proto/ateletpb" @@ -105,8 +107,168 @@ func gvisorFile(def sandboxruntimes.Attributes, restorable ...sandboxruntimes.At return &sandboxruntimes.File{SandboxClasses: []sandboxruntimes.Class{c}} } -func TestReconcilerSync(t *testing.T) { +func TestParsePodIPs(t *testing.T) { + for _, tc := range []struct { + value string + want []string + }{ + {value: "", want: nil}, + {value: "10.0.0.1", want: []string{"10.0.0.1"}}, + {value: "10.0.0.1,fd00::1", want: []string{"10.0.0.1", "fd00::1"}}, + {value: " 10.0.0.1 , ,fd00::1,", want: []string{"10.0.0.1", "fd00::1"}}, + } { + if diff := cmp.Diff(tc.want, ParsePodIPs(tc.value)); diff != "" { + t.Errorf("ParsePodIPs(%q) mismatch (-want +got):\n%s", tc.value, diff) + } + } +} + +func TestEpochRequire(t *testing.T) { + var e Epoch + if _, err := e.Require(); apierror.Code(err) != codes.Unavailable { + t.Errorf("Require() before registration = %v, want Unavailable", err) + } + e.Store(3) + if got, err := e.Require(); err != nil || got != 3 { + t.Errorf("Require() after registration = (%d, %v), want (3, nil)", got, err) + } +} + +// The readiness check holds the pod not Ready until the worker process has +// registered, so a restarted one takes no placements before it can serve them. +func TestEpochRegistered(t *testing.T) { + var e Epoch + if err := e.Registered(context.Background()); err == nil { + t.Error("Registered() before registration = nil, want an error") + } + e.Store(3) + if err := e.Registered(context.Background()); err != nil { + t.Errorf("Registered() after registration = %v, want nil", err) + } +} + +// fakeSupport is a supportClient that fails the first acquireFails acquires +// and fails registration with registerErrs in turn. +type fakeSupport struct { + acquireFails int + acquires int + registerErrs []error + registered []*ateletpb.RegisterWorkerRequest +} + +func (f *fakeSupport) AcquireWorkerEpoch(context.Context) (int64, error) { + f.acquires++ + if f.acquires <= f.acquireFails { + return 0, status.Error(codes.NotFound, "worker record does not exist yet") + } + return int64(f.acquires), nil +} + +func (f *fakeSupport) RegisterWorker(_ context.Context, req *ateletpb.RegisterWorkerRequest) error { + f.registered = append(f.registered, proto.Clone(req).(*ateletpb.RegisterWorkerRequest)) + if len(f.registerErrs) > 0 { + err := f.registerErrs[0] + f.registerErrs = f.registerErrs[1:] + return err + } + return nil +} + +func (f *fakeSupport) SetWorkerSandboxRuntimes(context.Context, *ateletpb.SetWorkerSandboxRuntimesRequest) error { + return nil +} + +// immediately fires every timer, recording the durations armed. +func immediately(waits *[]time.Duration) func(time.Duration) <-chan time.Time { + return func(d time.Duration) <-chan time.Time { + *waits = append(*waits, d) + ch := make(chan time.Time, 1) + ch <- time.Now() + return ch + } +} + +func TestRegisterOnce(t *testing.T) { capacity := &ateletpb.WorkerResources{Actors: testActors} + ips := []string{"10.0.0.1"} + + t.Run("retries acquire and register, resending the same epoch", func(t *testing.T) { + client := &fakeSupport{ + acquireFails: 2, + registerErrs: []error{status.Error(codes.DeadlineExceeded, "timed out"), status.Error(codes.Aborted, "conflict")}, + } + var waits []time.Duration + epoch, err := registerOnce(context.Background(), client, &ateletpb.RegisterWorkerRequest{Ips: ips, Capacity: capacity}, 10*time.Millisecond, 25*time.Millisecond, immediately(&waits)) + if err != nil { + t.Fatalf("registerOnce() = %v, want nil", err) + } + if epoch != 3 { + t.Errorf("epoch = %d, want 3 (the first acquire that succeeded)", epoch) + } + if client.acquires != 3 { + t.Errorf("acquires = %d, want 3", client.acquires) + } + want := &ateletpb.RegisterWorkerRequest{Epoch: 3, Ips: ips, Capacity: capacity} + if diff := cmp.Diff([]*ateletpb.RegisterWorkerRequest{want, want, want}, client.registered, protocmp.Transform()); diff != "" { + t.Errorf("register calls mismatch (-want +got):\n%s", diff) + } + wantWaits := []time.Duration{10 * time.Millisecond, 20 * time.Millisecond, 25 * time.Millisecond, 25 * time.Millisecond} + if diff := cmp.Diff(wantWaits, waits); diff != "" { + t.Errorf("backoff mismatch (-want +got):\n%s", diff) + } + }) + + for _, code := range []codes.Code{codes.FailedPrecondition, codes.InvalidArgument} { + t.Run(code.String()+" is fatal", func(t *testing.T) { + client := &fakeSupport{registerErrs: []error{status.Error(code, "refused")}} + var waits []time.Duration + _, err := registerOnce(context.Background(), client, &ateletpb.RegisterWorkerRequest{Ips: ips, Capacity: capacity}, time.Millisecond, time.Millisecond, immediately(&waits)) + if status.Code(err) != code { + t.Errorf("registerOnce() = %v, want %v", err, code) + } + if len(client.registered) != 1 { + t.Errorf("register calls = %d, want 1", len(client.registered)) + } + }) + } +} + +func TestRunStoresEpochBeforeReportingRuntimes(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + path := filepath.Join(t.TempDir(), nodepath.SandboxRuntimesFileName) + if _, err := sandboxruntimes.Write(path, gvisorFile(sandboxruntimes.Attributes{hardware.AttrSandboxBundleHash: "sha256:v1"})); err != nil { + t.Fatal(err) + } + var epoch Epoch + client := &runtimesRecorder{fakeSupport: &fakeSupport{}, onReport: func(req *ateletpb.SetWorkerSandboxRuntimesRequest) { + if got := epoch.Load(); got != req.GetEpoch() { + t.Errorf("epoch stored = %d when runtimes were reported with %d", got, req.GetEpoch()) + } + cancel() + }} + err := run(ctx, ReportConfig{SandboxClass: "gvisor", RuntimesFile: path, PodIPs: []string{"10.0.0.1"}}, &ateletpb.WorkerResources{Actors: testActors}, client, &epoch, nil) + if !errors.Is(err, context.Canceled) { + t.Fatalf("run() = %v, want context.Canceled", err) + } + if epoch.Load() != 1 { + t.Errorf("epoch = %d, want 1", epoch.Load()) + } +} + +type runtimesRecorder struct { + *fakeSupport + onReport func(*ateletpb.SetWorkerSandboxRuntimesRequest) +} + +func (r *runtimesRecorder) SetWorkerSandboxRuntimes(_ context.Context, req *ateletpb.SetWorkerSandboxRuntimesRequest) error { + r.onReport(req) + return nil +} + +func TestReconcilerSync(t *testing.T) { + const epoch = 4 v1 := gvisorFile(sandboxruntimes.Attributes{ hardware.AttrArchitecture: runtime.GOARCH, hardware.AttrCPUFeatures: "3", @@ -117,17 +279,18 @@ func TestReconcilerSync(t *testing.T) { hardware.AttrCPUFeatures: "7", hardware.AttrSandboxBundleHash: "sha256:v2", }, v1.SandboxClasses[0].DefaultRuntime.Version) - reqV1 := buildRequest(capacity, &v1.SandboxClasses[0]) - reqV2 := buildRequest(capacity, &v2.SandboxClasses[0]) + reqV1 := buildRuntimesRequest(epoch, &v1.SandboxClasses[0]) + reqV2 := buildRuntimesRequest(epoch, &v2.SandboxClasses[0]) tests := []struct { name string setupFile func(t *testing.T, path string) - registered *ateletpb.RegisterWorkerRequest + registered *ateletpb.SetWorkerSandboxRuntimesRequest registerErr error wantRetry bool - wantCalls []*ateletpb.RegisterWorkerRequest - wantRegistered *ateletpb.RegisterWorkerRequest + wantErr bool + wantCalls []*ateletpb.SetWorkerSandboxRuntimesRequest + wantRegistered *ateletpb.SetWorkerSandboxRuntimesRequest }{ { // File not written yet: waits for fsnotify rather than spinning in backoff. @@ -191,6 +354,17 @@ func TestReconcilerSync(t *testing.T) { registerErr: status.Error(codes.InvalidArgument, "invalid attribute key"), wantRetry: false, }, + { + // Another worker process registered since: this one must stop. + name: "FailedPrecondition from report is an error", + setupFile: func(t *testing.T, path string) { + if _, err := sandboxruntimes.Write(path, v1); err != nil { + t.Fatal(err) + } + }, + registerErr: status.Error(codes.FailedPrecondition, "epoch superseded"), + wantErr: true, + }, { name: "first success registers and records request", setupFile: func(t *testing.T, path string) { @@ -198,7 +372,7 @@ func TestReconcilerSync(t *testing.T) { t.Fatal(err) } }, - wantCalls: []*ateletpb.RegisterWorkerRequest{reqV1}, + wantCalls: []*ateletpb.SetWorkerSandboxRuntimesRequest{reqV1}, wantRegistered: reqV1, }, { @@ -219,7 +393,7 @@ func TestReconcilerSync(t *testing.T) { } }, registered: reqV1, - wantCalls: []*ateletpb.RegisterWorkerRequest{reqV2}, + wantCalls: []*ateletpb.SetWorkerSandboxRuntimesRequest{reqV2}, wantRegistered: reqV2, }, } @@ -229,12 +403,12 @@ func TestReconcilerSync(t *testing.T) { path := filepath.Join(t.TempDir(), nodepath.SandboxRuntimesFileName) tc.setupFile(t, path) - var calls []*ateletpb.RegisterWorkerRequest + var calls []*ateletpb.SetWorkerSandboxRuntimesRequest r := newReconciler( ReportConfig{SandboxClass: "gvisor", RuntimesFile: path}, - capacity, + epoch, func() (*sandboxruntimes.File, error) { return sandboxruntimes.Read(path) }, - func(_ context.Context, req *ateletpb.RegisterWorkerRequest) error { + func(_ context.Context, req *ateletpb.SetWorkerSandboxRuntimesRequest) error { if tc.registerErr != nil { return tc.registerErr } @@ -242,16 +416,19 @@ func TestReconcilerSync(t *testing.T) { return nil }, ) - r.registered = tc.registered + r.reported = tc.registered - gotRetry := r.sync(context.Background()) + gotRetry, err := r.sync(context.Background()) if gotRetry != tc.wantRetry { t.Errorf("sync() retry = %v, want %v", gotRetry, tc.wantRetry) } + if (err != nil) != tc.wantErr { + t.Errorf("sync() err = %v, want error %v", err, tc.wantErr) + } if diff := cmp.Diff(tc.wantCalls, calls, protocmp.Transform()); diff != "" { t.Errorf("register calls mismatch (-want +got):\n%s", diff) } - if diff := cmp.Diff(tc.wantRegistered, r.registered, protocmp.Transform()); diff != "" { + if diff := cmp.Diff(tc.wantRegistered, r.reported, protocmp.Transform()); diff != "" { t.Errorf("registered state mismatch (-want +got):\n%s", diff) } }) @@ -277,9 +454,9 @@ func TestReconcilerRunBackoff(t *testing.T) { ) r := newReconciler( ReportConfig{SandboxClass: "gvisor", RuntimesFile: "sandbox-runtimes.json"}, - &ateletpb.WorkerResources{Actors: testActors}, + 1, func() (*sandboxruntimes.File, error) { return file, nil }, - func(_ context.Context, _ *ateletpb.RegisterWorkerRequest) error { + func(_ context.Context, _ *ateletpb.SetWorkerSandboxRuntimesRequest) error { attempts++ switch attempts { case 1, 2, 3: @@ -346,9 +523,9 @@ func TestReconcilerIgnoresChangesDuringBackoff(t *testing.T) { attempts := 0 r := newReconciler( ReportConfig{SandboxClass: "gvisor", RuntimesFile: "sandbox-runtimes.json"}, - &ateletpb.WorkerResources{Actors: testActors}, + 1, func() (*sandboxruntimes.File, error) { return file, nil }, - func(context.Context, *ateletpb.RegisterWorkerRequest) error { + func(context.Context, *ateletpb.SetWorkerSandboxRuntimesRequest) error { attempts++ started <- attempts if attempts == 1 { @@ -417,7 +594,7 @@ func TestReconcilerRunResync(t *testing.T) { var reads int r := newReconciler( ReportConfig{SandboxClass: "gvisor", RuntimesFile: "sandbox-runtimes.json"}, - &ateletpb.WorkerResources{Actors: testActors}, + 1, func() (*sandboxruntimes.File, error) { reads++ if reads >= 2 { @@ -425,7 +602,7 @@ func TestReconcilerRunResync(t *testing.T) { } return gvisorFile(sandboxruntimes.Attributes{hardware.AttrSandboxBundleHash: "sha256:v1"}), nil }, - func(context.Context, *ateletpb.RegisterWorkerRequest) error { return nil }, + func(context.Context, *ateletpb.SetWorkerSandboxRuntimesRequest) error { return nil }, ) var armed []time.Duration r.after = func(d time.Duration) <-chan time.Time { @@ -540,12 +717,12 @@ func TestReconcilerRunWatchesFile(t *testing.T) { defer cancel() path := filepath.Join(t.TempDir(), nodepath.SandboxRuntimesFileName) - got := make(chan *ateletpb.RegisterWorkerRequest, 4) + got := make(chan *ateletpb.SetWorkerSandboxRuntimesRequest, 4) r := newReconciler( ReportConfig{SandboxClass: "gvisor", RuntimesFile: path}, - &ateletpb.WorkerResources{Actors: testActors}, + 1, func() (*sandboxruntimes.File, error) { return sandboxruntimes.Read(path) }, - func(_ context.Context, req *ateletpb.RegisterWorkerRequest) error { + func(_ context.Context, req *ateletpb.SetWorkerSandboxRuntimesRequest) error { got <- req return nil }, @@ -569,7 +746,7 @@ func TestReconcilerRunWatchesFile(t *testing.T) { } select { case req := <-got: - if diff := cmp.Diff(buildRequest(r.capacity, &v1.SandboxClasses[0]), req, protocmp.Transform()); diff != "" { + if diff := cmp.Diff(buildRuntimesRequest(r.epoch, &v1.SandboxClasses[0]), req, protocmp.Transform()); diff != "" { t.Errorf("initial registration mismatch (-want +got):\n%s", diff) } case <-ctx.Done(): @@ -588,7 +765,7 @@ func TestReconcilerRunWatchesFile(t *testing.T) { } select { case req := <-got: - if diff := cmp.Diff(buildRequest(r.capacity, &v2.SandboxClasses[0]), req, protocmp.Transform()); diff != "" { + if diff := cmp.Diff(buildRuntimesRequest(r.epoch, &v2.SandboxClasses[0]), req, protocmp.Transform()); diff != "" { t.Errorf("updated registration mismatch (-want +got):\n%s", diff) } case <-ctx.Done(): @@ -612,9 +789,10 @@ func TestReportFailsFastOnBadCredentials(t *testing.T) { SocketPath: filepath.Join(t.TempDir(), "atelet.sock"), CredentialBundlePath: filepath.Join(t.TempDir(), "does-not-exist.pem"), TrustBundlePath: filepath.Join(t.TempDir(), "also-missing.pem"), + PodIPs: []string{"10.0.0.1"}, SandboxClass: "gvisor", RuntimesFile: filepath.Join(t.TempDir(), nodepath.SandboxRuntimesFileName), - }) + }, &Epoch{}) if err == nil { t.Fatal("Report() with unreadable credentials succeeded, want an error the caller can exit on") } @@ -647,9 +825,17 @@ func TestReportFailsFastOnMissingConfig(t *testing.T) { SandboxClass: "gvisor", }, }, + { + name: "missing PodIPs", + cfg: ReportConfig{ + SocketPath: filepath.Join(t.TempDir(), "atelet.sock"), + SandboxClass: "gvisor", + RuntimesFile: filepath.Join(t.TempDir(), nodepath.SandboxRuntimesFileName), + }, + }, } { t.Run(tc.name, func(t *testing.T) { - err := Report(ctx, tc.cfg) + err := Report(ctx, tc.cfg, &Epoch{}) if err == nil { t.Fatal("Report() succeeded, want an error the caller can exit on") } diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index a2dbbbe46b..754fb1b093 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -271,13 +271,168 @@ func (UploadMode) EnumDescriptor() ([]byte, []int) { return file_atelet_proto_rawDescGZIP(), []int{3} } +type AcquireWorkerEpochRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AcquireWorkerEpochRequest) Reset() { + *x = AcquireWorkerEpochRequest{} + mi := &file_atelet_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AcquireWorkerEpochRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AcquireWorkerEpochRequest) ProtoMessage() {} + +func (x *AcquireWorkerEpochRequest) ProtoReflect() protoreflect.Message { + mi := &file_atelet_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AcquireWorkerEpochRequest.ProtoReflect.Descriptor instead. +func (*AcquireWorkerEpochRequest) Descriptor() ([]byte, []int) { + return file_atelet_proto_rawDescGZIP(), []int{0} +} + +type AcquireWorkerEpochResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The epoch handed out, for this worker process to register with. + Epoch int64 `protobuf:"varint,1,opt,name=epoch,proto3" json:"epoch,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AcquireWorkerEpochResponse) Reset() { + *x = AcquireWorkerEpochResponse{} + mi := &file_atelet_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AcquireWorkerEpochResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AcquireWorkerEpochResponse) ProtoMessage() {} + +func (x *AcquireWorkerEpochResponse) ProtoReflect() protoreflect.Message { + mi := &file_atelet_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AcquireWorkerEpochResponse.ProtoReflect.Descriptor instead. +func (*AcquireWorkerEpochResponse) Descriptor() ([]byte, []int) { + return file_atelet_proto_rawDescGZIP(), []int{1} +} + +func (x *AcquireWorkerEpochResponse) GetEpoch() int64 { + if x != nil { + return x.Epoch + } + return 0 +} + type RegisterWorkerRequest struct { state protoimpl.MessageState `protogen:"open.v1"` + // The epoch this worker process acquired through AcquireWorkerEpoch. + // + // +k8s:required + // +k8s:minimum=1 + Epoch int64 `protobuf:"varint,2,opt,name=epoch,proto3" json:"epoch,omitempty"` + // The worker pod's IPs, at most one per IP family, in the order Kubernetes + // reports them. + // + // +k8s:required + // +k8s:maxItems=2 + // +k8s:listType=atomic + Ips []string `protobuf:"bytes,3,rep,name=ips,proto3" json:"ips,omitempty"` // What the worker can supply. atelet converts it to the control plane's // WorkerResources when forwarding. // // +k8s:required - Capacity *WorkerResources `protobuf:"bytes,1,opt,name=capacity,proto3" json:"capacity,omitempty"` + Capacity *WorkerResources `protobuf:"bytes,1,opt,name=capacity,proto3" json:"capacity,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RegisterWorkerRequest) Reset() { + *x = RegisterWorkerRequest{} + mi := &file_atelet_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RegisterWorkerRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RegisterWorkerRequest) ProtoMessage() {} + +func (x *RegisterWorkerRequest) ProtoReflect() protoreflect.Message { + mi := &file_atelet_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RegisterWorkerRequest.ProtoReflect.Descriptor instead. +func (*RegisterWorkerRequest) Descriptor() ([]byte, []int) { + return file_atelet_proto_rawDescGZIP(), []int{2} +} + +func (x *RegisterWorkerRequest) GetEpoch() int64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *RegisterWorkerRequest) GetIps() []string { + if x != nil { + return x.Ips + } + return nil +} + +func (x *RegisterWorkerRequest) GetCapacity() *WorkerResources { + if x != nil { + return x.Capacity + } + return nil +} + +type SetWorkerSandboxRuntimesRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The epoch this worker process registered with. + // + // +k8s:required + // +k8s:minimum=1 + Epoch int64 `protobuf:"varint,1,opt,name=epoch,proto3" json:"epoch,omitempty"` // The sandbox classes this worker runs and the compatibility versions it // runs them under. atelet converts it to the control plane's // WorkerSandboxClass when forwarding. @@ -292,21 +447,21 @@ type RegisterWorkerRequest struct { sizeCache protoimpl.SizeCache } -func (x *RegisterWorkerRequest) Reset() { - *x = RegisterWorkerRequest{} - mi := &file_atelet_proto_msgTypes[0] +func (x *SetWorkerSandboxRuntimesRequest) Reset() { + *x = SetWorkerSandboxRuntimesRequest{} + mi := &file_atelet_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *RegisterWorkerRequest) String() string { +func (x *SetWorkerSandboxRuntimesRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*RegisterWorkerRequest) ProtoMessage() {} +func (*SetWorkerSandboxRuntimesRequest) ProtoMessage() {} -func (x *RegisterWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[0] +func (x *SetWorkerSandboxRuntimesRequest) ProtoReflect() protoreflect.Message { + mi := &file_atelet_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -317,25 +472,61 @@ func (x *RegisterWorkerRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use RegisterWorkerRequest.ProtoReflect.Descriptor instead. -func (*RegisterWorkerRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{0} +// Deprecated: Use SetWorkerSandboxRuntimesRequest.ProtoReflect.Descriptor instead. +func (*SetWorkerSandboxRuntimesRequest) Descriptor() ([]byte, []int) { + return file_atelet_proto_rawDescGZIP(), []int{3} } -func (x *RegisterWorkerRequest) GetCapacity() *WorkerResources { +func (x *SetWorkerSandboxRuntimesRequest) GetEpoch() int64 { if x != nil { - return x.Capacity + return x.Epoch } - return nil + return 0 } -func (x *RegisterWorkerRequest) GetSandboxClasses() []*WorkerSandboxClass { +func (x *SetWorkerSandboxRuntimesRequest) GetSandboxClasses() []*WorkerSandboxClass { if x != nil { return x.SandboxClasses } return nil } +type SetWorkerSandboxRuntimesResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SetWorkerSandboxRuntimesResponse) Reset() { + *x = SetWorkerSandboxRuntimesResponse{} + mi := &file_atelet_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SetWorkerSandboxRuntimesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SetWorkerSandboxRuntimesResponse) ProtoMessage() {} + +func (x *SetWorkerSandboxRuntimesResponse) ProtoReflect() protoreflect.Message { + mi := &file_atelet_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SetWorkerSandboxRuntimesResponse.ProtoReflect.Descriptor instead. +func (*SetWorkerSandboxRuntimesResponse) Descriptor() ([]byte, []int) { + return file_atelet_proto_rawDescGZIP(), []int{4} +} + // WorkerSandboxClass is one sandbox class a worker runs, with the runtimes // it runs it under. It mirrors the control plane's WorkerSandboxClass and is // validated by the same rules. @@ -365,7 +556,7 @@ type WorkerSandboxClass struct { func (x *WorkerSandboxClass) Reset() { *x = WorkerSandboxClass{} - mi := &file_atelet_proto_msgTypes[1] + mi := &file_atelet_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -377,7 +568,7 @@ func (x *WorkerSandboxClass) String() string { func (*WorkerSandboxClass) ProtoMessage() {} func (x *WorkerSandboxClass) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[1] + mi := &file_atelet_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -390,7 +581,7 @@ func (x *WorkerSandboxClass) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerSandboxClass.ProtoReflect.Descriptor instead. func (*WorkerSandboxClass) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{1} + return file_atelet_proto_rawDescGZIP(), []int{5} } func (x *WorkerSandboxClass) GetSandboxClass() string { @@ -431,7 +622,7 @@ type WorkerSandboxRuntime struct { func (x *WorkerSandboxRuntime) Reset() { *x = WorkerSandboxRuntime{} - mi := &file_atelet_proto_msgTypes[2] + mi := &file_atelet_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -443,7 +634,7 @@ func (x *WorkerSandboxRuntime) String() string { func (*WorkerSandboxRuntime) ProtoMessage() {} func (x *WorkerSandboxRuntime) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[2] + mi := &file_atelet_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -456,7 +647,7 @@ func (x *WorkerSandboxRuntime) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerSandboxRuntime.ProtoReflect.Descriptor instead. func (*WorkerSandboxRuntime) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{2} + return file_atelet_proto_rawDescGZIP(), []int{6} } func (x *WorkerSandboxRuntime) GetVersion() *VersionedSandboxCompat { @@ -487,7 +678,7 @@ type ActorSandboxRuntime struct { func (x *ActorSandboxRuntime) Reset() { *x = ActorSandboxRuntime{} - mi := &file_atelet_proto_msgTypes[3] + mi := &file_atelet_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -499,7 +690,7 @@ func (x *ActorSandboxRuntime) String() string { func (*ActorSandboxRuntime) ProtoMessage() {} func (x *ActorSandboxRuntime) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[3] + mi := &file_atelet_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -512,7 +703,7 @@ func (x *ActorSandboxRuntime) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorSandboxRuntime.ProtoReflect.Descriptor instead. func (*ActorSandboxRuntime) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{3} + return file_atelet_proto_rawDescGZIP(), []int{7} } func (x *ActorSandboxRuntime) GetSandboxClass() string { @@ -562,7 +753,7 @@ type VersionedSandboxCompat struct { func (x *VersionedSandboxCompat) Reset() { *x = VersionedSandboxCompat{} - mi := &file_atelet_proto_msgTypes[4] + mi := &file_atelet_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -574,7 +765,7 @@ func (x *VersionedSandboxCompat) String() string { func (*VersionedSandboxCompat) ProtoMessage() {} func (x *VersionedSandboxCompat) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[4] + mi := &file_atelet_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -587,7 +778,7 @@ func (x *VersionedSandboxCompat) ProtoReflect() protoreflect.Message { // Deprecated: Use VersionedSandboxCompat.ProtoReflect.Descriptor instead. func (*VersionedSandboxCompat) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{4} + return file_atelet_proto_rawDescGZIP(), []int{8} } func (x *VersionedSandboxCompat) GetSchemaVersion() string { @@ -623,7 +814,7 @@ type AttributeEntry struct { func (x *AttributeEntry) Reset() { *x = AttributeEntry{} - mi := &file_atelet_proto_msgTypes[5] + mi := &file_atelet_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -635,7 +826,7 @@ func (x *AttributeEntry) String() string { func (*AttributeEntry) ProtoMessage() {} func (x *AttributeEntry) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[5] + mi := &file_atelet_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -648,7 +839,7 @@ func (x *AttributeEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use AttributeEntry.ProtoReflect.Descriptor instead. func (*AttributeEntry) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{5} + return file_atelet_proto_rawDescGZIP(), []int{9} } func (x *AttributeEntry) GetKey() string { @@ -695,7 +886,7 @@ type WorkerResources struct { func (x *WorkerResources) Reset() { *x = WorkerResources{} - mi := &file_atelet_proto_msgTypes[6] + mi := &file_atelet_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -707,7 +898,7 @@ func (x *WorkerResources) String() string { func (*WorkerResources) ProtoMessage() {} func (x *WorkerResources) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[6] + mi := &file_atelet_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -720,7 +911,7 @@ func (x *WorkerResources) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerResources.ProtoReflect.Descriptor instead. func (*WorkerResources) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{6} + return file_atelet_proto_rawDescGZIP(), []int{10} } func (x *WorkerResources) GetCpuMillicores() int64 { @@ -757,7 +948,7 @@ type ResourceRequirements struct { func (x *ResourceRequirements) Reset() { *x = ResourceRequirements{} - mi := &file_atelet_proto_msgTypes[7] + mi := &file_atelet_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -769,7 +960,7 @@ func (x *ResourceRequirements) String() string { func (*ResourceRequirements) ProtoMessage() {} func (x *ResourceRequirements) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[7] + mi := &file_atelet_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -782,7 +973,7 @@ func (x *ResourceRequirements) ProtoReflect() protoreflect.Message { // Deprecated: Use ResourceRequirements.ProtoReflect.Descriptor instead. func (*ResourceRequirements) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{7} + return file_atelet_proto_rawDescGZIP(), []int{11} } func (x *ResourceRequirements) GetLimits() *Resources { @@ -812,7 +1003,7 @@ type Resources struct { func (x *Resources) Reset() { *x = Resources{} - mi := &file_atelet_proto_msgTypes[8] + mi := &file_atelet_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -824,7 +1015,7 @@ func (x *Resources) String() string { func (*Resources) ProtoMessage() {} func (x *Resources) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[8] + mi := &file_atelet_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -837,7 +1028,7 @@ func (x *Resources) ProtoReflect() protoreflect.Message { // Deprecated: Use Resources.ProtoReflect.Descriptor instead. func (*Resources) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{8} + return file_atelet_proto_rawDescGZIP(), []int{12} } func (x *Resources) GetCpuMillicores() int64 { @@ -862,7 +1053,7 @@ type RegisterWorkerResponse struct { func (x *RegisterWorkerResponse) Reset() { *x = RegisterWorkerResponse{} - mi := &file_atelet_proto_msgTypes[9] + mi := &file_atelet_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -874,7 +1065,7 @@ func (x *RegisterWorkerResponse) String() string { func (*RegisterWorkerResponse) ProtoMessage() {} func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[9] + mi := &file_atelet_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -887,7 +1078,7 @@ func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerResponse.ProtoReflect.Descriptor instead. func (*RegisterWorkerResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{9} + return file_atelet_proto_rawDescGZIP(), []int{13} } type RequestActorSuspendRequest struct { @@ -914,7 +1105,7 @@ type RequestActorSuspendRequest struct { func (x *RequestActorSuspendRequest) Reset() { *x = RequestActorSuspendRequest{} - mi := &file_atelet_proto_msgTypes[10] + mi := &file_atelet_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -926,7 +1117,7 @@ func (x *RequestActorSuspendRequest) String() string { func (*RequestActorSuspendRequest) ProtoMessage() {} func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[10] + mi := &file_atelet_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -939,7 +1130,7 @@ func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendRequest.ProtoReflect.Descriptor instead. func (*RequestActorSuspendRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{10} + return file_atelet_proto_rawDescGZIP(), []int{14} } func (x *RequestActorSuspendRequest) GetActorAtespace() string { @@ -971,7 +1162,7 @@ type RequestActorSuspendResponse struct { func (x *RequestActorSuspendResponse) Reset() { *x = RequestActorSuspendResponse{} - mi := &file_atelet_proto_msgTypes[11] + mi := &file_atelet_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -983,7 +1174,7 @@ func (x *RequestActorSuspendResponse) String() string { func (*RequestActorSuspendResponse) ProtoMessage() {} func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[11] + mi := &file_atelet_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -996,7 +1187,7 @@ func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendResponse.ProtoReflect.Descriptor instead. func (*RequestActorSuspendResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{11} + return file_atelet_proto_rawDescGZIP(), []int{15} } type ReportActorCrashedRequest struct { @@ -1023,7 +1214,7 @@ type ReportActorCrashedRequest struct { func (x *ReportActorCrashedRequest) Reset() { *x = ReportActorCrashedRequest{} - mi := &file_atelet_proto_msgTypes[12] + mi := &file_atelet_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1035,7 +1226,7 @@ func (x *ReportActorCrashedRequest) String() string { func (*ReportActorCrashedRequest) ProtoMessage() {} func (x *ReportActorCrashedRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[12] + mi := &file_atelet_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1048,7 +1239,7 @@ func (x *ReportActorCrashedRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportActorCrashedRequest.ProtoReflect.Descriptor instead. func (*ReportActorCrashedRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{12} + return file_atelet_proto_rawDescGZIP(), []int{16} } func (x *ReportActorCrashedRequest) GetActorAtespace() string { @@ -1087,7 +1278,7 @@ type ReportActorCrashedResponse struct { func (x *ReportActorCrashedResponse) Reset() { *x = ReportActorCrashedResponse{} - mi := &file_atelet_proto_msgTypes[13] + mi := &file_atelet_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1099,7 +1290,7 @@ func (x *ReportActorCrashedResponse) String() string { func (*ReportActorCrashedResponse) ProtoMessage() {} func (x *ReportActorCrashedResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[13] + mi := &file_atelet_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1112,7 +1303,7 @@ func (x *ReportActorCrashedResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportActorCrashedResponse.ProtoReflect.Descriptor instead. func (*ReportActorCrashedResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{13} + return file_atelet_proto_rawDescGZIP(), []int{17} } type GetActorEgressPolicyRequest struct { @@ -1132,7 +1323,7 @@ type GetActorEgressPolicyRequest struct { func (x *GetActorEgressPolicyRequest) Reset() { *x = GetActorEgressPolicyRequest{} - mi := &file_atelet_proto_msgTypes[14] + mi := &file_atelet_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1144,7 +1335,7 @@ func (x *GetActorEgressPolicyRequest) String() string { func (*GetActorEgressPolicyRequest) ProtoMessage() {} func (x *GetActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[14] + mi := &file_atelet_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1157,7 +1348,7 @@ func (x *GetActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActorEgressPolicyRequest.ProtoReflect.Descriptor instead. func (*GetActorEgressPolicyRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{14} + return file_atelet_proto_rawDescGZIP(), []int{18} } func (x *GetActorEgressPolicyRequest) GetActorAtespace() string { @@ -1185,7 +1376,7 @@ type GetActorEgressPolicyResponse struct { func (x *GetActorEgressPolicyResponse) Reset() { *x = GetActorEgressPolicyResponse{} - mi := &file_atelet_proto_msgTypes[15] + mi := &file_atelet_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1197,7 +1388,7 @@ func (x *GetActorEgressPolicyResponse) String() string { func (*GetActorEgressPolicyResponse) ProtoMessage() {} func (x *GetActorEgressPolicyResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[15] + mi := &file_atelet_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1210,7 +1401,7 @@ func (x *GetActorEgressPolicyResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActorEgressPolicyResponse.ProtoReflect.Descriptor instead. func (*GetActorEgressPolicyResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{15} + return file_atelet_proto_rawDescGZIP(), []int{19} } func (x *GetActorEgressPolicyResponse) GetEgressPolicy() *EgressPolicy { @@ -1234,7 +1425,7 @@ type EgressPolicy struct { func (x *EgressPolicy) Reset() { *x = EgressPolicy{} - mi := &file_atelet_proto_msgTypes[16] + mi := &file_atelet_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1246,7 +1437,7 @@ func (x *EgressPolicy) String() string { func (*EgressPolicy) ProtoMessage() {} func (x *EgressPolicy) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[16] + mi := &file_atelet_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1259,7 +1450,7 @@ func (x *EgressPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressPolicy.ProtoReflect.Descriptor instead. func (*EgressPolicy) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{16} + return file_atelet_proto_rawDescGZIP(), []int{20} } func (x *EgressPolicy) GetRules() []*EgressRule { @@ -1286,7 +1477,7 @@ type EgressRule struct { func (x *EgressRule) Reset() { *x = EgressRule{} - mi := &file_atelet_proto_msgTypes[17] + mi := &file_atelet_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1298,7 +1489,7 @@ func (x *EgressRule) String() string { func (*EgressRule) ProtoMessage() {} func (x *EgressRule) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[17] + mi := &file_atelet_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1311,7 +1502,7 @@ func (x *EgressRule) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressRule.ProtoReflect.Descriptor instead. func (*EgressRule) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{17} + return file_atelet_proto_rawDescGZIP(), []int{21} } func (x *EgressRule) GetHttp() *EgressDestinations { @@ -1349,7 +1540,7 @@ type EgressDestinations struct { func (x *EgressDestinations) Reset() { *x = EgressDestinations{} - mi := &file_atelet_proto_msgTypes[18] + mi := &file_atelet_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1361,7 +1552,7 @@ func (x *EgressDestinations) String() string { func (*EgressDestinations) ProtoMessage() {} func (x *EgressDestinations) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[18] + mi := &file_atelet_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1374,7 +1565,7 @@ func (x *EgressDestinations) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressDestinations.ProtoReflect.Descriptor instead. func (*EgressDestinations) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{18} + return file_atelet_proto_rawDescGZIP(), []int{22} } func (x *EgressDestinations) GetHostnames() []string { @@ -1405,7 +1596,7 @@ type EgressPorts struct { func (x *EgressPorts) Reset() { *x = EgressPorts{} - mi := &file_atelet_proto_msgTypes[19] + mi := &file_atelet_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1417,7 +1608,7 @@ func (x *EgressPorts) String() string { func (*EgressPorts) ProtoMessage() {} func (x *EgressPorts) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[19] + mi := &file_atelet_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1430,7 +1621,7 @@ func (x *EgressPorts) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressPorts.ProtoReflect.Descriptor instead. func (*EgressPorts) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{19} + return file_atelet_proto_rawDescGZIP(), []int{23} } func (x *EgressPorts) GetAll() *EgressAllPorts { @@ -1456,7 +1647,7 @@ type EgressAllPorts struct { func (x *EgressAllPorts) Reset() { *x = EgressAllPorts{} - mi := &file_atelet_proto_msgTypes[20] + mi := &file_atelet_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1468,7 +1659,7 @@ func (x *EgressAllPorts) String() string { func (*EgressAllPorts) ProtoMessage() {} func (x *EgressAllPorts) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[20] + mi := &file_atelet_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1481,7 +1672,7 @@ func (x *EgressAllPorts) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressAllPorts.ProtoReflect.Descriptor instead. func (*EgressAllPorts) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{20} + return file_atelet_proto_rawDescGZIP(), []int{24} } type MintActorCertificateRequest struct { @@ -1512,7 +1703,7 @@ type MintActorCertificateRequest struct { func (x *MintActorCertificateRequest) Reset() { *x = MintActorCertificateRequest{} - mi := &file_atelet_proto_msgTypes[21] + mi := &file_atelet_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1524,7 +1715,7 @@ func (x *MintActorCertificateRequest) String() string { func (*MintActorCertificateRequest) ProtoMessage() {} func (x *MintActorCertificateRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[21] + mi := &file_atelet_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1537,7 +1728,7 @@ func (x *MintActorCertificateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorCertificateRequest.ProtoReflect.Descriptor instead. func (*MintActorCertificateRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{21} + return file_atelet_proto_rawDescGZIP(), []int{25} } func (x *MintActorCertificateRequest) GetActorAtespace() string { @@ -1578,7 +1769,7 @@ type MintActorCertificateResponse struct { func (x *MintActorCertificateResponse) Reset() { *x = MintActorCertificateResponse{} - mi := &file_atelet_proto_msgTypes[22] + mi := &file_atelet_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1590,7 +1781,7 @@ func (x *MintActorCertificateResponse) String() string { func (*MintActorCertificateResponse) ProtoMessage() {} func (x *MintActorCertificateResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[22] + mi := &file_atelet_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1603,7 +1794,7 @@ func (x *MintActorCertificateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorCertificateResponse.ProtoReflect.Descriptor instead. func (*MintActorCertificateResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{22} + return file_atelet_proto_rawDescGZIP(), []int{26} } func (x *MintActorCertificateResponse) GetActorCertificates() [][]byte { @@ -1649,7 +1840,7 @@ type TerminateRequest struct { func (x *TerminateRequest) Reset() { *x = TerminateRequest{} - mi := &file_atelet_proto_msgTypes[23] + mi := &file_atelet_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1661,7 +1852,7 @@ func (x *TerminateRequest) String() string { func (*TerminateRequest) ProtoMessage() {} func (x *TerminateRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[23] + mi := &file_atelet_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1674,7 +1865,7 @@ func (x *TerminateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use TerminateRequest.ProtoReflect.Descriptor instead. func (*TerminateRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{23} + return file_atelet_proto_rawDescGZIP(), []int{27} } func (x *TerminateRequest) GetWorkerPodUid() string { @@ -1734,7 +1925,7 @@ type TerminateResponse struct { func (x *TerminateResponse) Reset() { *x = TerminateResponse{} - mi := &file_atelet_proto_msgTypes[24] + mi := &file_atelet_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1746,7 +1937,7 @@ func (x *TerminateResponse) String() string { func (*TerminateResponse) ProtoMessage() {} func (x *TerminateResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[24] + mi := &file_atelet_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1759,7 +1950,7 @@ func (x *TerminateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use TerminateResponse.ProtoReflect.Descriptor instead. func (*TerminateResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{24} + return file_atelet_proto_rawDescGZIP(), []int{28} } type RunRequest struct { @@ -1811,7 +2002,7 @@ type RunRequest struct { func (x *RunRequest) Reset() { *x = RunRequest{} - mi := &file_atelet_proto_msgTypes[25] + mi := &file_atelet_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1823,7 +2014,7 @@ func (x *RunRequest) String() string { func (*RunRequest) ProtoMessage() {} func (x *RunRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[25] + mi := &file_atelet_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1836,7 +2027,7 @@ func (x *RunRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RunRequest.ProtoReflect.Descriptor instead. func (*RunRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{25} + return file_atelet_proto_rawDescGZIP(), []int{29} } func (x *RunRequest) GetWorkerPodUid() string { @@ -1925,7 +2116,7 @@ type EgressGateway struct { func (x *EgressGateway) Reset() { *x = EgressGateway{} - mi := &file_atelet_proto_msgTypes[26] + mi := &file_atelet_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1937,7 +2128,7 @@ func (x *EgressGateway) String() string { func (*EgressGateway) ProtoMessage() {} func (x *EgressGateway) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[26] + mi := &file_atelet_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1950,7 +2141,7 @@ func (x *EgressGateway) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressGateway.ProtoReflect.Descriptor instead. func (*EgressGateway) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{26} + return file_atelet_proto_rawDescGZIP(), []int{30} } func (x *EgressGateway) GetAddress() string { @@ -1981,7 +2172,7 @@ type WorkloadSpec struct { func (x *WorkloadSpec) Reset() { *x = WorkloadSpec{} - mi := &file_atelet_proto_msgTypes[27] + mi := &file_atelet_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1993,7 +2184,7 @@ func (x *WorkloadSpec) String() string { func (*WorkloadSpec) ProtoMessage() {} func (x *WorkloadSpec) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[27] + mi := &file_atelet_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2006,7 +2197,7 @@ func (x *WorkloadSpec) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkloadSpec.ProtoReflect.Descriptor instead. func (*WorkloadSpec) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{27} + return file_atelet_proto_rawDescGZIP(), []int{31} } func (x *WorkloadSpec) GetContainers() []*Container { @@ -2031,7 +2222,7 @@ type DurableDirVolumeSource struct { func (x *DurableDirVolumeSource) Reset() { *x = DurableDirVolumeSource{} - mi := &file_atelet_proto_msgTypes[28] + mi := &file_atelet_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2043,7 +2234,7 @@ func (x *DurableDirVolumeSource) String() string { func (*DurableDirVolumeSource) ProtoMessage() {} func (x *DurableDirVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[28] + mi := &file_atelet_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2056,7 +2247,7 @@ func (x *DurableDirVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use DurableDirVolumeSource.ProtoReflect.Descriptor instead. func (*DurableDirVolumeSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{28} + return file_atelet_proto_rawDescGZIP(), []int{32} } type ExternalVolumeSource struct { @@ -2090,7 +2281,7 @@ type ExternalVolumeSource struct { func (x *ExternalVolumeSource) Reset() { *x = ExternalVolumeSource{} - mi := &file_atelet_proto_msgTypes[29] + mi := &file_atelet_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2102,7 +2293,7 @@ func (x *ExternalVolumeSource) String() string { func (*ExternalVolumeSource) ProtoMessage() {} func (x *ExternalVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[29] + mi := &file_atelet_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2115,7 +2306,7 @@ func (x *ExternalVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ExternalVolumeSource.ProtoReflect.Descriptor instead. func (*ExternalVolumeSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{29} + return file_atelet_proto_rawDescGZIP(), []int{33} } func (x *ExternalVolumeSource) GetStorageVolumeId() string { @@ -2160,7 +2351,7 @@ type ImageVolumeSource struct { func (x *ImageVolumeSource) Reset() { *x = ImageVolumeSource{} - mi := &file_atelet_proto_msgTypes[30] + mi := &file_atelet_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2172,7 +2363,7 @@ func (x *ImageVolumeSource) String() string { func (*ImageVolumeSource) ProtoMessage() {} func (x *ImageVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[30] + mi := &file_atelet_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2185,7 +2376,7 @@ func (x *ImageVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ImageVolumeSource.ProtoReflect.Descriptor instead. func (*ImageVolumeSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{30} + return file_atelet_proto_rawDescGZIP(), []int{34} } func (x *ImageVolumeSource) GetReference() string { @@ -2217,7 +2408,7 @@ type ActorMetadataItem struct { func (x *ActorMetadataItem) Reset() { *x = ActorMetadataItem{} - mi := &file_atelet_proto_msgTypes[31] + mi := &file_atelet_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2229,7 +2420,7 @@ func (x *ActorMetadataItem) String() string { func (*ActorMetadataItem) ProtoMessage() {} func (x *ActorMetadataItem) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[31] + mi := &file_atelet_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2242,7 +2433,7 @@ func (x *ActorMetadataItem) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorMetadataItem.ProtoReflect.Descriptor instead. func (*ActorMetadataItem) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{31} + return file_atelet_proto_rawDescGZIP(), []int{35} } func (x *ActorMetadataItem) GetField() ActorMetadataField { @@ -2278,7 +2469,7 @@ type ActorMetadataDataSource struct { func (x *ActorMetadataDataSource) Reset() { *x = ActorMetadataDataSource{} - mi := &file_atelet_proto_msgTypes[32] + mi := &file_atelet_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2290,7 +2481,7 @@ func (x *ActorMetadataDataSource) String() string { func (*ActorMetadataDataSource) ProtoMessage() {} func (x *ActorMetadataDataSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[32] + mi := &file_atelet_proto_msgTypes[36] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2303,7 +2494,7 @@ func (x *ActorMetadataDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorMetadataDataSource.ProtoReflect.Descriptor instead. func (*ActorMetadataDataSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{32} + return file_atelet_proto_rawDescGZIP(), []int{36} } func (x *ActorMetadataDataSource) GetItems() []*ActorMetadataItem { @@ -2337,7 +2528,7 @@ type TrustBundleDataSource struct { func (x *TrustBundleDataSource) Reset() { *x = TrustBundleDataSource{} - mi := &file_atelet_proto_msgTypes[33] + mi := &file_atelet_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2349,7 +2540,7 @@ func (x *TrustBundleDataSource) String() string { func (*TrustBundleDataSource) ProtoMessage() {} func (x *TrustBundleDataSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[33] + mi := &file_atelet_proto_msgTypes[37] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2362,7 +2553,7 @@ func (x *TrustBundleDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use TrustBundleDataSource.ProtoReflect.Descriptor instead. func (*TrustBundleDataSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{33} + return file_atelet_proto_rawDescGZIP(), []int{37} } func (x *TrustBundleDataSource) GetPath() string { @@ -2397,7 +2588,7 @@ type SystemInfoDataSource struct { func (x *SystemInfoDataSource) Reset() { *x = SystemInfoDataSource{} - mi := &file_atelet_proto_msgTypes[34] + mi := &file_atelet_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2409,7 +2600,7 @@ func (x *SystemInfoDataSource) String() string { func (*SystemInfoDataSource) ProtoMessage() {} func (x *SystemInfoDataSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[34] + mi := &file_atelet_proto_msgTypes[38] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2422,7 +2613,7 @@ func (x *SystemInfoDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemInfoDataSource.ProtoReflect.Descriptor instead. func (*SystemInfoDataSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{34} + return file_atelet_proto_rawDescGZIP(), []int{38} } func (x *SystemInfoDataSource) GetActorMetadata() *ActorMetadataDataSource { @@ -2455,7 +2646,7 @@ type SystemInfoVolumeSource struct { func (x *SystemInfoVolumeSource) Reset() { *x = SystemInfoVolumeSource{} - mi := &file_atelet_proto_msgTypes[35] + mi := &file_atelet_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2467,7 +2658,7 @@ func (x *SystemInfoVolumeSource) String() string { func (*SystemInfoVolumeSource) ProtoMessage() {} func (x *SystemInfoVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[35] + mi := &file_atelet_proto_msgTypes[39] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2480,7 +2671,7 @@ func (x *SystemInfoVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemInfoVolumeSource.ProtoReflect.Descriptor instead. func (*SystemInfoVolumeSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{35} + return file_atelet_proto_rawDescGZIP(), []int{39} } func (x *SystemInfoVolumeSource) GetDataSources() []*SystemInfoDataSource { @@ -2516,7 +2707,7 @@ type Volume struct { func (x *Volume) Reset() { *x = Volume{} - mi := &file_atelet_proto_msgTypes[36] + mi := &file_atelet_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2528,7 +2719,7 @@ func (x *Volume) String() string { func (*Volume) ProtoMessage() {} func (x *Volume) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[36] + mi := &file_atelet_proto_msgTypes[40] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2541,7 +2732,7 @@ func (x *Volume) ProtoReflect() protoreflect.Message { // Deprecated: Use Volume.ProtoReflect.Descriptor instead. func (*Volume) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{36} + return file_atelet_proto_rawDescGZIP(), []int{40} } func (x *Volume) GetName() string { @@ -2598,7 +2789,7 @@ type VolumeMount struct { func (x *VolumeMount) Reset() { *x = VolumeMount{} - mi := &file_atelet_proto_msgTypes[37] + mi := &file_atelet_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2610,7 +2801,7 @@ func (x *VolumeMount) String() string { func (*VolumeMount) ProtoMessage() {} func (x *VolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[37] + mi := &file_atelet_proto_msgTypes[41] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2623,7 +2814,7 @@ func (x *VolumeMount) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeMount.ProtoReflect.Descriptor instead. func (*VolumeMount) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{37} + return file_atelet_proto_rawDescGZIP(), []int{41} } func (x *VolumeMount) GetName() string { @@ -2688,7 +2879,7 @@ type Container struct { func (x *Container) Reset() { *x = Container{} - mi := &file_atelet_proto_msgTypes[38] + mi := &file_atelet_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2700,7 +2891,7 @@ func (x *Container) String() string { func (*Container) ProtoMessage() {} func (x *Container) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[38] + mi := &file_atelet_proto_msgTypes[42] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2713,7 +2904,7 @@ func (x *Container) ProtoReflect() protoreflect.Message { // Deprecated: Use Container.ProtoReflect.Descriptor instead. func (*Container) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{38} + return file_atelet_proto_rawDescGZIP(), []int{42} } func (x *Container) GetName() string { @@ -2790,7 +2981,7 @@ type SecurityContext struct { func (x *SecurityContext) Reset() { *x = SecurityContext{} - mi := &file_atelet_proto_msgTypes[39] + mi := &file_atelet_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2802,7 +2993,7 @@ func (x *SecurityContext) String() string { func (*SecurityContext) ProtoMessage() {} func (x *SecurityContext) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[39] + mi := &file_atelet_proto_msgTypes[43] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2815,7 +3006,7 @@ func (x *SecurityContext) ProtoReflect() protoreflect.Message { // Deprecated: Use SecurityContext.ProtoReflect.Descriptor instead. func (*SecurityContext) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{39} + return file_atelet_proto_rawDescGZIP(), []int{43} } func (x *SecurityContext) GetCapabilities() *Capabilities { @@ -2845,7 +3036,7 @@ type Capabilities struct { func (x *Capabilities) Reset() { *x = Capabilities{} - mi := &file_atelet_proto_msgTypes[40] + mi := &file_atelet_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2857,7 +3048,7 @@ func (x *Capabilities) String() string { func (*Capabilities) ProtoMessage() {} func (x *Capabilities) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[40] + mi := &file_atelet_proto_msgTypes[44] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2870,7 +3061,7 @@ func (x *Capabilities) ProtoReflect() protoreflect.Message { // Deprecated: Use Capabilities.ProtoReflect.Descriptor instead. func (*Capabilities) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{40} + return file_atelet_proto_rawDescGZIP(), []int{44} } func (x *Capabilities) GetAdd() []string { @@ -2904,7 +3095,7 @@ type EnvVar struct { func (x *EnvVar) Reset() { *x = EnvVar{} - mi := &file_atelet_proto_msgTypes[41] + mi := &file_atelet_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2916,7 +3107,7 @@ func (x *EnvVar) String() string { func (*EnvVar) ProtoMessage() {} func (x *EnvVar) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[41] + mi := &file_atelet_proto_msgTypes[45] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2929,7 +3120,7 @@ func (x *EnvVar) ProtoReflect() protoreflect.Message { // Deprecated: Use EnvVar.ProtoReflect.Descriptor instead. func (*EnvVar) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{41} + return file_atelet_proto_rawDescGZIP(), []int{45} } func (x *EnvVar) GetName() string { @@ -2964,7 +3155,7 @@ type WakeupProbe struct { func (x *WakeupProbe) Reset() { *x = WakeupProbe{} - mi := &file_atelet_proto_msgTypes[42] + mi := &file_atelet_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2976,7 +3167,7 @@ func (x *WakeupProbe) String() string { func (*WakeupProbe) ProtoMessage() {} func (x *WakeupProbe) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[42] + mi := &file_atelet_proto_msgTypes[46] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2989,7 +3180,7 @@ func (x *WakeupProbe) ProtoReflect() protoreflect.Message { // Deprecated: Use WakeupProbe.ProtoReflect.Descriptor instead. func (*WakeupProbe) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{42} + return file_atelet_proto_rawDescGZIP(), []int{46} } func (x *WakeupProbe) GetHttpGet() *HttpGetAction { @@ -3027,7 +3218,7 @@ type HttpGetAction struct { func (x *HttpGetAction) Reset() { *x = HttpGetAction{} - mi := &file_atelet_proto_msgTypes[43] + mi := &file_atelet_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3039,7 +3230,7 @@ func (x *HttpGetAction) String() string { func (*HttpGetAction) ProtoMessage() {} func (x *HttpGetAction) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[43] + mi := &file_atelet_proto_msgTypes[47] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3052,7 +3243,7 @@ func (x *HttpGetAction) ProtoReflect() protoreflect.Message { // Deprecated: Use HttpGetAction.ProtoReflect.Descriptor instead. func (*HttpGetAction) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{43} + return file_atelet_proto_rawDescGZIP(), []int{47} } func (x *HttpGetAction) GetPath() string { @@ -3070,14 +3261,17 @@ func (x *HttpGetAction) GetPort() int32 { } type RunResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` + state protoimpl.MessageState `protogen:"open.v1"` + // The epoch of the worker process that started the workload, as it + // registered with the control plane. + WorkerEpoch int64 `protobuf:"varint,1,opt,name=worker_epoch,json=workerEpoch,proto3" json:"worker_epoch,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *RunResponse) Reset() { *x = RunResponse{} - mi := &file_atelet_proto_msgTypes[44] + mi := &file_atelet_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3089,7 +3283,7 @@ func (x *RunResponse) String() string { func (*RunResponse) ProtoMessage() {} func (x *RunResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[44] + mi := &file_atelet_proto_msgTypes[48] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3102,7 +3296,14 @@ func (x *RunResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RunResponse.ProtoReflect.Descriptor instead. func (*RunResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{44} + return file_atelet_proto_rawDescGZIP(), []int{48} +} + +func (x *RunResponse) GetWorkerEpoch() int64 { + if x != nil { + return x.WorkerEpoch + } + return 0 } type ObjectStorage struct { @@ -3128,7 +3329,7 @@ type ObjectStorage struct { func (x *ObjectStorage) Reset() { *x = ObjectStorage{} - mi := &file_atelet_proto_msgTypes[45] + mi := &file_atelet_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3140,7 +3341,7 @@ func (x *ObjectStorage) String() string { func (*ObjectStorage) ProtoMessage() {} func (x *ObjectStorage) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[45] + mi := &file_atelet_proto_msgTypes[49] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3153,7 +3354,7 @@ func (x *ObjectStorage) ProtoReflect() protoreflect.Message { // Deprecated: Use ObjectStorage.ProtoReflect.Descriptor instead. func (*ObjectStorage) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{45} + return file_atelet_proto_rawDescGZIP(), []int{49} } func (x *ObjectStorage) GetSnapshotUri() string { @@ -3214,7 +3415,7 @@ type Snapshot struct { func (x *Snapshot) Reset() { *x = Snapshot{} - mi := &file_atelet_proto_msgTypes[46] + mi := &file_atelet_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3226,7 +3427,7 @@ func (x *Snapshot) String() string { func (*Snapshot) ProtoMessage() {} func (x *Snapshot) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[46] + mi := &file_atelet_proto_msgTypes[50] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3239,7 +3440,7 @@ func (x *Snapshot) ProtoReflect() protoreflect.Message { // Deprecated: Use Snapshot.ProtoReflect.Descriptor instead. func (*Snapshot) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{46} + return file_atelet_proto_rawDescGZIP(), []int{50} } func (x *Snapshot) GetSnapshotUuid() string { @@ -3286,7 +3487,7 @@ type SnapshotReport struct { func (x *SnapshotReport) Reset() { *x = SnapshotReport{} - mi := &file_atelet_proto_msgTypes[47] + mi := &file_atelet_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3298,7 +3499,7 @@ func (x *SnapshotReport) String() string { func (*SnapshotReport) ProtoMessage() {} func (x *SnapshotReport) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[47] + mi := &file_atelet_proto_msgTypes[51] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3311,7 +3512,7 @@ func (x *SnapshotReport) ProtoReflect() protoreflect.Message { // Deprecated: Use SnapshotReport.ProtoReflect.Descriptor instead. func (*SnapshotReport) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{47} + return file_atelet_proto_rawDescGZIP(), []int{51} } func (x *SnapshotReport) GetSnapshot() *Snapshot { @@ -3368,7 +3569,7 @@ type ActorSnapshotState struct { func (x *ActorSnapshotState) Reset() { *x = ActorSnapshotState{} - mi := &file_atelet_proto_msgTypes[48] + mi := &file_atelet_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3380,7 +3581,7 @@ func (x *ActorSnapshotState) String() string { func (*ActorSnapshotState) ProtoMessage() {} func (x *ActorSnapshotState) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[48] + mi := &file_atelet_proto_msgTypes[52] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3393,7 +3594,7 @@ func (x *ActorSnapshotState) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorSnapshotState.ProtoReflect.Descriptor instead. func (*ActorSnapshotState) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{48} + return file_atelet_proto_rawDescGZIP(), []int{52} } func (x *ActorSnapshotState) GetLatestLocal() *SnapshotReport { @@ -3437,7 +3638,7 @@ type UploadConfig struct { func (x *UploadConfig) Reset() { *x = UploadConfig{} - mi := &file_atelet_proto_msgTypes[49] + mi := &file_atelet_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3449,7 +3650,7 @@ func (x *UploadConfig) String() string { func (*UploadConfig) ProtoMessage() {} func (x *UploadConfig) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[49] + mi := &file_atelet_proto_msgTypes[53] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3462,7 +3663,7 @@ func (x *UploadConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use UploadConfig.ProtoReflect.Descriptor instead. func (*UploadConfig) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{49} + return file_atelet_proto_rawDescGZIP(), []int{53} } func (x *UploadConfig) GetMode() UploadMode { @@ -3524,7 +3725,7 @@ type CheckpointRequest struct { func (x *CheckpointRequest) Reset() { *x = CheckpointRequest{} - mi := &file_atelet_proto_msgTypes[50] + mi := &file_atelet_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3536,7 +3737,7 @@ func (x *CheckpointRequest) String() string { func (*CheckpointRequest) ProtoMessage() {} func (x *CheckpointRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[50] + mi := &file_atelet_proto_msgTypes[54] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3549,7 +3750,7 @@ func (x *CheckpointRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointRequest.ProtoReflect.Descriptor instead. func (*CheckpointRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{50} + return file_atelet_proto_rawDescGZIP(), []int{54} } func (x *CheckpointRequest) GetWorkerPodUid() string { @@ -3633,7 +3834,7 @@ type CheckpointResponse struct { func (x *CheckpointResponse) Reset() { *x = CheckpointResponse{} - mi := &file_atelet_proto_msgTypes[51] + mi := &file_atelet_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3645,7 +3846,7 @@ func (x *CheckpointResponse) String() string { func (*CheckpointResponse) ProtoMessage() {} func (x *CheckpointResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[51] + mi := &file_atelet_proto_msgTypes[55] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3658,7 +3859,7 @@ func (x *CheckpointResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointResponse.ProtoReflect.Descriptor instead. func (*CheckpointResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{51} + return file_atelet_proto_rawDescGZIP(), []int{55} } func (x *CheckpointResponse) GetState() *ActorSnapshotState { @@ -3732,7 +3933,7 @@ type RestoreRequest struct { func (x *RestoreRequest) Reset() { *x = RestoreRequest{} - mi := &file_atelet_proto_msgTypes[52] + mi := &file_atelet_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3744,7 +3945,7 @@ func (x *RestoreRequest) String() string { func (*RestoreRequest) ProtoMessage() {} func (x *RestoreRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[52] + mi := &file_atelet_proto_msgTypes[56] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3757,7 +3958,7 @@ func (x *RestoreRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreRequest.ProtoReflect.Descriptor instead. func (*RestoreRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{52} + return file_atelet_proto_rawDescGZIP(), []int{56} } func (x *RestoreRequest) GetWorkerPodUid() string { @@ -3838,14 +4039,17 @@ func (x *RestoreRequest) GetSandboxRuntime() *ActorSandboxRuntime { } type RestoreResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` + state protoimpl.MessageState `protogen:"open.v1"` + // The epoch of the worker process that restored the workload, as it + // registered with the control plane. + WorkerEpoch int64 `protobuf:"varint,1,opt,name=worker_epoch,json=workerEpoch,proto3" json:"worker_epoch,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *RestoreResponse) Reset() { *x = RestoreResponse{} - mi := &file_atelet_proto_msgTypes[53] + mi := &file_atelet_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3857,7 +4061,7 @@ func (x *RestoreResponse) String() string { func (*RestoreResponse) ProtoMessage() {} func (x *RestoreResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[53] + mi := &file_atelet_proto_msgTypes[57] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3870,7 +4074,14 @@ func (x *RestoreResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreResponse.ProtoReflect.Descriptor instead. func (*RestoreResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{53} + return file_atelet_proto_rawDescGZIP(), []int{57} +} + +func (x *RestoreResponse) GetWorkerEpoch() int64 { + if x != nil { + return x.WorkerEpoch + } + return 0 } type PromoteSnapshotRequest struct { @@ -3903,7 +4114,7 @@ type PromoteSnapshotRequest struct { func (x *PromoteSnapshotRequest) Reset() { *x = PromoteSnapshotRequest{} - mi := &file_atelet_proto_msgTypes[54] + mi := &file_atelet_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3915,7 +4126,7 @@ func (x *PromoteSnapshotRequest) String() string { func (*PromoteSnapshotRequest) ProtoMessage() {} func (x *PromoteSnapshotRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[54] + mi := &file_atelet_proto_msgTypes[58] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3928,7 +4139,7 @@ func (x *PromoteSnapshotRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PromoteSnapshotRequest.ProtoReflect.Descriptor instead. func (*PromoteSnapshotRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{54} + return file_atelet_proto_rawDescGZIP(), []int{58} } func (x *PromoteSnapshotRequest) GetAtespace() string { @@ -3984,7 +4195,7 @@ type PromoteSnapshotResponse struct { func (x *PromoteSnapshotResponse) Reset() { *x = PromoteSnapshotResponse{} - mi := &file_atelet_proto_msgTypes[55] + mi := &file_atelet_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3996,7 +4207,7 @@ func (x *PromoteSnapshotResponse) String() string { func (*PromoteSnapshotResponse) ProtoMessage() {} func (x *PromoteSnapshotResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[55] + mi := &file_atelet_proto_msgTypes[59] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4009,7 +4220,7 @@ func (x *PromoteSnapshotResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PromoteSnapshotResponse.ProtoReflect.Descriptor instead. func (*PromoteSnapshotResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{55} + return file_atelet_proto_rawDescGZIP(), []int{59} } func (x *PromoteSnapshotResponse) GetState() *ActorSnapshotState { @@ -4038,7 +4249,7 @@ type DetachActorRequest struct { func (x *DetachActorRequest) Reset() { *x = DetachActorRequest{} - mi := &file_atelet_proto_msgTypes[56] + mi := &file_atelet_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4050,7 +4261,7 @@ func (x *DetachActorRequest) String() string { func (*DetachActorRequest) ProtoMessage() {} func (x *DetachActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[56] + mi := &file_atelet_proto_msgTypes[60] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4063,7 +4274,7 @@ func (x *DetachActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DetachActorRequest.ProtoReflect.Descriptor instead. func (*DetachActorRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{56} + return file_atelet_proto_rawDescGZIP(), []int{60} } func (x *DetachActorRequest) GetAtespace() string { @@ -4099,7 +4310,7 @@ type DetachActorResponse struct { func (x *DetachActorResponse) Reset() { *x = DetachActorResponse{} - mi := &file_atelet_proto_msgTypes[57] + mi := &file_atelet_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4111,7 +4322,7 @@ func (x *DetachActorResponse) String() string { func (*DetachActorResponse) ProtoMessage() {} func (x *DetachActorResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[57] + mi := &file_atelet_proto_msgTypes[61] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4124,7 +4335,7 @@ func (x *DetachActorResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DetachActorResponse.ProtoReflect.Descriptor instead. func (*DetachActorResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{57} + return file_atelet_proto_rawDescGZIP(), []int{61} } func (x *DetachActorResponse) GetState() *ActorSnapshotState { @@ -4138,10 +4349,18 @@ var File_atelet_proto protoreflect.FileDescriptor const file_atelet_proto_rawDesc = "" + "\n" + - "\fatelet.proto\x12\x06atelet\x1a\x1fgoogle/protobuf/timestamp.proto\"\x91\x01\n" + - "\x15RegisterWorkerRequest\x123\n" + - "\bcapacity\x18\x01 \x01(\v2\x17.atelet.WorkerResourcesR\bcapacity\x12C\n" + - "\x0fsandbox_classes\x18\x02 \x03(\v2\x1a.atelet.WorkerSandboxClassR\x0esandboxClasses\"\xcf\x01\n" + + "\fatelet.proto\x12\x06atelet\x1a\x1fgoogle/protobuf/timestamp.proto\"\x1b\n" + + "\x19AcquireWorkerEpochRequest\"2\n" + + "\x1aAcquireWorkerEpochResponse\x12\x14\n" + + "\x05epoch\x18\x01 \x01(\x03R\x05epoch\"t\n" + + "\x15RegisterWorkerRequest\x12\x14\n" + + "\x05epoch\x18\x02 \x01(\x03R\x05epoch\x12\x10\n" + + "\x03ips\x18\x03 \x03(\tR\x03ips\x123\n" + + "\bcapacity\x18\x01 \x01(\v2\x17.atelet.WorkerResourcesR\bcapacity\"|\n" + + "\x1fSetWorkerSandboxRuntimesRequest\x12\x14\n" + + "\x05epoch\x18\x01 \x01(\x03R\x05epoch\x12C\n" + + "\x0fsandbox_classes\x18\x02 \x03(\v2\x1a.atelet.WorkerSandboxClassR\x0esandboxClasses\"\"\n" + + " SetWorkerSandboxRuntimesResponse\"\xcf\x01\n" + "\x12WorkerSandboxClass\x12#\n" + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\x12E\n" + "\x0fdefault_runtime\x18\x02 \x01(\v2\x1c.atelet.WorkerSandboxRuntimeR\x0edefaultRuntime\x12M\n" + @@ -4305,8 +4524,9 @@ const file_atelet_proto_rawDesc = "" + "\x0ftimeout_seconds\x18\x02 \x01(\x05R\x0etimeoutSeconds\"7\n" + "\rHttpGetAction\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x12\n" + - "\x04port\x18\x02 \x01(\x05R\x04port\"\r\n" + - "\vRunResponse\"e\n" + + "\x04port\x18\x02 \x01(\x05R\x04port\"0\n" + + "\vRunResponse\x12!\n" + + "\fworker_epoch\x18\x01 \x01(\x03R\vworkerEpoch\"e\n" + "\rObjectStorage\x12!\n" + "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\x121\n" + "\asharing\x18\x02 \x01(\x0e2\x17.atelet.SnapshotSharingR\asharing\"\xb4\x01\n" + @@ -4355,8 +4575,9 @@ const file_atelet_proto_rawDesc = "" + "\tresources\x18\n" + " \x01(\v2\x1c.atelet.ResourceRequirementsR\tresources\x12D\n" + "\x0fsandbox_runtime\x18\f \x01(\v2\x1b.atelet.ActorSandboxRuntimeR\x0esandboxRuntimeB\x11\n" + - "\x0f_egress_gateway\"\x11\n" + - "\x0fRestoreResponse\"\x86\x02\n" + + "\x0f_egress_gateway\"4\n" + + "\x0fRestoreResponse\x12!\n" + + "\fworker_epoch\x18\x01 \x01(\x03R\vworkerEpoch\"\x86\x02\n" + "\x16PromoteSnapshotRequest\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1d\n" + "\n" + @@ -4392,10 +4613,12 @@ const file_atelet_proto_rawDesc = "" + "UploadMode\x12\x1b\n" + "\x17UPLOAD_MODE_UNSPECIFIED\x10\x00\x12\x18\n" + "\x14UPLOAD_MODE_DEFERRED\x10\x01\x12\x19\n" + - "\x15UPLOAD_MODE_IMMEDIATE\x10\x022\xed\x03\n" + + "\x15UPLOAD_MODE_IMMEDIATE\x10\x022\xbd\x05\n" + "\rWorkerSupport\x12c\n" + - "\x14MintActorCertificate\x12#.atelet.MintActorCertificateRequest\x1a$.atelet.MintActorCertificateResponse\"\x00\x12Q\n" + - "\x0eRegisterWorker\x12\x1d.atelet.RegisterWorkerRequest\x1a\x1e.atelet.RegisterWorkerResponse\"\x00\x12`\n" + + "\x14MintActorCertificate\x12#.atelet.MintActorCertificateRequest\x1a$.atelet.MintActorCertificateResponse\"\x00\x12]\n" + + "\x12AcquireWorkerEpoch\x12!.atelet.AcquireWorkerEpochRequest\x1a\".atelet.AcquireWorkerEpochResponse\"\x00\x12Q\n" + + "\x0eRegisterWorker\x12\x1d.atelet.RegisterWorkerRequest\x1a\x1e.atelet.RegisterWorkerResponse\"\x00\x12o\n" + + "\x18SetWorkerSandboxRuntimes\x12'.atelet.SetWorkerSandboxRuntimesRequest\x1a(.atelet.SetWorkerSandboxRuntimesResponse\"\x00\x12`\n" + "\x13RequestActorSuspend\x12\".atelet.RequestActorSuspendRequest\x1a#.atelet.RequestActorSuspendResponse\"\x00\x12c\n" + "\x14GetActorEgressPolicy\x12#.atelet.GetActorEgressPolicyRequest\x1a$.atelet.GetActorEgressPolicyResponse\"\x00\x12]\n" + "\x12ReportActorCrashed\x12!.atelet.ReportActorCrashedRequest\x1a\".atelet.ReportActorCrashedResponse\"\x002\xa3\x03\n" + @@ -4421,162 +4644,170 @@ func file_atelet_proto_rawDescGZIP() []byte { } var file_atelet_proto_enumTypes = make([]protoimpl.EnumInfo, 4) -var file_atelet_proto_msgTypes = make([]protoimpl.MessageInfo, 60) +var file_atelet_proto_msgTypes = make([]protoimpl.MessageInfo, 64) var file_atelet_proto_goTypes = []any{ - (ActorMetadataField)(0), // 0: atelet.ActorMetadataField - (SnapshotFidelity)(0), // 1: atelet.SnapshotFidelity - (SnapshotSharing)(0), // 2: atelet.SnapshotSharing - (UploadMode)(0), // 3: atelet.UploadMode - (*RegisterWorkerRequest)(nil), // 4: atelet.RegisterWorkerRequest - (*WorkerSandboxClass)(nil), // 5: atelet.WorkerSandboxClass - (*WorkerSandboxRuntime)(nil), // 6: atelet.WorkerSandboxRuntime - (*ActorSandboxRuntime)(nil), // 7: atelet.ActorSandboxRuntime - (*VersionedSandboxCompat)(nil), // 8: atelet.VersionedSandboxCompat - (*AttributeEntry)(nil), // 9: atelet.AttributeEntry - (*WorkerResources)(nil), // 10: atelet.WorkerResources - (*ResourceRequirements)(nil), // 11: atelet.ResourceRequirements - (*Resources)(nil), // 12: atelet.Resources - (*RegisterWorkerResponse)(nil), // 13: atelet.RegisterWorkerResponse - (*RequestActorSuspendRequest)(nil), // 14: atelet.RequestActorSuspendRequest - (*RequestActorSuspendResponse)(nil), // 15: atelet.RequestActorSuspendResponse - (*ReportActorCrashedRequest)(nil), // 16: atelet.ReportActorCrashedRequest - (*ReportActorCrashedResponse)(nil), // 17: atelet.ReportActorCrashedResponse - (*GetActorEgressPolicyRequest)(nil), // 18: atelet.GetActorEgressPolicyRequest - (*GetActorEgressPolicyResponse)(nil), // 19: atelet.GetActorEgressPolicyResponse - (*EgressPolicy)(nil), // 20: atelet.EgressPolicy - (*EgressRule)(nil), // 21: atelet.EgressRule - (*EgressDestinations)(nil), // 22: atelet.EgressDestinations - (*EgressPorts)(nil), // 23: atelet.EgressPorts - (*EgressAllPorts)(nil), // 24: atelet.EgressAllPorts - (*MintActorCertificateRequest)(nil), // 25: atelet.MintActorCertificateRequest - (*MintActorCertificateResponse)(nil), // 26: atelet.MintActorCertificateResponse - (*TerminateRequest)(nil), // 27: atelet.TerminateRequest - (*TerminateResponse)(nil), // 28: atelet.TerminateResponse - (*RunRequest)(nil), // 29: atelet.RunRequest - (*EgressGateway)(nil), // 30: atelet.EgressGateway - (*WorkloadSpec)(nil), // 31: atelet.WorkloadSpec - (*DurableDirVolumeSource)(nil), // 32: atelet.DurableDirVolumeSource - (*ExternalVolumeSource)(nil), // 33: atelet.ExternalVolumeSource - (*ImageVolumeSource)(nil), // 34: atelet.ImageVolumeSource - (*ActorMetadataItem)(nil), // 35: atelet.ActorMetadataItem - (*ActorMetadataDataSource)(nil), // 36: atelet.ActorMetadataDataSource - (*TrustBundleDataSource)(nil), // 37: atelet.TrustBundleDataSource - (*SystemInfoDataSource)(nil), // 38: atelet.SystemInfoDataSource - (*SystemInfoVolumeSource)(nil), // 39: atelet.SystemInfoVolumeSource - (*Volume)(nil), // 40: atelet.Volume - (*VolumeMount)(nil), // 41: atelet.VolumeMount - (*Container)(nil), // 42: atelet.Container - (*SecurityContext)(nil), // 43: atelet.SecurityContext - (*Capabilities)(nil), // 44: atelet.Capabilities - (*EnvVar)(nil), // 45: atelet.EnvVar - (*WakeupProbe)(nil), // 46: atelet.WakeupProbe - (*HttpGetAction)(nil), // 47: atelet.HttpGetAction - (*RunResponse)(nil), // 48: atelet.RunResponse - (*ObjectStorage)(nil), // 49: atelet.ObjectStorage - (*Snapshot)(nil), // 50: atelet.Snapshot - (*SnapshotReport)(nil), // 51: atelet.SnapshotReport - (*ActorSnapshotState)(nil), // 52: atelet.ActorSnapshotState - (*UploadConfig)(nil), // 53: atelet.UploadConfig - (*CheckpointRequest)(nil), // 54: atelet.CheckpointRequest - (*CheckpointResponse)(nil), // 55: atelet.CheckpointResponse - (*RestoreRequest)(nil), // 56: atelet.RestoreRequest - (*RestoreResponse)(nil), // 57: atelet.RestoreResponse - (*PromoteSnapshotRequest)(nil), // 58: atelet.PromoteSnapshotRequest - (*PromoteSnapshotResponse)(nil), // 59: atelet.PromoteSnapshotResponse - (*DetachActorRequest)(nil), // 60: atelet.DetachActorRequest - (*DetachActorResponse)(nil), // 61: atelet.DetachActorResponse - nil, // 62: atelet.ExternalVolumeSource.VolumeContextEntry - nil, // 63: atelet.ExternalVolumeSource.PublishContextEntry - (*timestamppb.Timestamp)(nil), // 64: google.protobuf.Timestamp + (ActorMetadataField)(0), // 0: atelet.ActorMetadataField + (SnapshotFidelity)(0), // 1: atelet.SnapshotFidelity + (SnapshotSharing)(0), // 2: atelet.SnapshotSharing + (UploadMode)(0), // 3: atelet.UploadMode + (*AcquireWorkerEpochRequest)(nil), // 4: atelet.AcquireWorkerEpochRequest + (*AcquireWorkerEpochResponse)(nil), // 5: atelet.AcquireWorkerEpochResponse + (*RegisterWorkerRequest)(nil), // 6: atelet.RegisterWorkerRequest + (*SetWorkerSandboxRuntimesRequest)(nil), // 7: atelet.SetWorkerSandboxRuntimesRequest + (*SetWorkerSandboxRuntimesResponse)(nil), // 8: atelet.SetWorkerSandboxRuntimesResponse + (*WorkerSandboxClass)(nil), // 9: atelet.WorkerSandboxClass + (*WorkerSandboxRuntime)(nil), // 10: atelet.WorkerSandboxRuntime + (*ActorSandboxRuntime)(nil), // 11: atelet.ActorSandboxRuntime + (*VersionedSandboxCompat)(nil), // 12: atelet.VersionedSandboxCompat + (*AttributeEntry)(nil), // 13: atelet.AttributeEntry + (*WorkerResources)(nil), // 14: atelet.WorkerResources + (*ResourceRequirements)(nil), // 15: atelet.ResourceRequirements + (*Resources)(nil), // 16: atelet.Resources + (*RegisterWorkerResponse)(nil), // 17: atelet.RegisterWorkerResponse + (*RequestActorSuspendRequest)(nil), // 18: atelet.RequestActorSuspendRequest + (*RequestActorSuspendResponse)(nil), // 19: atelet.RequestActorSuspendResponse + (*ReportActorCrashedRequest)(nil), // 20: atelet.ReportActorCrashedRequest + (*ReportActorCrashedResponse)(nil), // 21: atelet.ReportActorCrashedResponse + (*GetActorEgressPolicyRequest)(nil), // 22: atelet.GetActorEgressPolicyRequest + (*GetActorEgressPolicyResponse)(nil), // 23: atelet.GetActorEgressPolicyResponse + (*EgressPolicy)(nil), // 24: atelet.EgressPolicy + (*EgressRule)(nil), // 25: atelet.EgressRule + (*EgressDestinations)(nil), // 26: atelet.EgressDestinations + (*EgressPorts)(nil), // 27: atelet.EgressPorts + (*EgressAllPorts)(nil), // 28: atelet.EgressAllPorts + (*MintActorCertificateRequest)(nil), // 29: atelet.MintActorCertificateRequest + (*MintActorCertificateResponse)(nil), // 30: atelet.MintActorCertificateResponse + (*TerminateRequest)(nil), // 31: atelet.TerminateRequest + (*TerminateResponse)(nil), // 32: atelet.TerminateResponse + (*RunRequest)(nil), // 33: atelet.RunRequest + (*EgressGateway)(nil), // 34: atelet.EgressGateway + (*WorkloadSpec)(nil), // 35: atelet.WorkloadSpec + (*DurableDirVolumeSource)(nil), // 36: atelet.DurableDirVolumeSource + (*ExternalVolumeSource)(nil), // 37: atelet.ExternalVolumeSource + (*ImageVolumeSource)(nil), // 38: atelet.ImageVolumeSource + (*ActorMetadataItem)(nil), // 39: atelet.ActorMetadataItem + (*ActorMetadataDataSource)(nil), // 40: atelet.ActorMetadataDataSource + (*TrustBundleDataSource)(nil), // 41: atelet.TrustBundleDataSource + (*SystemInfoDataSource)(nil), // 42: atelet.SystemInfoDataSource + (*SystemInfoVolumeSource)(nil), // 43: atelet.SystemInfoVolumeSource + (*Volume)(nil), // 44: atelet.Volume + (*VolumeMount)(nil), // 45: atelet.VolumeMount + (*Container)(nil), // 46: atelet.Container + (*SecurityContext)(nil), // 47: atelet.SecurityContext + (*Capabilities)(nil), // 48: atelet.Capabilities + (*EnvVar)(nil), // 49: atelet.EnvVar + (*WakeupProbe)(nil), // 50: atelet.WakeupProbe + (*HttpGetAction)(nil), // 51: atelet.HttpGetAction + (*RunResponse)(nil), // 52: atelet.RunResponse + (*ObjectStorage)(nil), // 53: atelet.ObjectStorage + (*Snapshot)(nil), // 54: atelet.Snapshot + (*SnapshotReport)(nil), // 55: atelet.SnapshotReport + (*ActorSnapshotState)(nil), // 56: atelet.ActorSnapshotState + (*UploadConfig)(nil), // 57: atelet.UploadConfig + (*CheckpointRequest)(nil), // 58: atelet.CheckpointRequest + (*CheckpointResponse)(nil), // 59: atelet.CheckpointResponse + (*RestoreRequest)(nil), // 60: atelet.RestoreRequest + (*RestoreResponse)(nil), // 61: atelet.RestoreResponse + (*PromoteSnapshotRequest)(nil), // 62: atelet.PromoteSnapshotRequest + (*PromoteSnapshotResponse)(nil), // 63: atelet.PromoteSnapshotResponse + (*DetachActorRequest)(nil), // 64: atelet.DetachActorRequest + (*DetachActorResponse)(nil), // 65: atelet.DetachActorResponse + nil, // 66: atelet.ExternalVolumeSource.VolumeContextEntry + nil, // 67: atelet.ExternalVolumeSource.PublishContextEntry + (*timestamppb.Timestamp)(nil), // 68: google.protobuf.Timestamp } var file_atelet_proto_depIdxs = []int32{ - 10, // 0: atelet.RegisterWorkerRequest.capacity:type_name -> atelet.WorkerResources - 5, // 1: atelet.RegisterWorkerRequest.sandbox_classes:type_name -> atelet.WorkerSandboxClass - 6, // 2: atelet.WorkerSandboxClass.default_runtime:type_name -> atelet.WorkerSandboxRuntime - 6, // 3: atelet.WorkerSandboxClass.restorable_runtimes:type_name -> atelet.WorkerSandboxRuntime - 8, // 4: atelet.WorkerSandboxRuntime.version:type_name -> atelet.VersionedSandboxCompat - 8, // 5: atelet.ActorSandboxRuntime.version:type_name -> atelet.VersionedSandboxCompat - 9, // 6: atelet.VersionedSandboxCompat.attributes:type_name -> atelet.AttributeEntry - 12, // 7: atelet.ResourceRequirements.limits:type_name -> atelet.Resources - 20, // 8: atelet.GetActorEgressPolicyResponse.egress_policy:type_name -> atelet.EgressPolicy - 21, // 9: atelet.EgressPolicy.rules:type_name -> atelet.EgressRule - 22, // 10: atelet.EgressRule.http:type_name -> atelet.EgressDestinations - 22, // 11: atelet.EgressRule.https:type_name -> atelet.EgressDestinations - 22, // 12: atelet.EgressRule.tls_passthrough:type_name -> atelet.EgressDestinations - 23, // 13: atelet.EgressDestinations.ports:type_name -> atelet.EgressPorts - 24, // 14: atelet.EgressPorts.all:type_name -> atelet.EgressAllPorts - 31, // 15: atelet.TerminateRequest.spec:type_name -> atelet.WorkloadSpec - 31, // 16: atelet.RunRequest.spec:type_name -> atelet.WorkloadSpec - 30, // 17: atelet.RunRequest.egress_gateway:type_name -> atelet.EgressGateway - 11, // 18: atelet.RunRequest.resources:type_name -> atelet.ResourceRequirements - 7, // 19: atelet.RunRequest.sandbox_runtime:type_name -> atelet.ActorSandboxRuntime - 42, // 20: atelet.WorkloadSpec.containers:type_name -> atelet.Container - 40, // 21: atelet.WorkloadSpec.volumes:type_name -> atelet.Volume - 62, // 22: atelet.ExternalVolumeSource.volume_context:type_name -> atelet.ExternalVolumeSource.VolumeContextEntry - 63, // 23: atelet.ExternalVolumeSource.publish_context:type_name -> atelet.ExternalVolumeSource.PublishContextEntry + 14, // 0: atelet.RegisterWorkerRequest.capacity:type_name -> atelet.WorkerResources + 9, // 1: atelet.SetWorkerSandboxRuntimesRequest.sandbox_classes:type_name -> atelet.WorkerSandboxClass + 10, // 2: atelet.WorkerSandboxClass.default_runtime:type_name -> atelet.WorkerSandboxRuntime + 10, // 3: atelet.WorkerSandboxClass.restorable_runtimes:type_name -> atelet.WorkerSandboxRuntime + 12, // 4: atelet.WorkerSandboxRuntime.version:type_name -> atelet.VersionedSandboxCompat + 12, // 5: atelet.ActorSandboxRuntime.version:type_name -> atelet.VersionedSandboxCompat + 13, // 6: atelet.VersionedSandboxCompat.attributes:type_name -> atelet.AttributeEntry + 16, // 7: atelet.ResourceRequirements.limits:type_name -> atelet.Resources + 24, // 8: atelet.GetActorEgressPolicyResponse.egress_policy:type_name -> atelet.EgressPolicy + 25, // 9: atelet.EgressPolicy.rules:type_name -> atelet.EgressRule + 26, // 10: atelet.EgressRule.http:type_name -> atelet.EgressDestinations + 26, // 11: atelet.EgressRule.https:type_name -> atelet.EgressDestinations + 26, // 12: atelet.EgressRule.tls_passthrough:type_name -> atelet.EgressDestinations + 27, // 13: atelet.EgressDestinations.ports:type_name -> atelet.EgressPorts + 28, // 14: atelet.EgressPorts.all:type_name -> atelet.EgressAllPorts + 35, // 15: atelet.TerminateRequest.spec:type_name -> atelet.WorkloadSpec + 35, // 16: atelet.RunRequest.spec:type_name -> atelet.WorkloadSpec + 34, // 17: atelet.RunRequest.egress_gateway:type_name -> atelet.EgressGateway + 15, // 18: atelet.RunRequest.resources:type_name -> atelet.ResourceRequirements + 11, // 19: atelet.RunRequest.sandbox_runtime:type_name -> atelet.ActorSandboxRuntime + 46, // 20: atelet.WorkloadSpec.containers:type_name -> atelet.Container + 44, // 21: atelet.WorkloadSpec.volumes:type_name -> atelet.Volume + 66, // 22: atelet.ExternalVolumeSource.volume_context:type_name -> atelet.ExternalVolumeSource.VolumeContextEntry + 67, // 23: atelet.ExternalVolumeSource.publish_context:type_name -> atelet.ExternalVolumeSource.PublishContextEntry 0, // 24: atelet.ActorMetadataItem.field:type_name -> atelet.ActorMetadataField - 35, // 25: atelet.ActorMetadataDataSource.items:type_name -> atelet.ActorMetadataItem - 36, // 26: atelet.SystemInfoDataSource.actor_metadata:type_name -> atelet.ActorMetadataDataSource - 37, // 27: atelet.SystemInfoDataSource.trust_bundle:type_name -> atelet.TrustBundleDataSource - 38, // 28: atelet.SystemInfoVolumeSource.data_sources:type_name -> atelet.SystemInfoDataSource - 32, // 29: atelet.Volume.durable_dir:type_name -> atelet.DurableDirVolumeSource - 33, // 30: atelet.Volume.external:type_name -> atelet.ExternalVolumeSource - 39, // 31: atelet.Volume.system_info:type_name -> atelet.SystemInfoVolumeSource - 34, // 32: atelet.Volume.image:type_name -> atelet.ImageVolumeSource - 45, // 33: atelet.Container.env:type_name -> atelet.EnvVar - 46, // 34: atelet.Container.wakeup_probe:type_name -> atelet.WakeupProbe - 41, // 35: atelet.Container.volume_mounts:type_name -> atelet.VolumeMount - 43, // 36: atelet.Container.security_context:type_name -> atelet.SecurityContext - 11, // 37: atelet.Container.resources:type_name -> atelet.ResourceRequirements - 44, // 38: atelet.SecurityContext.capabilities:type_name -> atelet.Capabilities - 47, // 39: atelet.WakeupProbe.http_get:type_name -> atelet.HttpGetAction + 39, // 25: atelet.ActorMetadataDataSource.items:type_name -> atelet.ActorMetadataItem + 40, // 26: atelet.SystemInfoDataSource.actor_metadata:type_name -> atelet.ActorMetadataDataSource + 41, // 27: atelet.SystemInfoDataSource.trust_bundle:type_name -> atelet.TrustBundleDataSource + 42, // 28: atelet.SystemInfoVolumeSource.data_sources:type_name -> atelet.SystemInfoDataSource + 36, // 29: atelet.Volume.durable_dir:type_name -> atelet.DurableDirVolumeSource + 37, // 30: atelet.Volume.external:type_name -> atelet.ExternalVolumeSource + 43, // 31: atelet.Volume.system_info:type_name -> atelet.SystemInfoVolumeSource + 38, // 32: atelet.Volume.image:type_name -> atelet.ImageVolumeSource + 49, // 33: atelet.Container.env:type_name -> atelet.EnvVar + 50, // 34: atelet.Container.wakeup_probe:type_name -> atelet.WakeupProbe + 45, // 35: atelet.Container.volume_mounts:type_name -> atelet.VolumeMount + 47, // 36: atelet.Container.security_context:type_name -> atelet.SecurityContext + 15, // 37: atelet.Container.resources:type_name -> atelet.ResourceRequirements + 48, // 38: atelet.SecurityContext.capabilities:type_name -> atelet.Capabilities + 51, // 39: atelet.WakeupProbe.http_get:type_name -> atelet.HttpGetAction 2, // 40: atelet.ObjectStorage.sharing:type_name -> atelet.SnapshotSharing 1, // 41: atelet.Snapshot.fidelity:type_name -> atelet.SnapshotFidelity - 49, // 42: atelet.Snapshot.object:type_name -> atelet.ObjectStorage - 50, // 43: atelet.SnapshotReport.snapshot:type_name -> atelet.Snapshot - 64, // 44: atelet.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp - 64, // 45: atelet.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp - 51, // 46: atelet.ActorSnapshotState.latest_local:type_name -> atelet.SnapshotReport - 51, // 47: atelet.ActorSnapshotState.uploading:type_name -> atelet.SnapshotReport - 51, // 48: atelet.ActorSnapshotState.latest_durable:type_name -> atelet.SnapshotReport - 64, // 49: atelet.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp + 53, // 42: atelet.Snapshot.object:type_name -> atelet.ObjectStorage + 54, // 43: atelet.SnapshotReport.snapshot:type_name -> atelet.Snapshot + 68, // 44: atelet.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp + 68, // 45: atelet.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp + 55, // 46: atelet.ActorSnapshotState.latest_local:type_name -> atelet.SnapshotReport + 55, // 47: atelet.ActorSnapshotState.uploading:type_name -> atelet.SnapshotReport + 55, // 48: atelet.ActorSnapshotState.latest_durable:type_name -> atelet.SnapshotReport + 68, // 49: atelet.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp 3, // 50: atelet.UploadConfig.mode:type_name -> atelet.UploadMode - 31, // 51: atelet.CheckpointRequest.spec:type_name -> atelet.WorkloadSpec - 50, // 52: atelet.CheckpointRequest.snapshot:type_name -> atelet.Snapshot - 53, // 53: atelet.CheckpointRequest.upload_config:type_name -> atelet.UploadConfig - 52, // 54: atelet.CheckpointResponse.state:type_name -> atelet.ActorSnapshotState - 31, // 55: atelet.RestoreRequest.spec:type_name -> atelet.WorkloadSpec - 50, // 56: atelet.RestoreRequest.snapshot:type_name -> atelet.Snapshot - 30, // 57: atelet.RestoreRequest.egress_gateway:type_name -> atelet.EgressGateway - 11, // 58: atelet.RestoreRequest.resources:type_name -> atelet.ResourceRequirements - 7, // 59: atelet.RestoreRequest.sandbox_runtime:type_name -> atelet.ActorSandboxRuntime - 50, // 60: atelet.PromoteSnapshotRequest.snapshot:type_name -> atelet.Snapshot - 52, // 61: atelet.PromoteSnapshotResponse.state:type_name -> atelet.ActorSnapshotState - 52, // 62: atelet.DetachActorResponse.state:type_name -> atelet.ActorSnapshotState - 25, // 63: atelet.WorkerSupport.MintActorCertificate:input_type -> atelet.MintActorCertificateRequest - 4, // 64: atelet.WorkerSupport.RegisterWorker:input_type -> atelet.RegisterWorkerRequest - 14, // 65: atelet.WorkerSupport.RequestActorSuspend:input_type -> atelet.RequestActorSuspendRequest - 18, // 66: atelet.WorkerSupport.GetActorEgressPolicy:input_type -> atelet.GetActorEgressPolicyRequest - 16, // 67: atelet.WorkerSupport.ReportActorCrashed:input_type -> atelet.ReportActorCrashedRequest - 29, // 68: atelet.Atelet.Run:input_type -> atelet.RunRequest - 54, // 69: atelet.Atelet.Checkpoint:input_type -> atelet.CheckpointRequest - 56, // 70: atelet.Atelet.Restore:input_type -> atelet.RestoreRequest - 58, // 71: atelet.Atelet.PromoteSnapshot:input_type -> atelet.PromoteSnapshotRequest - 60, // 72: atelet.Atelet.DetachActor:input_type -> atelet.DetachActorRequest - 27, // 73: atelet.Atelet.Terminate:input_type -> atelet.TerminateRequest - 26, // 74: atelet.WorkerSupport.MintActorCertificate:output_type -> atelet.MintActorCertificateResponse - 13, // 75: atelet.WorkerSupport.RegisterWorker:output_type -> atelet.RegisterWorkerResponse - 15, // 76: atelet.WorkerSupport.RequestActorSuspend:output_type -> atelet.RequestActorSuspendResponse - 19, // 77: atelet.WorkerSupport.GetActorEgressPolicy:output_type -> atelet.GetActorEgressPolicyResponse - 17, // 78: atelet.WorkerSupport.ReportActorCrashed:output_type -> atelet.ReportActorCrashedResponse - 48, // 79: atelet.Atelet.Run:output_type -> atelet.RunResponse - 55, // 80: atelet.Atelet.Checkpoint:output_type -> atelet.CheckpointResponse - 57, // 81: atelet.Atelet.Restore:output_type -> atelet.RestoreResponse - 59, // 82: atelet.Atelet.PromoteSnapshot:output_type -> atelet.PromoteSnapshotResponse - 61, // 83: atelet.Atelet.DetachActor:output_type -> atelet.DetachActorResponse - 28, // 84: atelet.Atelet.Terminate:output_type -> atelet.TerminateResponse - 74, // [74:85] is the sub-list for method output_type - 63, // [63:74] is the sub-list for method input_type + 35, // 51: atelet.CheckpointRequest.spec:type_name -> atelet.WorkloadSpec + 54, // 52: atelet.CheckpointRequest.snapshot:type_name -> atelet.Snapshot + 57, // 53: atelet.CheckpointRequest.upload_config:type_name -> atelet.UploadConfig + 56, // 54: atelet.CheckpointResponse.state:type_name -> atelet.ActorSnapshotState + 35, // 55: atelet.RestoreRequest.spec:type_name -> atelet.WorkloadSpec + 54, // 56: atelet.RestoreRequest.snapshot:type_name -> atelet.Snapshot + 34, // 57: atelet.RestoreRequest.egress_gateway:type_name -> atelet.EgressGateway + 15, // 58: atelet.RestoreRequest.resources:type_name -> atelet.ResourceRequirements + 11, // 59: atelet.RestoreRequest.sandbox_runtime:type_name -> atelet.ActorSandboxRuntime + 54, // 60: atelet.PromoteSnapshotRequest.snapshot:type_name -> atelet.Snapshot + 56, // 61: atelet.PromoteSnapshotResponse.state:type_name -> atelet.ActorSnapshotState + 56, // 62: atelet.DetachActorResponse.state:type_name -> atelet.ActorSnapshotState + 29, // 63: atelet.WorkerSupport.MintActorCertificate:input_type -> atelet.MintActorCertificateRequest + 4, // 64: atelet.WorkerSupport.AcquireWorkerEpoch:input_type -> atelet.AcquireWorkerEpochRequest + 6, // 65: atelet.WorkerSupport.RegisterWorker:input_type -> atelet.RegisterWorkerRequest + 7, // 66: atelet.WorkerSupport.SetWorkerSandboxRuntimes:input_type -> atelet.SetWorkerSandboxRuntimesRequest + 18, // 67: atelet.WorkerSupport.RequestActorSuspend:input_type -> atelet.RequestActorSuspendRequest + 22, // 68: atelet.WorkerSupport.GetActorEgressPolicy:input_type -> atelet.GetActorEgressPolicyRequest + 20, // 69: atelet.WorkerSupport.ReportActorCrashed:input_type -> atelet.ReportActorCrashedRequest + 33, // 70: atelet.Atelet.Run:input_type -> atelet.RunRequest + 58, // 71: atelet.Atelet.Checkpoint:input_type -> atelet.CheckpointRequest + 60, // 72: atelet.Atelet.Restore:input_type -> atelet.RestoreRequest + 62, // 73: atelet.Atelet.PromoteSnapshot:input_type -> atelet.PromoteSnapshotRequest + 64, // 74: atelet.Atelet.DetachActor:input_type -> atelet.DetachActorRequest + 31, // 75: atelet.Atelet.Terminate:input_type -> atelet.TerminateRequest + 30, // 76: atelet.WorkerSupport.MintActorCertificate:output_type -> atelet.MintActorCertificateResponse + 5, // 77: atelet.WorkerSupport.AcquireWorkerEpoch:output_type -> atelet.AcquireWorkerEpochResponse + 17, // 78: atelet.WorkerSupport.RegisterWorker:output_type -> atelet.RegisterWorkerResponse + 8, // 79: atelet.WorkerSupport.SetWorkerSandboxRuntimes:output_type -> atelet.SetWorkerSandboxRuntimesResponse + 19, // 80: atelet.WorkerSupport.RequestActorSuspend:output_type -> atelet.RequestActorSuspendResponse + 23, // 81: atelet.WorkerSupport.GetActorEgressPolicy:output_type -> atelet.GetActorEgressPolicyResponse + 21, // 82: atelet.WorkerSupport.ReportActorCrashed:output_type -> atelet.ReportActorCrashedResponse + 52, // 83: atelet.Atelet.Run:output_type -> atelet.RunResponse + 59, // 84: atelet.Atelet.Checkpoint:output_type -> atelet.CheckpointResponse + 61, // 85: atelet.Atelet.Restore:output_type -> atelet.RestoreResponse + 63, // 86: atelet.Atelet.PromoteSnapshot:output_type -> atelet.PromoteSnapshotResponse + 65, // 87: atelet.Atelet.DetachActor:output_type -> atelet.DetachActorResponse + 32, // 88: atelet.Atelet.Terminate:output_type -> atelet.TerminateResponse + 76, // [76:89] is the sub-list for method output_type + 63, // [63:76] is the sub-list for method input_type 63, // [63:63] is the sub-list for extension type_name 63, // [63:63] is the sub-list for extension extendee 0, // [0:63] is the sub-list for field type_name @@ -4587,15 +4818,15 @@ func file_atelet_proto_init() { if File_atelet_proto != nil { return } - file_atelet_proto_msgTypes[25].OneofWrappers = []any{} - file_atelet_proto_msgTypes[52].OneofWrappers = []any{} + file_atelet_proto_msgTypes[29].OneofWrappers = []any{} + file_atelet_proto_msgTypes[56].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_atelet_proto_rawDesc), len(file_atelet_proto_rawDesc)), NumEnums: 4, - NumMessages: 60, + NumMessages: 64, NumExtensions: 0, NumServices: 2, }, diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index 7d1680cec4..f68551c7b6 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -32,9 +32,21 @@ service WorkerSupport { // the actor filesystem, when/if we support those). rpc MintActorCertificate(MintActorCertificateRequest) returns (MintActorCertificateResponse) {} - // Register capacity and supported sandbox runtimes for this worker with atelet. + // AcquireWorkerEpoch hands out a new epoch for this worker process, for + // atelet to forward to the control plane's WorkerService, which this + // mirrors. A worker process calls it once at startup, before RegisterWorker. + rpc AcquireWorkerEpoch(AcquireWorkerEpochRequest) returns (AcquireWorkerEpochResponse) {} + + // RegisterWorker activates the epoch this worker process acquired and + // records its pod IPs and capacity. A worker process registers once, and + // takes no workloads until it has. FAILED_PRECONDITION means another worker + // process has registered since; this one must not serve. rpc RegisterWorker(RegisterWorkerRequest) returns (RegisterWorkerResponse) {} + // SetWorkerSandboxRuntimes records the sandbox runtimes this worker process + // can run. Called after RegisterWorker, and again whenever they change. + rpc SetWorkerSandboxRuntimes(SetWorkerSandboxRuntimesRequest) returns (SetWorkerSandboxRuntimesResponse) {} + // RequestActorSuspend asks that the actor this worker hosts be suspended, // for atelet to forward to the control plane's WorkerService, which this // mirrors. @@ -58,12 +70,42 @@ service WorkerSupport { rpc ReportActorCrashed(ReportActorCrashedRequest) returns (ReportActorCrashedResponse) {} } +message AcquireWorkerEpochRequest { +} + +message AcquireWorkerEpochResponse { + // The epoch handed out, for this worker process to register with. + int64 epoch = 1; +} + message RegisterWorkerRequest { + // The epoch this worker process acquired through AcquireWorkerEpoch. + // + // +k8s:required + // +k8s:minimum=1 + int64 epoch = 2; + + // The worker pod's IPs, at most one per IP family, in the order Kubernetes + // reports them. + // + // +k8s:required + // +k8s:maxItems=2 + // +k8s:listType=atomic + repeated string ips = 3; + // What the worker can supply. atelet converts it to the control plane's // WorkerResources when forwarding. // // +k8s:required WorkerResources capacity = 1; +} + +message SetWorkerSandboxRuntimesRequest { + // The epoch this worker process registered with. + // + // +k8s:required + // +k8s:minimum=1 + int64 epoch = 1; // The sandbox classes this worker runs and the compatibility versions it // runs them under. atelet converts it to the control plane's @@ -77,6 +119,9 @@ message RegisterWorkerRequest { repeated WorkerSandboxClass sandbox_classes = 2; } +message SetWorkerSandboxRuntimesResponse { +} + // WorkerSandboxClass is one sandbox class a worker runs, with the runtimes // it runs it under. It mirrors the control plane's WorkerSandboxClass and is // validated by the same rules. @@ -844,6 +889,9 @@ message HttpGetAction { } message RunResponse { + // The epoch of the worker process that started the workload, as it + // registered with the control plane. + int64 worker_epoch = 1; } // SnapshotFidelity mirrors the public ateapi enum: the state layers a @@ -1152,6 +1200,9 @@ message RestoreRequest { } message RestoreResponse { + // The epoch of the worker process that restored the workload, as it + // registered with the control plane. + int64 worker_epoch = 1; } message PromoteSnapshotRequest { diff --git a/internal/proto/ateletpb/atelet_grpc.pb.go b/internal/proto/ateletpb/atelet_grpc.pb.go index 87d4d8915a..a3c2a54c82 100644 --- a/internal/proto/ateletpb/atelet_grpc.pb.go +++ b/internal/proto/ateletpb/atelet_grpc.pb.go @@ -33,11 +33,13 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - WorkerSupport_MintActorCertificate_FullMethodName = "/atelet.WorkerSupport/MintActorCertificate" - WorkerSupport_RegisterWorker_FullMethodName = "/atelet.WorkerSupport/RegisterWorker" - WorkerSupport_RequestActorSuspend_FullMethodName = "/atelet.WorkerSupport/RequestActorSuspend" - WorkerSupport_GetActorEgressPolicy_FullMethodName = "/atelet.WorkerSupport/GetActorEgressPolicy" - WorkerSupport_ReportActorCrashed_FullMethodName = "/atelet.WorkerSupport/ReportActorCrashed" + WorkerSupport_MintActorCertificate_FullMethodName = "/atelet.WorkerSupport/MintActorCertificate" + WorkerSupport_AcquireWorkerEpoch_FullMethodName = "/atelet.WorkerSupport/AcquireWorkerEpoch" + WorkerSupport_RegisterWorker_FullMethodName = "/atelet.WorkerSupport/RegisterWorker" + WorkerSupport_SetWorkerSandboxRuntimes_FullMethodName = "/atelet.WorkerSupport/SetWorkerSandboxRuntimes" + WorkerSupport_RequestActorSuspend_FullMethodName = "/atelet.WorkerSupport/RequestActorSuspend" + WorkerSupport_GetActorEgressPolicy_FullMethodName = "/atelet.WorkerSupport/GetActorEgressPolicy" + WorkerSupport_ReportActorCrashed_FullMethodName = "/atelet.WorkerSupport/ReportActorCrashed" ) // WorkerSupportClient is the client API for WorkerSupport service. @@ -55,8 +57,18 @@ type WorkerSupportClient interface { // MintActorCertificate (which would be used for certificates projected into // the actor filesystem, when/if we support those). MintActorCertificate(ctx context.Context, in *MintActorCertificateRequest, opts ...grpc.CallOption) (*MintActorCertificateResponse, error) - // Register capacity and supported sandbox runtimes for this worker with atelet. + // AcquireWorkerEpoch hands out a new epoch for this worker process, for + // atelet to forward to the control plane's WorkerService, which this + // mirrors. A worker process calls it once at startup, before RegisterWorker. + AcquireWorkerEpoch(ctx context.Context, in *AcquireWorkerEpochRequest, opts ...grpc.CallOption) (*AcquireWorkerEpochResponse, error) + // RegisterWorker activates the epoch this worker process acquired and + // records its pod IPs and capacity. A worker process registers once, and + // takes no workloads until it has. FAILED_PRECONDITION means another worker + // process has registered since; this one must not serve. RegisterWorker(ctx context.Context, in *RegisterWorkerRequest, opts ...grpc.CallOption) (*RegisterWorkerResponse, error) + // SetWorkerSandboxRuntimes records the sandbox runtimes this worker process + // can run. Called after RegisterWorker, and again whenever they change. + SetWorkerSandboxRuntimes(ctx context.Context, in *SetWorkerSandboxRuntimesRequest, opts ...grpc.CallOption) (*SetWorkerSandboxRuntimesResponse, error) // RequestActorSuspend asks that the actor this worker hosts be suspended, // for atelet to forward to the control plane's WorkerService, which this // mirrors. @@ -96,6 +108,16 @@ func (c *workerSupportClient) MintActorCertificate(ctx context.Context, in *Mint return out, nil } +func (c *workerSupportClient) AcquireWorkerEpoch(ctx context.Context, in *AcquireWorkerEpochRequest, opts ...grpc.CallOption) (*AcquireWorkerEpochResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(AcquireWorkerEpochResponse) + err := c.cc.Invoke(ctx, WorkerSupport_AcquireWorkerEpoch_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *workerSupportClient) RegisterWorker(ctx context.Context, in *RegisterWorkerRequest, opts ...grpc.CallOption) (*RegisterWorkerResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(RegisterWorkerResponse) @@ -106,6 +128,16 @@ func (c *workerSupportClient) RegisterWorker(ctx context.Context, in *RegisterWo return out, nil } +func (c *workerSupportClient) SetWorkerSandboxRuntimes(ctx context.Context, in *SetWorkerSandboxRuntimesRequest, opts ...grpc.CallOption) (*SetWorkerSandboxRuntimesResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SetWorkerSandboxRuntimesResponse) + err := c.cc.Invoke(ctx, WorkerSupport_SetWorkerSandboxRuntimes_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *workerSupportClient) RequestActorSuspend(ctx context.Context, in *RequestActorSuspendRequest, opts ...grpc.CallOption) (*RequestActorSuspendResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(RequestActorSuspendResponse) @@ -151,8 +183,18 @@ type WorkerSupportServer interface { // MintActorCertificate (which would be used for certificates projected into // the actor filesystem, when/if we support those). MintActorCertificate(context.Context, *MintActorCertificateRequest) (*MintActorCertificateResponse, error) - // Register capacity and supported sandbox runtimes for this worker with atelet. + // AcquireWorkerEpoch hands out a new epoch for this worker process, for + // atelet to forward to the control plane's WorkerService, which this + // mirrors. A worker process calls it once at startup, before RegisterWorker. + AcquireWorkerEpoch(context.Context, *AcquireWorkerEpochRequest) (*AcquireWorkerEpochResponse, error) + // RegisterWorker activates the epoch this worker process acquired and + // records its pod IPs and capacity. A worker process registers once, and + // takes no workloads until it has. FAILED_PRECONDITION means another worker + // process has registered since; this one must not serve. RegisterWorker(context.Context, *RegisterWorkerRequest) (*RegisterWorkerResponse, error) + // SetWorkerSandboxRuntimes records the sandbox runtimes this worker process + // can run. Called after RegisterWorker, and again whenever they change. + SetWorkerSandboxRuntimes(context.Context, *SetWorkerSandboxRuntimesRequest) (*SetWorkerSandboxRuntimesResponse, error) // RequestActorSuspend asks that the actor this worker hosts be suspended, // for atelet to forward to the control plane's WorkerService, which this // mirrors. @@ -185,9 +227,15 @@ type UnimplementedWorkerSupportServer struct{} func (UnimplementedWorkerSupportServer) MintActorCertificate(context.Context, *MintActorCertificateRequest) (*MintActorCertificateResponse, error) { return nil, status.Error(codes.Unimplemented, "method MintActorCertificate not implemented") } +func (UnimplementedWorkerSupportServer) AcquireWorkerEpoch(context.Context, *AcquireWorkerEpochRequest) (*AcquireWorkerEpochResponse, error) { + return nil, status.Error(codes.Unimplemented, "method AcquireWorkerEpoch not implemented") +} func (UnimplementedWorkerSupportServer) RegisterWorker(context.Context, *RegisterWorkerRequest) (*RegisterWorkerResponse, error) { return nil, status.Error(codes.Unimplemented, "method RegisterWorker not implemented") } +func (UnimplementedWorkerSupportServer) SetWorkerSandboxRuntimes(context.Context, *SetWorkerSandboxRuntimesRequest) (*SetWorkerSandboxRuntimesResponse, error) { + return nil, status.Error(codes.Unimplemented, "method SetWorkerSandboxRuntimes not implemented") +} func (UnimplementedWorkerSupportServer) RequestActorSuspend(context.Context, *RequestActorSuspendRequest) (*RequestActorSuspendResponse, error) { return nil, status.Error(codes.Unimplemented, "method RequestActorSuspend not implemented") } @@ -236,6 +284,24 @@ func _WorkerSupport_MintActorCertificate_Handler(srv interface{}, ctx context.Co return interceptor(ctx, in, info, handler) } +func _WorkerSupport_AcquireWorkerEpoch_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AcquireWorkerEpochRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(WorkerSupportServer).AcquireWorkerEpoch(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: WorkerSupport_AcquireWorkerEpoch_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(WorkerSupportServer).AcquireWorkerEpoch(ctx, req.(*AcquireWorkerEpochRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _WorkerSupport_RegisterWorker_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(RegisterWorkerRequest) if err := dec(in); err != nil { @@ -254,6 +320,24 @@ func _WorkerSupport_RegisterWorker_Handler(srv interface{}, ctx context.Context, return interceptor(ctx, in, info, handler) } +func _WorkerSupport_SetWorkerSandboxRuntimes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SetWorkerSandboxRuntimesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(WorkerSupportServer).SetWorkerSandboxRuntimes(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: WorkerSupport_SetWorkerSandboxRuntimes_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(WorkerSupportServer).SetWorkerSandboxRuntimes(ctx, req.(*SetWorkerSandboxRuntimesRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _WorkerSupport_RequestActorSuspend_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(RequestActorSuspendRequest) if err := dec(in); err != nil { @@ -319,10 +403,18 @@ var WorkerSupport_ServiceDesc = grpc.ServiceDesc{ MethodName: "MintActorCertificate", Handler: _WorkerSupport_MintActorCertificate_Handler, }, + { + MethodName: "AcquireWorkerEpoch", + Handler: _WorkerSupport_AcquireWorkerEpoch_Handler, + }, { MethodName: "RegisterWorker", Handler: _WorkerSupport_RegisterWorker_Handler, }, + { + MethodName: "SetWorkerSandboxRuntimes", + Handler: _WorkerSupport_SetWorkerSandboxRuntimes_Handler, + }, { MethodName: "RequestActorSuspend", Handler: _WorkerSupport_RequestActorSuspend_Handler, diff --git a/internal/proto/ateworkerpb/ateworker.pb.go b/internal/proto/ateworkerpb/ateworker.pb.go index 3e8e84a58d..66c5162b32 100644 --- a/internal/proto/ateworkerpb/ateworker.pb.go +++ b/internal/proto/ateworkerpb/ateworker.pb.go @@ -1231,7 +1231,9 @@ func (x *HttpGetAction) GetPort() int32 { } type RunWorkloadResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` + state protoimpl.MessageState `protogen:"open.v1"` + // The epoch this worker process registered with. + WorkerEpoch int64 `protobuf:"varint,1,opt,name=worker_epoch,json=workerEpoch,proto3" json:"worker_epoch,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1266,6 +1268,13 @@ func (*RunWorkloadResponse) Descriptor() ([]byte, []int) { return file_ateworker_proto_rawDescGZIP(), []int{13} } +func (x *RunWorkloadResponse) GetWorkerEpoch() int64 { + if x != nil { + return x.WorkerEpoch + } + return 0 +} + type CheckpointWorkloadRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // +k8s:required @@ -1635,7 +1644,9 @@ func (x *RestoreWorkloadRequest) GetPreserveRestoreDir() bool { } type RestoreWorkloadResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` + state protoimpl.MessageState `protogen:"open.v1"` + // The epoch this worker process registered with. + WorkerEpoch int64 `protobuf:"varint,1,opt,name=worker_epoch,json=workerEpoch,proto3" json:"worker_epoch,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1670,6 +1681,13 @@ func (*RestoreWorkloadResponse) Descriptor() ([]byte, []int) { return file_ateworker_proto_rawDescGZIP(), []int{17} } +func (x *RestoreWorkloadResponse) GetWorkerEpoch() int64 { + if x != nil { + return x.WorkerEpoch + } + return 0 +} + type GetWorkloadStatsRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // The actor the caller believes is executing here. A worker can be recycled @@ -2108,8 +2126,9 @@ const file_ateworker_proto_rawDesc = "" + "\x0ftimeout_seconds\x18\x02 \x01(\x05R\x0etimeoutSeconds\"7\n" + "\rHttpGetAction\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x12\n" + - "\x04port\x18\x02 \x01(\x05R\x04port\"\x15\n" + - "\x13RunWorkloadResponse\"\x95\x03\n" + + "\x04port\x18\x02 \x01(\x05R\x04port\"8\n" + + "\x13RunWorkloadResponse\x12!\n" + + "\fworker_epoch\x18\x01 \x01(\x03R\vworkerEpoch\"\x95\x03\n" + "\x19CheckpointWorkloadRequest\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1d\n" + "\n" + @@ -2147,8 +2166,9 @@ const file_ateworker_proto_rawDesc = "" + "\x16RuntimeAssetPathsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01B\x11\n" + - "\x0f_egress_gateway\"\x19\n" + - "\x17RestoreWorkloadResponse\"6\n" + + "\x0f_egress_gateway\"<\n" + + "\x17RestoreWorkloadResponse\x12!\n" + + "\fworker_epoch\x18\x01 \x01(\x03R\vworkerEpoch\"6\n" + "\x17GetWorkloadStatsRequest\x12\x1b\n" + "\tactor_uid\x18\x01 \x01(\tR\bactorUid\"\xc2\x04\n" + "\x13WorkloadStatsSample\x12\x1a\n" + diff --git a/internal/proto/ateworkerpb/ateworker.proto b/internal/proto/ateworkerpb/ateworker.proto index 75cc5049f7..cc868f6223 100644 --- a/internal/proto/ateworkerpb/ateworker.proto +++ b/internal/proto/ateworkerpb/ateworker.proto @@ -424,6 +424,8 @@ message HttpGetAction { } message RunWorkloadResponse { + // The epoch this worker process registered with. + int64 worker_epoch = 1; } // SnapshotFidelity mirrors the public ateapi enum: the state layers a @@ -556,6 +558,8 @@ message RestoreWorkloadRequest { } message RestoreWorkloadResponse { + // The epoch this worker process registered with. + int64 worker_epoch = 1; } message GetWorkloadStatsRequest { diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index 82f8fa7f4a..2a7a62178f 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -2110,8 +2110,10 @@ type ActorStatus struct { // worker_assignment points at the worker currently hosting this Actor. // Unset whenever the Actor has no worker (SUSPENDED, CRASHED). // + // It can be set and cleared, but not pointed at another Worker in place: + // the fields that identify the Worker cannot change while it is set. + // // +k8s:optional - // +k8s:update=NoModify # can be set and cleared, but not changed in place WorkerAssignment *WorkerAssignment `protobuf:"bytes,2,opt,name=worker_assignment,json=workerAssignment,proto3" json:"worker_assignment,omitempty"` // external_volumes is the state of the external volumes attached to the // actor. These volumes only live as long as the actor and are deleted with @@ -2297,8 +2299,11 @@ func (x *ActorCrash) GetCrashTime() *timestamppb.Timestamp { // // This is a denormalized snapshot, not merely a reference: atenet reads // worker_pod_ips on the request-routing path and must not need a second lookup, -// and kubectl-ate displays the pod name. The copies stay valid for the life of -// the assignment because a Worker's identity fields are immutable. +// and kubectl-ate displays the pod name. The copies of the Worker's identity +// fields stay valid for the life of the assignment because those fields are +// immutable, and cannot change in place. worker_pod_ips and worker_epoch +// describe the worker process the Actor runs under, and are updated when the +// Actor becomes RUNNING. // // Pass `worker` to GetWorker; none of the denormalized fields identify the // Worker to the API. @@ -2312,28 +2317,34 @@ type WorkerAssignment struct { // // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix // +k8s:required + // +k8s:update=NoModify Worker *ObjectRef `protobuf:"bytes,6,opt,name=worker,proto3" json:"worker,omitempty"` // worker_namespace is the Kubernetes namespace of the WorkerPool and worker Pod. // // +k8s:required // +k8s:format=k8s-short-name + // +k8s:update=NoModify WorkerNamespace string `protobuf:"bytes,1,opt,name=worker_namespace,json=workerNamespace,proto3" json:"worker_namespace,omitempty"` // worker_pool is the name of the WorkerPool the worker belongs to. // // +k8s:required // +k8s:format=k8s-long-name + // +k8s:update=NoModify WorkerPool string `protobuf:"bytes,2,opt,name=worker_pool,json=workerPool,proto3" json:"worker_pool,omitempty"` // worker_pod is the name of the Kubernetes pod hosting the Actor. // // +k8s:required // +k8s:format=k8s-long-name + // +k8s:update=NoModify WorkerPod string `protobuf:"bytes,3,opt,name=worker_pod,json=workerPod,proto3" json:"worker_pod,omitempty"` // worker_pod_uid is the Kubernetes UID of worker_pod. // // +k8s:required // +k8s:format=k8s-uuid + // +k8s:update=NoModify WorkerPodUid string `protobuf:"bytes,4,opt,name=worker_pod_uid,json=workerPodUid,proto3" json:"worker_pod_uid,omitempty"` - // worker_pod_ips are the IPs of worker_pod, copied from Worker.ips. + // worker_pod_ips are the IPs of worker_pod, copied from Worker.status.ips + // when the Actor is placed, and again when it becomes RUNNING. // // +k8s:required // +k8s:maxItems=2 @@ -2344,9 +2355,11 @@ type WorkerAssignment struct { // // +k8s:required // +k8s:format=k8s-long-name + // +k8s:update=NoModify NodeName string `protobuf:"bytes,7,opt,name=node_name,json=nodeName,proto3" json:"node_name,omitempty"` - // worker_epoch is the Worker's epoch when the Actor was placed on it. 0 if - // the Worker had not reported one. + // worker_epoch is the epoch of the worker process the Actor runs under, as + // that process reports it from Run or Restore. 0 until the Actor becomes + // RUNNING. // // +k8s:optional // +k8s:minimum=0 @@ -2355,6 +2368,7 @@ type WorkerAssignment struct { // was started or restored under on this Worker. // // +k8s:optional + // +k8s:update=NoModify SandboxRuntime *ActorSandboxRuntime `protobuf:"bytes,9,opt,name=sandbox_runtime,json=sandboxRuntime,proto3" json:"sandbox_runtime,omitempty"` // worker_spiffe_id is the SPIFFE ID worker_pod's certificate carries, copied // from Worker.worker_spiffe_id. @@ -2362,6 +2376,7 @@ type WorkerAssignment struct { // +k8s:optional // +k8s:maxLength=2048 // +k8s:customValidation # a SPIFFE ID + // +k8s:update=NoModify WorkerSpiffeId string `protobuf:"bytes,10,opt,name=worker_spiffe_id,json=workerSpiffeId,proto3" json:"worker_spiffe_id,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -7336,15 +7351,6 @@ type Worker struct { // +k8s:format=k8s-long-name // +k8s:immutable NodeName string `protobuf:"bytes,6,opt,name=node_name,json=nodeName,proto3" json:"node_name,omitempty"` - // ips are the worker pod's IPs, at most one per IP family, in the order - // Kubernetes reports them; the first is in the cluster's primary family. - // - // +k8s:required - // +k8s:maxItems=2 - // +k8s:listType=atomic - // +k8s:customValidation # until `format=k8s-ip` is supported - // +k8s:immutable - Ips []string `protobuf:"bytes,7,rep,name=ips,proto3" json:"ips,omitempty"` // worker_spiffe_id is the SPIFFE ID the worker pod's certificate carries. // atenet-router requires it on the certificate of the worker's atunnel. // @@ -7352,7 +7358,7 @@ type Worker struct { // +k8s:maxLength=2048 // +k8s:customValidation # a SPIFFE ID // +k8s:immutable - WorkerSpiffeId string `protobuf:"bytes,8,opt,name=worker_spiffe_id,json=workerSpiffeId,proto3" json:"worker_spiffe_id,omitempty"` + WorkerSpiffeId string `protobuf:"bytes,7,opt,name=worker_spiffe_id,json=workerSpiffeId,proto3" json:"worker_spiffe_id,omitempty"` // labels mirror the WorkerPool object's Kubernetes labels, which selectors // match against. // Kubernetes does not cap an object's label count, so the bound below is @@ -7362,27 +7368,14 @@ type Worker struct { // +k8s:maxProperties=64 // +k8s:eachKey=+k8s:format=k8s-label-key // +k8s:eachVal=+k8s:format=k8s-label-value - Labels map[string]string `protobuf:"bytes,9,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - // epoch counts the runs of the worker pod's worker container: 1 for its first - // run and one more for each restart. 0 means it has not been reported. - // - // A restarted worker has lost the sandboxes of the Actors it was hosting, so - // once epoch rises the control plane crashes every Actor placed during an - // earlier epoch; status.observed_epoch reports when that is done. It may only - // increase. - // - // +k8s:optional - // +k8s:minimum=0 - // +k8s:update=NoUnset - // +k8s:customValidation - Epoch int64 `protobuf:"varint,10,opt,name=epoch,proto3" json:"epoch,omitempty"` + Labels map[string]string `protobuf:"bytes,8,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // Output-only server-managed state. Absent from Create/Update request // payloads; whatever a request carries here is ignored. A client moves // state only through DrainWorker, MarkWorkerAvailable and // MarkWorkerUnavailable, and the assignments are the scheduler's. // // +k8s:optional - Status *WorkerStatus `protobuf:"bytes,11,opt,name=status,proto3" json:"status,omitempty"` + Status *WorkerStatus `protobuf:"bytes,9,opt,name=status,proto3" json:"status,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -7459,13 +7452,6 @@ func (x *Worker) GetNodeName() string { return "" } -func (x *Worker) GetIps() []string { - if x != nil { - return x.Ips - } - return nil -} - func (x *Worker) GetWorkerSpiffeId() string { if x != nil { return x.WorkerSpiffeId @@ -7480,13 +7466,6 @@ func (x *Worker) GetLabels() map[string]string { return nil } -func (x *Worker) GetEpoch() int64 { - if x != nil { - return x.Epoch - } - return 0 -} - func (x *Worker) GetStatus() *WorkerStatus { if x != nil { return x.Status @@ -7518,25 +7497,53 @@ type WorkerStatus struct { // +k8s:optional Allocated *WorkerResources `protobuf:"bytes,3,opt,name=allocated,proto3" json:"allocated,omitempty"` // observed_epoch is the latest epoch whose earlier Actors the control plane - // has crashed and released. While it is below epoch, Actors placed before - // the worker's last restart may still be reported as running. + // has crashed and released. While it is below registered_epoch, Actors placed + // before the worker's last restart may still be reported as running. // // +k8s:optional // +k8s:minimum=0 ObservedEpoch int64 `protobuf:"varint,4,opt,name=observed_epoch,json=observedEpoch,proto3" json:"observed_epoch,omitempty"` - // The sandbox classes this Worker runs and the runtimes it runs them - // under, as reported by the Worker at registration; empty until then, and - // nothing is placed on the Worker until then. One entry today, like - // WorkerPoolSpec.sandboxClasses. Its sandbox_class is the Worker's class, - // fixed by the first RegisterWorker call. + // The sandbox classes this Worker runs and the runtimes it runs them under, + // as the registered worker process reports them through + // WorkerService.SetWorkerSandboxRuntimes. Cleared when a new worker process + // registers, and nothing is placed on the Worker until it reports them. One + // entry today, like WorkerPoolSpec.sandboxClasses. Its sandbox_class is the + // Worker's class. // // +k8s:optional // +k8s:maxItems=1 # keep in sync with WorkerPoolSpec.sandboxClasses // +k8s:listType=map // +k8s:listMapKey=sandbox_class SandboxClasses []*WorkerSandboxClass `protobuf:"bytes,5,rep,name=sandbox_classes,json=sandboxClasses,proto3" json:"sandbox_classes,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // ips are the worker pod's IPs, at most one per IP family, in the order + // Kubernetes reports them; the first is in the cluster's primary family. + // The worker process reports them when it registers, so they follow a pod + // sandbox that is recreated with new IPs. + // + // +k8s:optional + // +k8s:maxItems=2 + // +k8s:listType=atomic + // +k8s:customValidation # until `format=k8s-ip` is supported + Ips []string `protobuf:"bytes,6,rep,name=ips,proto3" json:"ips,omitempty"` + // issued_epoch is the last epoch handed out by + // WorkerService.AcquireWorkerEpoch. Each worker process acquires one at + // startup. + // + // +k8s:optional + // +k8s:minimum=0 + IssuedEpoch int64 `protobuf:"varint,7,opt,name=issued_epoch,json=issuedEpoch,proto3" json:"issued_epoch,omitempty"` + // registered_epoch is the epoch of the worker process that registered last, + // through WorkerService.RegisterWorker. Only one worker process runs in a pod + // at a time, and a process registers only an epoch it acquired, so a rise + // means every earlier process has exited and lost its sandboxes. The control + // plane then crashes the Actors that ran under an earlier epoch. 0 until the + // first registration; a Worker takes no Actors before then. + // + // +k8s:optional + // +k8s:minimum=0 + RegisteredEpoch int64 `protobuf:"varint,8,opt,name=registered_epoch,json=registeredEpoch,proto3" json:"registered_epoch,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *WorkerStatus) Reset() { @@ -7604,6 +7611,27 @@ func (x *WorkerStatus) GetSandboxClasses() []*WorkerSandboxClass { return nil } +func (x *WorkerStatus) GetIps() []string { + if x != nil { + return x.Ips + } + return nil +} + +func (x *WorkerStatus) GetIssuedEpoch() int64 { + if x != nil { + return x.IssuedEpoch + } + return 0 +} + +func (x *WorkerStatus) GetRegisteredEpoch() int64 { + if x != nil { + return x.RegisteredEpoch + } + return 0 +} + // WorkerSandboxClass is one sandbox class a Worker runs, with the runtimes // it runs it under. A Worker has one entry per WorkerPool sandboxClasses // entry, so exactly one today. @@ -8040,12 +8068,7 @@ type ActorAssignment struct { // The resources allocated to the Actor when it was assigned. // // +k8s:optional - Resources *Resources `protobuf:"bytes,4,opt,name=resources,proto3" json:"resources,omitempty"` - // worker_epoch is the Worker's epoch when the Actor was bound. - // - // +k8s:optional - // +k8s:minimum=0 - WorkerEpoch int64 `protobuf:"varint,5,opt,name=worker_epoch,json=workerEpoch,proto3" json:"worker_epoch,omitempty"` + Resources *Resources `protobuf:"bytes,4,opt,name=resources,proto3" json:"resources,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -8108,9 +8131,96 @@ func (x *ActorAssignment) GetResources() *Resources { return nil } -func (x *ActorAssignment) GetWorkerEpoch() int64 { +type AcquireWorkerEpochRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The Worker whose worker process is starting. atespace is always empty; + // Workers are global-scoped. + // + // +k8s:required + // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix + Worker *ObjectRef `protobuf:"bytes,1,opt,name=worker,proto3" json:"worker,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AcquireWorkerEpochRequest) Reset() { + *x = AcquireWorkerEpochRequest{} + mi := &file_ateapi_proto_msgTypes[112] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AcquireWorkerEpochRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AcquireWorkerEpochRequest) ProtoMessage() {} + +func (x *AcquireWorkerEpochRequest) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[112] if x != nil { - return x.WorkerEpoch + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AcquireWorkerEpochRequest.ProtoReflect.Descriptor instead. +func (*AcquireWorkerEpochRequest) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{112} +} + +func (x *AcquireWorkerEpochRequest) GetWorker() *ObjectRef { + if x != nil { + return x.Worker + } + return nil +} + +type AcquireWorkerEpochResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The epoch handed out, for the worker process to register with. + Epoch int64 `protobuf:"varint,1,opt,name=epoch,proto3" json:"epoch,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AcquireWorkerEpochResponse) Reset() { + *x = AcquireWorkerEpochResponse{} + mi := &file_ateapi_proto_msgTypes[113] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AcquireWorkerEpochResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AcquireWorkerEpochResponse) ProtoMessage() {} + +func (x *AcquireWorkerEpochResponse) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[113] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AcquireWorkerEpochResponse.ProtoReflect.Descriptor instead. +func (*AcquireWorkerEpochResponse) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{113} +} + +func (x *AcquireWorkerEpochResponse) GetEpoch() int64 { + if x != nil { + return x.Epoch } return 0 } @@ -8123,30 +8233,32 @@ type RegisterWorkerRequest struct { // +k8s:required // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix Worker *ObjectRef `protobuf:"bytes,1,opt,name=worker,proto3" json:"worker,omitempty"` + // The epoch the worker process acquired through AcquireWorkerEpoch. + // + // +k8s:required + // +k8s:minimum=1 + Epoch int64 `protobuf:"varint,3,opt,name=epoch,proto3" json:"epoch,omitempty"` + // The worker pod's IPs, at most one per IP family, in the order Kubernetes + // reports them. They replace what is recorded. + // + // +k8s:required + // +k8s:maxItems=2 + // +k8s:listType=atomic + // +k8s:customValidation # until `format=k8s-ip` is supported + Ips []string `protobuf:"bytes,4,rep,name=ips,proto3" json:"ips,omitempty"` // Everything the Worker can hold. This replaces what is recorded rather than // merging into it: a dimension left out is one the Worker no longer supplies, // and an Actor asking for that dimension will not be placed here. // // +k8s:required - Capacity *WorkerResources `protobuf:"bytes,2,opt,name=capacity,proto3" json:"capacity,omitempty"` - // The sandbox classes this worker runs and the runtimes it runs them - // under. The first registration fixes the Worker's sandbox class; later - // reports replace the runtimes, but an entry of another class is refused. - // Like capacity, this replaces what is recorded on the Worker. - // - // +k8s:required - // +k8s:minItems=1 - // +k8s:maxItems=1 # keep in sync with WorkerPoolSpec.sandboxClasses - // +k8s:listType=map - // +k8s:listMapKey=sandbox_class - SandboxClasses []*WorkerSandboxClass `protobuf:"bytes,3,rep,name=sandbox_classes,json=sandboxClasses,proto3" json:"sandbox_classes,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Capacity *WorkerResources `protobuf:"bytes,2,opt,name=capacity,proto3" json:"capacity,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RegisterWorkerRequest) Reset() { *x = RegisterWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[112] + mi := &file_ateapi_proto_msgTypes[114] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8158,7 +8270,7 @@ func (x *RegisterWorkerRequest) String() string { func (*RegisterWorkerRequest) ProtoMessage() {} func (x *RegisterWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[112] + mi := &file_ateapi_proto_msgTypes[114] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8171,7 +8283,7 @@ func (x *RegisterWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerRequest.ProtoReflect.Descriptor instead. func (*RegisterWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{112} + return file_ateapi_proto_rawDescGZIP(), []int{114} } func (x *RegisterWorkerRequest) GetWorker() *ObjectRef { @@ -8181,16 +8293,23 @@ func (x *RegisterWorkerRequest) GetWorker() *ObjectRef { return nil } -func (x *RegisterWorkerRequest) GetCapacity() *WorkerResources { +func (x *RegisterWorkerRequest) GetEpoch() int64 { if x != nil { - return x.Capacity + return x.Epoch + } + return 0 +} + +func (x *RegisterWorkerRequest) GetIps() []string { + if x != nil { + return x.Ips } return nil } -func (x *RegisterWorkerRequest) GetSandboxClasses() []*WorkerSandboxClass { +func (x *RegisterWorkerRequest) GetCapacity() *WorkerResources { if x != nil { - return x.SandboxClasses + return x.Capacity } return nil } @@ -8205,7 +8324,7 @@ type RegisterWorkerResponse struct { func (x *RegisterWorkerResponse) Reset() { *x = RegisterWorkerResponse{} - mi := &file_ateapi_proto_msgTypes[113] + mi := &file_ateapi_proto_msgTypes[115] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8217,7 +8336,7 @@ func (x *RegisterWorkerResponse) String() string { func (*RegisterWorkerResponse) ProtoMessage() {} func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[113] + mi := &file_ateapi_proto_msgTypes[115] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8230,7 +8349,7 @@ func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerResponse.ProtoReflect.Descriptor instead. func (*RegisterWorkerResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{113} + return file_ateapi_proto_rawDescGZIP(), []int{115} } func (x *RegisterWorkerResponse) GetWorker() *Worker { @@ -8240,6 +8359,130 @@ func (x *RegisterWorkerResponse) GetWorker() *Worker { return nil } +type SetWorkerSandboxRuntimesRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The Worker whose runtimes these are. atespace is always empty; Workers are + // global-scoped. + // + // +k8s:required + // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix + Worker *ObjectRef `protobuf:"bytes,1,opt,name=worker,proto3" json:"worker,omitempty"` + // The epoch the worker process registered with. + // + // +k8s:required + // +k8s:minimum=1 + Epoch int64 `protobuf:"varint,2,opt,name=epoch,proto3" json:"epoch,omitempty"` + // The sandbox classes this worker process runs and the runtimes it runs them + // under. The first report of a worker process fixes the Worker's sandbox + // class; later reports replace the runtimes, but an entry of another class is + // refused. This replaces what is recorded on the Worker. + // + // +k8s:required + // +k8s:minItems=1 + // +k8s:maxItems=1 # keep in sync with WorkerPoolSpec.sandboxClasses + // +k8s:listType=map + // +k8s:listMapKey=sandbox_class + SandboxClasses []*WorkerSandboxClass `protobuf:"bytes,3,rep,name=sandbox_classes,json=sandboxClasses,proto3" json:"sandbox_classes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SetWorkerSandboxRuntimesRequest) Reset() { + *x = SetWorkerSandboxRuntimesRequest{} + mi := &file_ateapi_proto_msgTypes[116] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SetWorkerSandboxRuntimesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SetWorkerSandboxRuntimesRequest) ProtoMessage() {} + +func (x *SetWorkerSandboxRuntimesRequest) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[116] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SetWorkerSandboxRuntimesRequest.ProtoReflect.Descriptor instead. +func (*SetWorkerSandboxRuntimesRequest) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{116} +} + +func (x *SetWorkerSandboxRuntimesRequest) GetWorker() *ObjectRef { + if x != nil { + return x.Worker + } + return nil +} + +func (x *SetWorkerSandboxRuntimesRequest) GetEpoch() int64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *SetWorkerSandboxRuntimesRequest) GetSandboxClasses() []*WorkerSandboxClass { + if x != nil { + return x.SandboxClasses + } + return nil +} + +type SetWorkerSandboxRuntimesResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The Worker as recorded. + Worker *Worker `protobuf:"bytes,1,opt,name=worker,proto3" json:"worker,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SetWorkerSandboxRuntimesResponse) Reset() { + *x = SetWorkerSandboxRuntimesResponse{} + mi := &file_ateapi_proto_msgTypes[117] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SetWorkerSandboxRuntimesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SetWorkerSandboxRuntimesResponse) ProtoMessage() {} + +func (x *SetWorkerSandboxRuntimesResponse) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[117] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SetWorkerSandboxRuntimesResponse.ProtoReflect.Descriptor instead. +func (*SetWorkerSandboxRuntimesResponse) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{117} +} + +func (x *SetWorkerSandboxRuntimesResponse) GetWorker() *Worker { + if x != nil { + return x.Worker + } + return nil +} + // Request message for MintWorkerActorCertificate. type MintWorkerActorCertificateRequest struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -8277,7 +8520,7 @@ type MintWorkerActorCertificateRequest struct { func (x *MintWorkerActorCertificateRequest) Reset() { *x = MintWorkerActorCertificateRequest{} - mi := &file_ateapi_proto_msgTypes[114] + mi := &file_ateapi_proto_msgTypes[118] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8289,7 +8532,7 @@ func (x *MintWorkerActorCertificateRequest) String() string { func (*MintWorkerActorCertificateRequest) ProtoMessage() {} func (x *MintWorkerActorCertificateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[114] + mi := &file_ateapi_proto_msgTypes[118] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8302,7 +8545,7 @@ func (x *MintWorkerActorCertificateRequest) ProtoReflect() protoreflect.Message // Deprecated: Use MintWorkerActorCertificateRequest.ProtoReflect.Descriptor instead. func (*MintWorkerActorCertificateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{114} + return file_ateapi_proto_rawDescGZIP(), []int{118} } func (x *MintWorkerActorCertificateRequest) GetActor() *ObjectRef { @@ -8346,7 +8589,7 @@ type MintWorkerActorCertificateResponse struct { func (x *MintWorkerActorCertificateResponse) Reset() { *x = MintWorkerActorCertificateResponse{} - mi := &file_ateapi_proto_msgTypes[115] + mi := &file_ateapi_proto_msgTypes[119] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8358,7 +8601,7 @@ func (x *MintWorkerActorCertificateResponse) String() string { func (*MintWorkerActorCertificateResponse) ProtoMessage() {} func (x *MintWorkerActorCertificateResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[115] + mi := &file_ateapi_proto_msgTypes[119] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8371,7 +8614,7 @@ func (x *MintWorkerActorCertificateResponse) ProtoReflect() protoreflect.Message // Deprecated: Use MintWorkerActorCertificateResponse.ProtoReflect.Descriptor instead. func (*MintWorkerActorCertificateResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{115} + return file_ateapi_proto_rawDescGZIP(), []int{119} } func (x *MintWorkerActorCertificateResponse) GetActorCertificates() [][]byte { @@ -8412,7 +8655,7 @@ type MintWorkerActorJwtRequest struct { func (x *MintWorkerActorJwtRequest) Reset() { *x = MintWorkerActorJwtRequest{} - mi := &file_ateapi_proto_msgTypes[116] + mi := &file_ateapi_proto_msgTypes[120] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8424,7 +8667,7 @@ func (x *MintWorkerActorJwtRequest) String() string { func (*MintWorkerActorJwtRequest) ProtoMessage() {} func (x *MintWorkerActorJwtRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[116] + mi := &file_ateapi_proto_msgTypes[120] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8437,7 +8680,7 @@ func (x *MintWorkerActorJwtRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintWorkerActorJwtRequest.ProtoReflect.Descriptor instead. func (*MintWorkerActorJwtRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{116} + return file_ateapi_proto_rawDescGZIP(), []int{120} } func (x *MintWorkerActorJwtRequest) GetActor() *ObjectRef { @@ -8485,7 +8728,7 @@ type MintWorkerActorJwtResponse struct { func (x *MintWorkerActorJwtResponse) Reset() { *x = MintWorkerActorJwtResponse{} - mi := &file_ateapi_proto_msgTypes[117] + mi := &file_ateapi_proto_msgTypes[121] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8497,7 +8740,7 @@ func (x *MintWorkerActorJwtResponse) String() string { func (*MintWorkerActorJwtResponse) ProtoMessage() {} func (x *MintWorkerActorJwtResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[117] + mi := &file_ateapi_proto_msgTypes[121] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8510,7 +8753,7 @@ func (x *MintWorkerActorJwtResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintWorkerActorJwtResponse.ProtoReflect.Descriptor instead. func (*MintWorkerActorJwtResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{117} + return file_ateapi_proto_rawDescGZIP(), []int{121} } func (x *MintWorkerActorJwtResponse) GetActorJwt() string { @@ -8558,7 +8801,7 @@ type RequestActorSuspendRequest struct { func (x *RequestActorSuspendRequest) Reset() { *x = RequestActorSuspendRequest{} - mi := &file_ateapi_proto_msgTypes[118] + mi := &file_ateapi_proto_msgTypes[122] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8570,7 +8813,7 @@ func (x *RequestActorSuspendRequest) String() string { func (*RequestActorSuspendRequest) ProtoMessage() {} func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[118] + mi := &file_ateapi_proto_msgTypes[122] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8583,7 +8826,7 @@ func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendRequest.ProtoReflect.Descriptor instead. func (*RequestActorSuspendRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{118} + return file_ateapi_proto_rawDescGZIP(), []int{122} } func (x *RequestActorSuspendRequest) GetWorker() *ObjectRef { @@ -8617,7 +8860,7 @@ type RequestActorSuspendResponse struct { func (x *RequestActorSuspendResponse) Reset() { *x = RequestActorSuspendResponse{} - mi := &file_ateapi_proto_msgTypes[119] + mi := &file_ateapi_proto_msgTypes[123] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8629,7 +8872,7 @@ func (x *RequestActorSuspendResponse) String() string { func (*RequestActorSuspendResponse) ProtoMessage() {} func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[119] + mi := &file_ateapi_proto_msgTypes[123] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8642,7 +8885,7 @@ func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendResponse.ProtoReflect.Descriptor instead. func (*RequestActorSuspendResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{119} + return file_ateapi_proto_rawDescGZIP(), []int{123} } func (x *RequestActorSuspendResponse) GetActor() *Actor { @@ -8682,7 +8925,7 @@ type ReportActorCrashedRequest struct { func (x *ReportActorCrashedRequest) Reset() { *x = ReportActorCrashedRequest{} - mi := &file_ateapi_proto_msgTypes[120] + mi := &file_ateapi_proto_msgTypes[124] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8694,7 +8937,7 @@ func (x *ReportActorCrashedRequest) String() string { func (*ReportActorCrashedRequest) ProtoMessage() {} func (x *ReportActorCrashedRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[120] + mi := &file_ateapi_proto_msgTypes[124] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8707,7 +8950,7 @@ func (x *ReportActorCrashedRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportActorCrashedRequest.ProtoReflect.Descriptor instead. func (*ReportActorCrashedRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{120} + return file_ateapi_proto_rawDescGZIP(), []int{124} } func (x *ReportActorCrashedRequest) GetWorker() *ObjectRef { @@ -8748,7 +8991,7 @@ type ReportActorCrashedResponse struct { func (x *ReportActorCrashedResponse) Reset() { *x = ReportActorCrashedResponse{} - mi := &file_ateapi_proto_msgTypes[121] + mi := &file_ateapi_proto_msgTypes[125] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8760,7 +9003,7 @@ func (x *ReportActorCrashedResponse) String() string { func (*ReportActorCrashedResponse) ProtoMessage() {} func (x *ReportActorCrashedResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[121] + mi := &file_ateapi_proto_msgTypes[125] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8773,7 +9016,7 @@ func (x *ReportActorCrashedResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportActorCrashedResponse.ProtoReflect.Descriptor instead. func (*ReportActorCrashedResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{121} + return file_ateapi_proto_rawDescGZIP(), []int{125} } func (x *ReportActorCrashedResponse) GetActor() *Actor { @@ -8804,7 +9047,7 @@ type ReportActorSnapshotStateRequest struct { func (x *ReportActorSnapshotStateRequest) Reset() { *x = ReportActorSnapshotStateRequest{} - mi := &file_ateapi_proto_msgTypes[122] + mi := &file_ateapi_proto_msgTypes[126] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8816,7 +9059,7 @@ func (x *ReportActorSnapshotStateRequest) String() string { func (*ReportActorSnapshotStateRequest) ProtoMessage() {} func (x *ReportActorSnapshotStateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[122] + mi := &file_ateapi_proto_msgTypes[126] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8829,7 +9072,7 @@ func (x *ReportActorSnapshotStateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportActorSnapshotStateRequest.ProtoReflect.Descriptor instead. func (*ReportActorSnapshotStateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{122} + return file_ateapi_proto_rawDescGZIP(), []int{126} } func (x *ReportActorSnapshotStateRequest) GetActor() *ObjectRef { @@ -8861,7 +9104,7 @@ type ReportActorSnapshotStateResponse struct { func (x *ReportActorSnapshotStateResponse) Reset() { *x = ReportActorSnapshotStateResponse{} - mi := &file_ateapi_proto_msgTypes[123] + mi := &file_ateapi_proto_msgTypes[127] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8873,7 +9116,7 @@ func (x *ReportActorSnapshotStateResponse) String() string { func (*ReportActorSnapshotStateResponse) ProtoMessage() {} func (x *ReportActorSnapshotStateResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[123] + mi := &file_ateapi_proto_msgTypes[127] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8886,7 +9129,7 @@ func (x *ReportActorSnapshotStateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportActorSnapshotStateResponse.ProtoReflect.Descriptor instead. func (*ReportActorSnapshotStateResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{123} + return file_ateapi_proto_rawDescGZIP(), []int{127} } // ActorSnapshotState is what atelet holds for one Actor on its node. It @@ -8920,7 +9163,7 @@ type ActorSnapshotState struct { func (x *ActorSnapshotState) Reset() { *x = ActorSnapshotState{} - mi := &file_ateapi_proto_msgTypes[124] + mi := &file_ateapi_proto_msgTypes[128] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8932,7 +9175,7 @@ func (x *ActorSnapshotState) String() string { func (*ActorSnapshotState) ProtoMessage() {} func (x *ActorSnapshotState) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[124] + mi := &file_ateapi_proto_msgTypes[128] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8945,7 +9188,7 @@ func (x *ActorSnapshotState) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorSnapshotState.ProtoReflect.Descriptor instead. func (*ActorSnapshotState) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{124} + return file_ateapi_proto_rawDescGZIP(), []int{128} } func (x *ActorSnapshotState) GetLatestLocal() *SnapshotReport { @@ -9017,7 +9260,7 @@ type SnapshotReport struct { func (x *SnapshotReport) Reset() { *x = SnapshotReport{} - mi := &file_ateapi_proto_msgTypes[125] + mi := &file_ateapi_proto_msgTypes[129] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9029,7 +9272,7 @@ func (x *SnapshotReport) String() string { func (*SnapshotReport) ProtoMessage() {} func (x *SnapshotReport) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[125] + mi := &file_ateapi_proto_msgTypes[129] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9042,7 +9285,7 @@ func (x *SnapshotReport) ProtoReflect() protoreflect.Message { // Deprecated: Use SnapshotReport.ProtoReflect.Descriptor instead. func (*SnapshotReport) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{125} + return file_ateapi_proto_rawDescGZIP(), []int{129} } func (x *SnapshotReport) GetUuid() string { @@ -9112,7 +9355,7 @@ type AccessPolicy struct { func (x *AccessPolicy) Reset() { *x = AccessPolicy{} - mi := &file_ateapi_proto_msgTypes[126] + mi := &file_ateapi_proto_msgTypes[130] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9124,7 +9367,7 @@ func (x *AccessPolicy) String() string { func (*AccessPolicy) ProtoMessage() {} func (x *AccessPolicy) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[126] + mi := &file_ateapi_proto_msgTypes[130] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9137,7 +9380,7 @@ func (x *AccessPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use AccessPolicy.ProtoReflect.Descriptor instead. func (*AccessPolicy) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{126} + return file_ateapi_proto_rawDescGZIP(), []int{130} } func (x *AccessPolicy) GetMetadata() *ResourceMetadata { @@ -9179,7 +9422,7 @@ type Binding struct { func (x *Binding) Reset() { *x = Binding{} - mi := &file_ateapi_proto_msgTypes[127] + mi := &file_ateapi_proto_msgTypes[131] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9191,7 +9434,7 @@ func (x *Binding) String() string { func (*Binding) ProtoMessage() {} func (x *Binding) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[127] + mi := &file_ateapi_proto_msgTypes[131] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9204,7 +9447,7 @@ func (x *Binding) ProtoReflect() protoreflect.Message { // Deprecated: Use Binding.ProtoReflect.Descriptor instead. func (*Binding) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{127} + return file_ateapi_proto_rawDescGZIP(), []int{131} } func (x *Binding) GetRole() string { @@ -9236,7 +9479,7 @@ type GetGlobalAccessPolicyRequest struct { func (x *GetGlobalAccessPolicyRequest) Reset() { *x = GetGlobalAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[128] + mi := &file_ateapi_proto_msgTypes[132] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9248,7 +9491,7 @@ func (x *GetGlobalAccessPolicyRequest) String() string { func (*GetGlobalAccessPolicyRequest) ProtoMessage() {} func (x *GetGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[128] + mi := &file_ateapi_proto_msgTypes[132] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9261,7 +9504,7 @@ func (x *GetGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*GetGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{128} + return file_ateapi_proto_rawDescGZIP(), []int{132} } func (x *GetGlobalAccessPolicyRequest) GetName() string { @@ -9285,7 +9528,7 @@ type CreateGlobalAccessPolicyRequest struct { func (x *CreateGlobalAccessPolicyRequest) Reset() { *x = CreateGlobalAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[129] + mi := &file_ateapi_proto_msgTypes[133] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9297,7 +9540,7 @@ func (x *CreateGlobalAccessPolicyRequest) String() string { func (*CreateGlobalAccessPolicyRequest) ProtoMessage() {} func (x *CreateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[129] + mi := &file_ateapi_proto_msgTypes[133] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9310,7 +9553,7 @@ func (x *CreateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{129} + return file_ateapi_proto_rawDescGZIP(), []int{133} } func (x *CreateGlobalAccessPolicyRequest) GetAccessPolicy() *AccessPolicy { @@ -9334,7 +9577,7 @@ type UpdateGlobalAccessPolicyRequest struct { func (x *UpdateGlobalAccessPolicyRequest) Reset() { *x = UpdateGlobalAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[130] + mi := &file_ateapi_proto_msgTypes[134] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9346,7 +9589,7 @@ func (x *UpdateGlobalAccessPolicyRequest) String() string { func (*UpdateGlobalAccessPolicyRequest) ProtoMessage() {} func (x *UpdateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[130] + mi := &file_ateapi_proto_msgTypes[134] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9359,7 +9602,7 @@ func (x *UpdateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{130} + return file_ateapi_proto_rawDescGZIP(), []int{134} } func (x *UpdateGlobalAccessPolicyRequest) GetAccessPolicy() *AccessPolicy { @@ -9389,7 +9632,7 @@ type GetAtespaceAccessPolicyRequest struct { func (x *GetAtespaceAccessPolicyRequest) Reset() { *x = GetAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[131] + mi := &file_ateapi_proto_msgTypes[135] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9401,7 +9644,7 @@ func (x *GetAtespaceAccessPolicyRequest) String() string { func (*GetAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *GetAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[131] + mi := &file_ateapi_proto_msgTypes[135] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9414,7 +9657,7 @@ func (x *GetAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*GetAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{131} + return file_ateapi_proto_rawDescGZIP(), []int{135} } func (x *GetAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -9450,7 +9693,7 @@ type CreateAtespaceAccessPolicyRequest struct { func (x *CreateAtespaceAccessPolicyRequest) Reset() { *x = CreateAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[132] + mi := &file_ateapi_proto_msgTypes[136] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9462,7 +9705,7 @@ func (x *CreateAtespaceAccessPolicyRequest) String() string { func (*CreateAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *CreateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[132] + mi := &file_ateapi_proto_msgTypes[136] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9475,7 +9718,7 @@ func (x *CreateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use CreateAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{132} + return file_ateapi_proto_rawDescGZIP(), []int{136} } func (x *CreateAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -9511,7 +9754,7 @@ type UpdateAtespaceAccessPolicyRequest struct { func (x *UpdateAtespaceAccessPolicyRequest) Reset() { *x = UpdateAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[133] + mi := &file_ateapi_proto_msgTypes[137] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9523,7 +9766,7 @@ func (x *UpdateAtespaceAccessPolicyRequest) String() string { func (*UpdateAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *UpdateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[133] + mi := &file_ateapi_proto_msgTypes[137] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9536,7 +9779,7 @@ func (x *UpdateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use UpdateAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{133} + return file_ateapi_proto_rawDescGZIP(), []int{137} } func (x *UpdateAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -9577,7 +9820,7 @@ type DeleteAtespaceAccessPolicyRequest struct { func (x *DeleteAtespaceAccessPolicyRequest) Reset() { *x = DeleteAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[134] + mi := &file_ateapi_proto_msgTypes[138] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9589,7 +9832,7 @@ func (x *DeleteAtespaceAccessPolicyRequest) String() string { func (*DeleteAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *DeleteAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[134] + mi := &file_ateapi_proto_msgTypes[138] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9602,7 +9845,7 @@ func (x *DeleteAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use DeleteAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*DeleteAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{134} + return file_ateapi_proto_rawDescGZIP(), []int{138} } func (x *DeleteAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -9648,7 +9891,7 @@ type ResourceAnnotation struct { func (x *ResourceAnnotation) Reset() { *x = ResourceAnnotation{} - mi := &file_ateapi_proto_msgTypes[135] + mi := &file_ateapi_proto_msgTypes[139] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9660,7 +9903,7 @@ func (x *ResourceAnnotation) String() string { func (*ResourceAnnotation) ProtoMessage() {} func (x *ResourceAnnotation) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[135] + mi := &file_ateapi_proto_msgTypes[139] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9673,7 +9916,7 @@ func (x *ResourceAnnotation) ProtoReflect() protoreflect.Message { // Deprecated: Use ResourceAnnotation.ProtoReflect.Descriptor instead. func (*ResourceAnnotation) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{135} + return file_ateapi_proto_rawDescGZIP(), []int{139} } func (x *ResourceAnnotation) GetParents() []string { @@ -9716,7 +9959,7 @@ type MethodAnnotation struct { func (x *MethodAnnotation) Reset() { *x = MethodAnnotation{} - mi := &file_ateapi_proto_msgTypes[136] + mi := &file_ateapi_proto_msgTypes[140] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9728,7 +9971,7 @@ func (x *MethodAnnotation) String() string { func (*MethodAnnotation) ProtoMessage() {} func (x *MethodAnnotation) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[136] + mi := &file_ateapi_proto_msgTypes[140] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9741,7 +9984,7 @@ func (x *MethodAnnotation) ProtoReflect() protoreflect.Message { // Deprecated: Use MethodAnnotation.ProtoReflect.Descriptor instead. func (*MethodAnnotation) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{136} + return file_ateapi_proto_rawDescGZIP(), []int{140} } func (x *MethodAnnotation) GetResource() string { @@ -10137,7 +10380,7 @@ const file_ateapi_proto_rawDesc = "" + "\aoptions\x18\x02 \x01(\v2\x13.ateapi.ListOptionsR\aoptions\"c\n" + "\x12ListActorsResponse\x12%\n" + "\x06actors\x18\x01 \x03(\v2\r.ateapi.ActorR\x06actors\x12&\n" + - "\x0fnext_page_token\x18\x02 \x01(\tR\rnextPageToken\"\xec\x03\n" + + "\x0fnext_page_token\x18\x02 \x01(\tR\rnextPageToken\"\xc4\x03\n" + "\x06Worker\x124\n" + "\bmetadata\x18\x01 \x01(\v2\x18.ateapi.ResourceMetadataR\bmetadata\x12)\n" + "\x10worker_namespace\x18\x02 \x01(\tR\x0fworkerNamespace\x12\x1f\n" + @@ -10146,22 +10389,22 @@ const file_ateapi_proto_rawDesc = "" + "\n" + "worker_pod\x18\x04 \x01(\tR\tworkerPod\x12$\n" + "\x0eworker_pod_uid\x18\x05 \x01(\tR\fworkerPodUid\x12\x1b\n" + - "\tnode_name\x18\x06 \x01(\tR\bnodeName\x12\x10\n" + - "\x03ips\x18\a \x03(\tR\x03ips\x12(\n" + - "\x10worker_spiffe_id\x18\b \x01(\tR\x0eworkerSpiffeId\x122\n" + - "\x06labels\x18\t \x03(\v2\x1a.ateapi.Worker.LabelsEntryR\x06labels\x12\x14\n" + - "\x05epoch\x18\n" + - " \x01(\x03R\x05epoch\x12,\n" + - "\x06status\x18\v \x01(\v2\x14.ateapi.WorkerStatusR\x06status\x1a9\n" + + "\tnode_name\x18\x06 \x01(\tR\bnodeName\x12(\n" + + "\x10worker_spiffe_id\x18\a \x01(\tR\x0eworkerSpiffeId\x122\n" + + "\x06labels\x18\b \x03(\v2\x1a.ateapi.Worker.LabelsEntryR\x06labels\x12,\n" + + "\x06status\x18\t \x01(\v2\x14.ateapi.WorkerStatusR\x06status\x1a9\n" + "\vLabelsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01:\x0f\x82\xb5\x18\v\x18\x02\"\aWorkers\"\x91\x02\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01:\x0f\x82\xb5\x18\v\x18\x02\"\aWorkers\"\xf1\x02\n" + "\fWorkerStatus\x12)\n" + "\x05state\x18\x01 \x01(\x0e2\x13.ateapi.WorkerStateR\x05state\x123\n" + "\bcapacity\x18\x02 \x01(\v2\x17.ateapi.WorkerResourcesR\bcapacity\x125\n" + "\tallocated\x18\x03 \x01(\v2\x17.ateapi.WorkerResourcesR\tallocated\x12%\n" + "\x0eobserved_epoch\x18\x04 \x01(\x03R\robservedEpoch\x12C\n" + - "\x0fsandbox_classes\x18\x05 \x03(\v2\x1a.ateapi.WorkerSandboxClassR\x0esandboxClasses\"\xcf\x01\n" + + "\x0fsandbox_classes\x18\x05 \x03(\v2\x1a.ateapi.WorkerSandboxClassR\x0esandboxClasses\x12\x10\n" + + "\x03ips\x18\x06 \x03(\tR\x03ips\x12!\n" + + "\fissued_epoch\x18\a \x01(\x03R\vissuedEpoch\x12)\n" + + "\x10registered_epoch\x18\b \x01(\x03R\x0fregisteredEpoch\"\xcf\x01\n" + "\x12WorkerSandboxClass\x12#\n" + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\x12E\n" + "\x0fdefault_runtime\x18\x02 \x01(\v2\x1c.ateapi.WorkerSandboxRuntimeR\x0edefaultRuntime\x12M\n" + @@ -10182,19 +10425,29 @@ const file_ateapi_proto_rawDesc = "" + "\x0fWorkerResources\x12%\n" + "\x0ecpu_millicores\x18\x01 \x01(\x03R\rcpuMillicores\x12!\n" + "\fmemory_bytes\x18\x02 \x01(\x03R\vmemoryBytes\x12\x16\n" + - "\x06actors\x18\x03 \x01(\x05R\x06actors\"\x81\x02\n" + + "\x06actors\x18\x03 \x01(\x05R\x06actors\"\xde\x01\n" + "\x0fActorAssignment\x124\n" + "\bmetadata\x18\x01 \x01(\v2\x18.ateapi.ResourceMetadataR\bmetadata\x12'\n" + "\x05actor\x18\x02 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12\x1b\n" + "\tactor_uid\x18\x03 \x01(\tR\bactorUid\x12/\n" + - "\tresources\x18\x04 \x01(\v2\x11.ateapi.ResourcesR\tresources\x12!\n" + - "\fworker_epoch\x18\x05 \x01(\x03R\vworkerEpoch:\x1e\x82\xb5\x18\x1a\n" + - "\x06Worker\"\x10ActorAssignments\"\xbc\x01\n" + + "\tresources\x18\x04 \x01(\v2\x11.ateapi.ResourcesR\tresources:\x1e\x82\xb5\x18\x1a\n" + + "\x06Worker\"\x10ActorAssignments\"F\n" + + "\x19AcquireWorkerEpochRequest\x12)\n" + + "\x06worker\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x06worker\"2\n" + + "\x1aAcquireWorkerEpochResponse\x12\x14\n" + + "\x05epoch\x18\x01 \x01(\x03R\x05epoch\"\x9f\x01\n" + "\x15RegisterWorkerRequest\x12)\n" + - "\x06worker\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x06worker\x123\n" + - "\bcapacity\x18\x02 \x01(\v2\x17.ateapi.WorkerResourcesR\bcapacity\x12C\n" + - "\x0fsandbox_classes\x18\x03 \x03(\v2\x1a.ateapi.WorkerSandboxClassR\x0esandboxClasses\"@\n" + + "\x06worker\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x06worker\x12\x14\n" + + "\x05epoch\x18\x03 \x01(\x03R\x05epoch\x12\x10\n" + + "\x03ips\x18\x04 \x03(\tR\x03ips\x123\n" + + "\bcapacity\x18\x02 \x01(\v2\x17.ateapi.WorkerResourcesR\bcapacity\"@\n" + "\x16RegisterWorkerResponse\x12&\n" + + "\x06worker\x18\x01 \x01(\v2\x0e.ateapi.WorkerR\x06worker\"\xa7\x01\n" + + "\x1fSetWorkerSandboxRuntimesRequest\x12)\n" + + "\x06worker\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x06worker\x12\x14\n" + + "\x05epoch\x18\x02 \x01(\x03R\x05epoch\x12C\n" + + "\x0fsandbox_classes\x18\x03 \x03(\v2\x1a.ateapi.WorkerSandboxClassR\x0esandboxClasses\"J\n" + + " SetWorkerSandboxRuntimesResponse\x12&\n" + "\x06worker\x18\x01 \x01(\v2\x0e.ateapi.WorkerR\x06worker\"\xd4\x01\n" + "!MintWorkerActorCertificateRequest\x12'\n" + "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12\x1b\n" + @@ -10417,9 +10670,11 @@ const file_ateapi_proto_rawDesc = "" + "\x1aUpdateAtespaceAccessPolicy\x12).ateapi.UpdateAtespaceAccessPolicyRequest\x1a\x14.ateapi.AccessPolicy\"\x12\x82\xb5\x18\x0e\n" + "\fAccessPolicy\x12q\n" + "\x1aDeleteAtespaceAccessPolicy\x12).ateapi.DeleteAtespaceAccessPolicyRequest\x1a\x14.ateapi.AccessPolicy\"\x12\x82\xb5\x18\x0e\n" + - "\fAccessPolicy2\xde\x04\n" + - "\rWorkerService\x12O\n" + - "\x0eRegisterWorker\x12\x1d.ateapi.RegisterWorkerRequest\x1a\x1e.ateapi.RegisterWorkerResponse\x12s\n" + + "\fAccessPolicy2\xaa\x06\n" + + "\rWorkerService\x12[\n" + + "\x12AcquireWorkerEpoch\x12!.ateapi.AcquireWorkerEpochRequest\x1a\".ateapi.AcquireWorkerEpochResponse\x12O\n" + + "\x0eRegisterWorker\x12\x1d.ateapi.RegisterWorkerRequest\x1a\x1e.ateapi.RegisterWorkerResponse\x12m\n" + + "\x18SetWorkerSandboxRuntimes\x12'.ateapi.SetWorkerSandboxRuntimesRequest\x1a(.ateapi.SetWorkerSandboxRuntimesResponse\x12s\n" + "\x1aMintWorkerActorCertificate\x12).ateapi.MintWorkerActorCertificateRequest\x1a*.ateapi.MintWorkerActorCertificateResponse\x12[\n" + "\x12MintWorkerActorJwt\x12!.ateapi.MintWorkerActorJwtRequest\x1a\".ateapi.MintWorkerActorJwtResponse\x12^\n" + "\x13RequestActorSuspend\x12\".ateapi.RequestActorSuspendRequest\x1a#.ateapi.RequestActorSuspendResponse\x12m\n" + @@ -10441,7 +10696,7 @@ func file_ateapi_proto_rawDescGZIP() []byte { } var file_ateapi_proto_enumTypes = make([]protoimpl.EnumInfo, 10) -var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 140) +var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 144) var file_ateapi_proto_goTypes = []any{ (SnapshotOwner)(0), // 0: ateapi.SnapshotOwner (SnapshotStorageStatus)(0), // 1: ateapi.SnapshotStorageStatus @@ -10565,37 +10820,41 @@ var file_ateapi_proto_goTypes = []any{ (*AttributeEntry)(nil), // 119: ateapi.AttributeEntry (*WorkerResources)(nil), // 120: ateapi.WorkerResources (*ActorAssignment)(nil), // 121: ateapi.ActorAssignment - (*RegisterWorkerRequest)(nil), // 122: ateapi.RegisterWorkerRequest - (*RegisterWorkerResponse)(nil), // 123: ateapi.RegisterWorkerResponse - (*MintWorkerActorCertificateRequest)(nil), // 124: ateapi.MintWorkerActorCertificateRequest - (*MintWorkerActorCertificateResponse)(nil), // 125: ateapi.MintWorkerActorCertificateResponse - (*MintWorkerActorJwtRequest)(nil), // 126: ateapi.MintWorkerActorJwtRequest - (*MintWorkerActorJwtResponse)(nil), // 127: ateapi.MintWorkerActorJwtResponse - (*RequestActorSuspendRequest)(nil), // 128: ateapi.RequestActorSuspendRequest - (*RequestActorSuspendResponse)(nil), // 129: ateapi.RequestActorSuspendResponse - (*ReportActorCrashedRequest)(nil), // 130: ateapi.ReportActorCrashedRequest - (*ReportActorCrashedResponse)(nil), // 131: ateapi.ReportActorCrashedResponse - (*ReportActorSnapshotStateRequest)(nil), // 132: ateapi.ReportActorSnapshotStateRequest - (*ReportActorSnapshotStateResponse)(nil), // 133: ateapi.ReportActorSnapshotStateResponse - (*ActorSnapshotState)(nil), // 134: ateapi.ActorSnapshotState - (*SnapshotReport)(nil), // 135: ateapi.SnapshotReport - (*AccessPolicy)(nil), // 136: ateapi.AccessPolicy - (*Binding)(nil), // 137: ateapi.Binding - (*GetGlobalAccessPolicyRequest)(nil), // 138: ateapi.GetGlobalAccessPolicyRequest - (*CreateGlobalAccessPolicyRequest)(nil), // 139: ateapi.CreateGlobalAccessPolicyRequest - (*UpdateGlobalAccessPolicyRequest)(nil), // 140: ateapi.UpdateGlobalAccessPolicyRequest - (*GetAtespaceAccessPolicyRequest)(nil), // 141: ateapi.GetAtespaceAccessPolicyRequest - (*CreateAtespaceAccessPolicyRequest)(nil), // 142: ateapi.CreateAtespaceAccessPolicyRequest - (*UpdateAtespaceAccessPolicyRequest)(nil), // 143: ateapi.UpdateAtespaceAccessPolicyRequest - (*DeleteAtespaceAccessPolicyRequest)(nil), // 144: ateapi.DeleteAtespaceAccessPolicyRequest - (*ResourceAnnotation)(nil), // 145: ateapi.ResourceAnnotation - (*MethodAnnotation)(nil), // 146: ateapi.MethodAnnotation - nil, // 147: ateapi.Selector.MatchLabelsEntry - nil, // 148: ateapi.ExternalVolumeStatus.VolumeContextEntry - nil, // 149: ateapi.Worker.LabelsEntry - (*timestamppb.Timestamp)(nil), // 150: google.protobuf.Timestamp - (*descriptorpb.MessageOptions)(nil), // 151: google.protobuf.MessageOptions - (*descriptorpb.MethodOptions)(nil), // 152: google.protobuf.MethodOptions + (*AcquireWorkerEpochRequest)(nil), // 122: ateapi.AcquireWorkerEpochRequest + (*AcquireWorkerEpochResponse)(nil), // 123: ateapi.AcquireWorkerEpochResponse + (*RegisterWorkerRequest)(nil), // 124: ateapi.RegisterWorkerRequest + (*RegisterWorkerResponse)(nil), // 125: ateapi.RegisterWorkerResponse + (*SetWorkerSandboxRuntimesRequest)(nil), // 126: ateapi.SetWorkerSandboxRuntimesRequest + (*SetWorkerSandboxRuntimesResponse)(nil), // 127: ateapi.SetWorkerSandboxRuntimesResponse + (*MintWorkerActorCertificateRequest)(nil), // 128: ateapi.MintWorkerActorCertificateRequest + (*MintWorkerActorCertificateResponse)(nil), // 129: ateapi.MintWorkerActorCertificateResponse + (*MintWorkerActorJwtRequest)(nil), // 130: ateapi.MintWorkerActorJwtRequest + (*MintWorkerActorJwtResponse)(nil), // 131: ateapi.MintWorkerActorJwtResponse + (*RequestActorSuspendRequest)(nil), // 132: ateapi.RequestActorSuspendRequest + (*RequestActorSuspendResponse)(nil), // 133: ateapi.RequestActorSuspendResponse + (*ReportActorCrashedRequest)(nil), // 134: ateapi.ReportActorCrashedRequest + (*ReportActorCrashedResponse)(nil), // 135: ateapi.ReportActorCrashedResponse + (*ReportActorSnapshotStateRequest)(nil), // 136: ateapi.ReportActorSnapshotStateRequest + (*ReportActorSnapshotStateResponse)(nil), // 137: ateapi.ReportActorSnapshotStateResponse + (*ActorSnapshotState)(nil), // 138: ateapi.ActorSnapshotState + (*SnapshotReport)(nil), // 139: ateapi.SnapshotReport + (*AccessPolicy)(nil), // 140: ateapi.AccessPolicy + (*Binding)(nil), // 141: ateapi.Binding + (*GetGlobalAccessPolicyRequest)(nil), // 142: ateapi.GetGlobalAccessPolicyRequest + (*CreateGlobalAccessPolicyRequest)(nil), // 143: ateapi.CreateGlobalAccessPolicyRequest + (*UpdateGlobalAccessPolicyRequest)(nil), // 144: ateapi.UpdateGlobalAccessPolicyRequest + (*GetAtespaceAccessPolicyRequest)(nil), // 145: ateapi.GetAtespaceAccessPolicyRequest + (*CreateAtespaceAccessPolicyRequest)(nil), // 146: ateapi.CreateAtespaceAccessPolicyRequest + (*UpdateAtespaceAccessPolicyRequest)(nil), // 147: ateapi.UpdateAtespaceAccessPolicyRequest + (*DeleteAtespaceAccessPolicyRequest)(nil), // 148: ateapi.DeleteAtespaceAccessPolicyRequest + (*ResourceAnnotation)(nil), // 149: ateapi.ResourceAnnotation + (*MethodAnnotation)(nil), // 150: ateapi.MethodAnnotation + nil, // 151: ateapi.Selector.MatchLabelsEntry + nil, // 152: ateapi.ExternalVolumeStatus.VolumeContextEntry + nil, // 153: ateapi.Worker.LabelsEntry + (*timestamppb.Timestamp)(nil), // 154: google.protobuf.Timestamp + (*descriptorpb.MessageOptions)(nil), // 155: google.protobuf.MessageOptions + (*descriptorpb.MethodOptions)(nil), // 156: google.protobuf.MethodOptions } var file_ateapi_proto_depIdxs = []int32{ 1, // 0: ateapi.SnapshotStorage.status:type_name -> ateapi.SnapshotStorageStatus @@ -10604,12 +10863,12 @@ var file_ateapi_proto_depIdxs = []int32{ 2, // 3: ateapi.Snapshot.fidelity:type_name -> ateapi.SnapshotFidelity 11, // 4: ateapi.Snapshot.durable_snapshot:type_name -> ateapi.SnapshotStorage 117, // 5: ateapi.Snapshot.sandbox_runtime:type_name -> ateapi.ActorSandboxRuntime - 147, // 6: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry - 150, // 7: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp - 150, // 8: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp - 150, // 9: ateapi.ResourceMetadata.delete_time:type_name -> google.protobuf.Timestamp + 151, // 6: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry + 154, // 7: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp + 154, // 8: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp + 154, // 9: ateapi.ResourceMetadata.delete_time:type_name -> google.protobuf.Timestamp 16, // 10: ateapi.ExternalVolumes.volumes:type_name -> ateapi.ExternalVolumeStatus - 148, // 11: ateapi.ExternalVolumeStatus.volume_context:type_name -> ateapi.ExternalVolumeStatus.VolumeContextEntry + 152, // 11: ateapi.ExternalVolumeStatus.volume_context:type_name -> ateapi.ExternalVolumeStatus.VolumeContextEntry 14, // 12: ateapi.Actor.metadata:type_name -> ateapi.ResourceMetadata 35, // 13: ateapi.Actor.actor_template:type_name -> ateapi.ObjectRef 13, // 14: ateapi.Actor.worker_selector:type_name -> ateapi.Selector @@ -10634,7 +10893,7 @@ var file_ateapi_proto_depIdxs = []int32{ 15, // 33: ateapi.ActorStatus.external_volumes:type_name -> ateapi.ExternalVolumes 30, // 34: ateapi.ActorStatus.crash:type_name -> ateapi.ActorCrash 12, // 35: ateapi.ActorStatus.snapshots:type_name -> ateapi.Snapshot - 150, // 36: ateapi.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp + 154, // 36: ateapi.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp 35, // 37: ateapi.WorkerAssignment.worker:type_name -> ateapi.ObjectRef 117, // 38: ateapi.WorkerAssignment.sandbox_runtime:type_name -> ateapi.ActorSandboxRuntime 12, // 39: ateapi.TagStatus.snapshot:type_name -> ateapi.Snapshot @@ -10653,7 +10912,7 @@ var file_ateapi_proto_depIdxs = []int32{ 40, // 52: ateapi.ActorTemplate.status:type_name -> ateapi.ActorTemplateStatus 38, // 53: ateapi.ResourceRequirements.limits:type_name -> ateapi.Resources 35, // 54: ateapi.GoldenSnapshotStatus.golden_tag:type_name -> ateapi.ObjectRef - 150, // 55: ateapi.GoldenSnapshotStatus.snapshot_time:type_name -> google.protobuf.Timestamp + 154, // 55: ateapi.GoldenSnapshotStatus.snapshot_time:type_name -> google.protobuf.Timestamp 39, // 56: ateapi.ActorTemplateStatus.golden_snapshot_status:type_name -> ateapi.GoldenSnapshotStatus 2, // 57: ateapi.SnapshotConfig.preferred_fidelity:type_name -> ateapi.SnapshotFidelity 42, // 58: ateapi.SnapshotConfig.golden_snapshot_config:type_name -> ateapi.GoldenSnapshotConfig @@ -10709,7 +10968,7 @@ var file_ateapi_proto_depIdxs = []int32{ 95, // 108: ateapi.DeleteActorEgressPolicyRequest.options:type_name -> ateapi.DeleteOptions 35, // 109: ateapi.GetTagRequest.tag:type_name -> ateapi.ObjectRef 35, // 110: ateapi.MintActorJwtRequest.actor:type_name -> ateapi.ObjectRef - 150, // 111: ateapi.MintActorJwtResponse.expire_time:type_name -> google.protobuf.Timestamp + 154, // 111: ateapi.MintActorJwtResponse.expire_time:type_name -> google.protobuf.Timestamp 35, // 112: ateapi.MintActorCertificateRequest.actor:type_name -> ateapi.ObjectRef 96, // 113: ateapi.ListTagsRequest.options:type_name -> ateapi.ListOptions 33, // 114: ateapi.ListTagsResponse.tags:type_name -> ateapi.Tag @@ -10736,7 +10995,7 @@ var file_ateapi_proto_depIdxs = []int32{ 96, // 135: ateapi.ListActorsRequest.options:type_name -> ateapi.ListOptions 17, // 136: ateapi.ListActorsResponse.actors:type_name -> ateapi.Actor 14, // 137: ateapi.Worker.metadata:type_name -> ateapi.ResourceMetadata - 149, // 138: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry + 153, // 138: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry 114, // 139: ateapi.Worker.status:type_name -> ateapi.WorkerStatus 8, // 140: ateapi.WorkerStatus.state:type_name -> ateapi.WorkerState 120, // 141: ateapi.WorkerStatus.capacity:type_name -> ateapi.WorkerResources @@ -10750,149 +11009,156 @@ var file_ateapi_proto_depIdxs = []int32{ 14, // 149: ateapi.ActorAssignment.metadata:type_name -> ateapi.ResourceMetadata 35, // 150: ateapi.ActorAssignment.actor:type_name -> ateapi.ObjectRef 38, // 151: ateapi.ActorAssignment.resources:type_name -> ateapi.Resources - 35, // 152: ateapi.RegisterWorkerRequest.worker:type_name -> ateapi.ObjectRef - 120, // 153: ateapi.RegisterWorkerRequest.capacity:type_name -> ateapi.WorkerResources - 115, // 154: ateapi.RegisterWorkerRequest.sandbox_classes:type_name -> ateapi.WorkerSandboxClass + 35, // 152: ateapi.AcquireWorkerEpochRequest.worker:type_name -> ateapi.ObjectRef + 35, // 153: ateapi.RegisterWorkerRequest.worker:type_name -> ateapi.ObjectRef + 120, // 154: ateapi.RegisterWorkerRequest.capacity:type_name -> ateapi.WorkerResources 113, // 155: ateapi.RegisterWorkerResponse.worker:type_name -> ateapi.Worker - 35, // 156: ateapi.MintWorkerActorCertificateRequest.actor:type_name -> ateapi.ObjectRef - 35, // 157: ateapi.MintWorkerActorCertificateRequest.worker:type_name -> ateapi.ObjectRef - 35, // 158: ateapi.MintWorkerActorJwtRequest.actor:type_name -> ateapi.ObjectRef - 150, // 159: ateapi.MintWorkerActorJwtResponse.expire_time:type_name -> google.protobuf.Timestamp - 35, // 160: ateapi.RequestActorSuspendRequest.worker:type_name -> ateapi.ObjectRef - 35, // 161: ateapi.RequestActorSuspendRequest.actor:type_name -> ateapi.ObjectRef - 17, // 162: ateapi.RequestActorSuspendResponse.actor:type_name -> ateapi.Actor - 35, // 163: ateapi.ReportActorCrashedRequest.worker:type_name -> ateapi.ObjectRef - 35, // 164: ateapi.ReportActorCrashedRequest.actor:type_name -> ateapi.ObjectRef - 17, // 165: ateapi.ReportActorCrashedResponse.actor:type_name -> ateapi.Actor - 35, // 166: ateapi.ReportActorSnapshotStateRequest.actor:type_name -> ateapi.ObjectRef - 134, // 167: ateapi.ReportActorSnapshotStateRequest.state:type_name -> ateapi.ActorSnapshotState - 135, // 168: ateapi.ActorSnapshotState.latest_local:type_name -> ateapi.SnapshotReport - 135, // 169: ateapi.ActorSnapshotState.uploading:type_name -> ateapi.SnapshotReport - 135, // 170: ateapi.ActorSnapshotState.latest_durable:type_name -> ateapi.SnapshotReport - 150, // 171: ateapi.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp - 2, // 172: ateapi.SnapshotReport.fidelity:type_name -> ateapi.SnapshotFidelity - 10, // 173: ateapi.SnapshotReport.object:type_name -> ateapi.ObjectSnapshot - 150, // 174: ateapi.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp - 150, // 175: ateapi.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp - 14, // 176: ateapi.AccessPolicy.metadata:type_name -> ateapi.ResourceMetadata - 137, // 177: ateapi.AccessPolicy.bindings:type_name -> ateapi.Binding - 136, // 178: ateapi.CreateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 136, // 179: ateapi.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 35, // 180: ateapi.GetAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 35, // 181: ateapi.CreateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 136, // 182: ateapi.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 35, // 183: ateapi.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 136, // 184: ateapi.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 35, // 185: ateapi.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 95, // 186: ateapi.DeleteAtespaceAccessPolicyRequest.options:type_name -> ateapi.DeleteOptions - 9, // 187: ateapi.ResourceAnnotation.scope:type_name -> ateapi.Scope - 151, // 188: ateapi.resource:extendee -> google.protobuf.MessageOptions - 152, // 189: ateapi.method:extendee -> google.protobuf.MethodOptions - 145, // 190: ateapi.resource:type_name -> ateapi.ResourceAnnotation - 146, // 191: ateapi.method:type_name -> ateapi.MethodAnnotation - 70, // 192: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest - 71, // 193: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest - 72, // 194: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest - 73, // 195: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest - 76, // 196: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest - 78, // 197: ateapi.Control.RevertActor:input_type -> ateapi.RevertActorRequest - 80, // 198: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest - 81, // 199: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest - 82, // 200: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest - 83, // 201: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest - 84, // 202: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest - 86, // 203: ateapi.Control.MintActorJwt:input_type -> ateapi.MintActorJwtRequest - 88, // 204: ateapi.Control.MintActorCertificate:input_type -> ateapi.MintActorCertificateRequest - 92, // 205: ateapi.Control.CreateTag:input_type -> ateapi.CreateTagRequest - 85, // 206: ateapi.Control.GetTag:input_type -> ateapi.GetTagRequest - 90, // 207: ateapi.Control.ListTags:input_type -> ateapi.ListTagsRequest - 93, // 208: ateapi.Control.UpdateTag:input_type -> ateapi.UpdateTagRequest - 94, // 209: ateapi.Control.DeleteTag:input_type -> ateapi.DeleteTagRequest - 99, // 210: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest - 101, // 211: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest - 102, // 212: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest - 103, // 213: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest - 104, // 214: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest - 105, // 215: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest - 107, // 216: ateapi.Control.MarkWorkerAvailable:input_type -> ateapi.MarkWorkerAvailableRequest - 109, // 217: ateapi.Control.MarkWorkerUnavailable:input_type -> ateapi.MarkWorkerUnavailableRequest - 97, // 218: ateapi.Control.ListWorkerActorAssignments:input_type -> ateapi.ListWorkerActorAssignmentsRequest - 111, // 219: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest - 60, // 220: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest - 61, // 221: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest - 62, // 222: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest - 64, // 223: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest - 65, // 224: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest - 66, // 225: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest - 67, // 226: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest - 69, // 227: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest - 138, // 228: ateapi.Control.GetGlobalAccessPolicy:input_type -> ateapi.GetGlobalAccessPolicyRequest - 139, // 229: ateapi.Control.CreateGlobalAccessPolicy:input_type -> ateapi.CreateGlobalAccessPolicyRequest - 140, // 230: ateapi.Control.UpdateGlobalAccessPolicy:input_type -> ateapi.UpdateGlobalAccessPolicyRequest - 141, // 231: ateapi.Control.GetAtespaceAccessPolicy:input_type -> ateapi.GetAtespaceAccessPolicyRequest - 142, // 232: ateapi.Control.CreateAtespaceAccessPolicy:input_type -> ateapi.CreateAtespaceAccessPolicyRequest - 143, // 233: ateapi.Control.UpdateAtespaceAccessPolicy:input_type -> ateapi.UpdateAtespaceAccessPolicyRequest - 144, // 234: ateapi.Control.DeleteAtespaceAccessPolicy:input_type -> ateapi.DeleteAtespaceAccessPolicyRequest - 122, // 235: ateapi.WorkerService.RegisterWorker:input_type -> ateapi.RegisterWorkerRequest - 124, // 236: ateapi.WorkerService.MintWorkerActorCertificate:input_type -> ateapi.MintWorkerActorCertificateRequest - 126, // 237: ateapi.WorkerService.MintWorkerActorJwt:input_type -> ateapi.MintWorkerActorJwtRequest - 128, // 238: ateapi.WorkerService.RequestActorSuspend:input_type -> ateapi.RequestActorSuspendRequest - 132, // 239: ateapi.WorkerService.ReportActorSnapshotState:input_type -> ateapi.ReportActorSnapshotStateRequest - 130, // 240: ateapi.WorkerService.ReportActorCrashed:input_type -> ateapi.ReportActorCrashedRequest - 17, // 241: ateapi.Control.GetActor:output_type -> ateapi.Actor - 17, // 242: ateapi.Control.CreateActor:output_type -> ateapi.Actor - 17, // 243: ateapi.Control.UpdateActor:output_type -> ateapi.Actor - 74, // 244: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse - 77, // 245: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse - 79, // 246: ateapi.Control.RevertActor:output_type -> ateapi.RevertActorResponse - 17, // 247: ateapi.Control.DeleteActor:output_type -> ateapi.Actor - 18, // 248: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy - 18, // 249: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 18, // 250: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 18, // 251: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy - 87, // 252: ateapi.Control.MintActorJwt:output_type -> ateapi.MintActorJwtResponse - 89, // 253: ateapi.Control.MintActorCertificate:output_type -> ateapi.MintActorCertificateResponse - 33, // 254: ateapi.Control.CreateTag:output_type -> ateapi.Tag - 33, // 255: ateapi.Control.GetTag:output_type -> ateapi.Tag - 91, // 256: ateapi.Control.ListTags:output_type -> ateapi.ListTagsResponse - 33, // 257: ateapi.Control.UpdateTag:output_type -> ateapi.Tag - 33, // 258: ateapi.Control.DeleteTag:output_type -> ateapi.Tag - 100, // 259: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse - 113, // 260: ateapi.Control.GetWorker:output_type -> ateapi.Worker - 113, // 261: ateapi.Control.CreateWorker:output_type -> ateapi.Worker - 113, // 262: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker - 113, // 263: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker - 106, // 264: ateapi.Control.DrainWorker:output_type -> ateapi.DrainWorkerResponse - 108, // 265: ateapi.Control.MarkWorkerAvailable:output_type -> ateapi.MarkWorkerAvailableResponse - 110, // 266: ateapi.Control.MarkWorkerUnavailable:output_type -> ateapi.MarkWorkerUnavailableResponse - 98, // 267: ateapi.Control.ListWorkerActorAssignments:output_type -> ateapi.ListWorkerActorAssignmentsResponse - 112, // 268: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse - 34, // 269: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace - 34, // 270: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace - 63, // 271: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse - 34, // 272: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace - 36, // 273: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate - 36, // 274: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate - 68, // 275: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse - 36, // 276: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate - 136, // 277: ateapi.Control.GetGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 136, // 278: ateapi.Control.CreateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 136, // 279: ateapi.Control.UpdateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 136, // 280: ateapi.Control.GetAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 136, // 281: ateapi.Control.CreateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 136, // 282: ateapi.Control.UpdateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 136, // 283: ateapi.Control.DeleteAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 123, // 284: ateapi.WorkerService.RegisterWorker:output_type -> ateapi.RegisterWorkerResponse - 125, // 285: ateapi.WorkerService.MintWorkerActorCertificate:output_type -> ateapi.MintWorkerActorCertificateResponse - 127, // 286: ateapi.WorkerService.MintWorkerActorJwt:output_type -> ateapi.MintWorkerActorJwtResponse - 129, // 287: ateapi.WorkerService.RequestActorSuspend:output_type -> ateapi.RequestActorSuspendResponse - 133, // 288: ateapi.WorkerService.ReportActorSnapshotState:output_type -> ateapi.ReportActorSnapshotStateResponse - 131, // 289: ateapi.WorkerService.ReportActorCrashed:output_type -> ateapi.ReportActorCrashedResponse - 241, // [241:290] is the sub-list for method output_type - 192, // [192:241] is the sub-list for method input_type - 190, // [190:192] is the sub-list for extension type_name - 188, // [188:190] is the sub-list for extension extendee - 0, // [0:188] is the sub-list for field type_name + 35, // 156: ateapi.SetWorkerSandboxRuntimesRequest.worker:type_name -> ateapi.ObjectRef + 115, // 157: ateapi.SetWorkerSandboxRuntimesRequest.sandbox_classes:type_name -> ateapi.WorkerSandboxClass + 113, // 158: ateapi.SetWorkerSandboxRuntimesResponse.worker:type_name -> ateapi.Worker + 35, // 159: ateapi.MintWorkerActorCertificateRequest.actor:type_name -> ateapi.ObjectRef + 35, // 160: ateapi.MintWorkerActorCertificateRequest.worker:type_name -> ateapi.ObjectRef + 35, // 161: ateapi.MintWorkerActorJwtRequest.actor:type_name -> ateapi.ObjectRef + 154, // 162: ateapi.MintWorkerActorJwtResponse.expire_time:type_name -> google.protobuf.Timestamp + 35, // 163: ateapi.RequestActorSuspendRequest.worker:type_name -> ateapi.ObjectRef + 35, // 164: ateapi.RequestActorSuspendRequest.actor:type_name -> ateapi.ObjectRef + 17, // 165: ateapi.RequestActorSuspendResponse.actor:type_name -> ateapi.Actor + 35, // 166: ateapi.ReportActorCrashedRequest.worker:type_name -> ateapi.ObjectRef + 35, // 167: ateapi.ReportActorCrashedRequest.actor:type_name -> ateapi.ObjectRef + 17, // 168: ateapi.ReportActorCrashedResponse.actor:type_name -> ateapi.Actor + 35, // 169: ateapi.ReportActorSnapshotStateRequest.actor:type_name -> ateapi.ObjectRef + 138, // 170: ateapi.ReportActorSnapshotStateRequest.state:type_name -> ateapi.ActorSnapshotState + 139, // 171: ateapi.ActorSnapshotState.latest_local:type_name -> ateapi.SnapshotReport + 139, // 172: ateapi.ActorSnapshotState.uploading:type_name -> ateapi.SnapshotReport + 139, // 173: ateapi.ActorSnapshotState.latest_durable:type_name -> ateapi.SnapshotReport + 154, // 174: ateapi.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp + 2, // 175: ateapi.SnapshotReport.fidelity:type_name -> ateapi.SnapshotFidelity + 10, // 176: ateapi.SnapshotReport.object:type_name -> ateapi.ObjectSnapshot + 154, // 177: ateapi.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp + 154, // 178: ateapi.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp + 14, // 179: ateapi.AccessPolicy.metadata:type_name -> ateapi.ResourceMetadata + 141, // 180: ateapi.AccessPolicy.bindings:type_name -> ateapi.Binding + 140, // 181: ateapi.CreateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 140, // 182: ateapi.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 35, // 183: ateapi.GetAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 35, // 184: ateapi.CreateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 140, // 185: ateapi.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 35, // 186: ateapi.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 140, // 187: ateapi.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 35, // 188: ateapi.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 95, // 189: ateapi.DeleteAtespaceAccessPolicyRequest.options:type_name -> ateapi.DeleteOptions + 9, // 190: ateapi.ResourceAnnotation.scope:type_name -> ateapi.Scope + 155, // 191: ateapi.resource:extendee -> google.protobuf.MessageOptions + 156, // 192: ateapi.method:extendee -> google.protobuf.MethodOptions + 149, // 193: ateapi.resource:type_name -> ateapi.ResourceAnnotation + 150, // 194: ateapi.method:type_name -> ateapi.MethodAnnotation + 70, // 195: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest + 71, // 196: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest + 72, // 197: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest + 73, // 198: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest + 76, // 199: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest + 78, // 200: ateapi.Control.RevertActor:input_type -> ateapi.RevertActorRequest + 80, // 201: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest + 81, // 202: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest + 82, // 203: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest + 83, // 204: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest + 84, // 205: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest + 86, // 206: ateapi.Control.MintActorJwt:input_type -> ateapi.MintActorJwtRequest + 88, // 207: ateapi.Control.MintActorCertificate:input_type -> ateapi.MintActorCertificateRequest + 92, // 208: ateapi.Control.CreateTag:input_type -> ateapi.CreateTagRequest + 85, // 209: ateapi.Control.GetTag:input_type -> ateapi.GetTagRequest + 90, // 210: ateapi.Control.ListTags:input_type -> ateapi.ListTagsRequest + 93, // 211: ateapi.Control.UpdateTag:input_type -> ateapi.UpdateTagRequest + 94, // 212: ateapi.Control.DeleteTag:input_type -> ateapi.DeleteTagRequest + 99, // 213: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest + 101, // 214: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest + 102, // 215: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest + 103, // 216: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest + 104, // 217: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest + 105, // 218: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest + 107, // 219: ateapi.Control.MarkWorkerAvailable:input_type -> ateapi.MarkWorkerAvailableRequest + 109, // 220: ateapi.Control.MarkWorkerUnavailable:input_type -> ateapi.MarkWorkerUnavailableRequest + 97, // 221: ateapi.Control.ListWorkerActorAssignments:input_type -> ateapi.ListWorkerActorAssignmentsRequest + 111, // 222: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest + 60, // 223: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest + 61, // 224: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest + 62, // 225: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest + 64, // 226: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest + 65, // 227: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest + 66, // 228: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest + 67, // 229: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest + 69, // 230: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest + 142, // 231: ateapi.Control.GetGlobalAccessPolicy:input_type -> ateapi.GetGlobalAccessPolicyRequest + 143, // 232: ateapi.Control.CreateGlobalAccessPolicy:input_type -> ateapi.CreateGlobalAccessPolicyRequest + 144, // 233: ateapi.Control.UpdateGlobalAccessPolicy:input_type -> ateapi.UpdateGlobalAccessPolicyRequest + 145, // 234: ateapi.Control.GetAtespaceAccessPolicy:input_type -> ateapi.GetAtespaceAccessPolicyRequest + 146, // 235: ateapi.Control.CreateAtespaceAccessPolicy:input_type -> ateapi.CreateAtespaceAccessPolicyRequest + 147, // 236: ateapi.Control.UpdateAtespaceAccessPolicy:input_type -> ateapi.UpdateAtespaceAccessPolicyRequest + 148, // 237: ateapi.Control.DeleteAtespaceAccessPolicy:input_type -> ateapi.DeleteAtespaceAccessPolicyRequest + 122, // 238: ateapi.WorkerService.AcquireWorkerEpoch:input_type -> ateapi.AcquireWorkerEpochRequest + 124, // 239: ateapi.WorkerService.RegisterWorker:input_type -> ateapi.RegisterWorkerRequest + 126, // 240: ateapi.WorkerService.SetWorkerSandboxRuntimes:input_type -> ateapi.SetWorkerSandboxRuntimesRequest + 128, // 241: ateapi.WorkerService.MintWorkerActorCertificate:input_type -> ateapi.MintWorkerActorCertificateRequest + 130, // 242: ateapi.WorkerService.MintWorkerActorJwt:input_type -> ateapi.MintWorkerActorJwtRequest + 132, // 243: ateapi.WorkerService.RequestActorSuspend:input_type -> ateapi.RequestActorSuspendRequest + 136, // 244: ateapi.WorkerService.ReportActorSnapshotState:input_type -> ateapi.ReportActorSnapshotStateRequest + 134, // 245: ateapi.WorkerService.ReportActorCrashed:input_type -> ateapi.ReportActorCrashedRequest + 17, // 246: ateapi.Control.GetActor:output_type -> ateapi.Actor + 17, // 247: ateapi.Control.CreateActor:output_type -> ateapi.Actor + 17, // 248: ateapi.Control.UpdateActor:output_type -> ateapi.Actor + 74, // 249: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse + 77, // 250: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse + 79, // 251: ateapi.Control.RevertActor:output_type -> ateapi.RevertActorResponse + 17, // 252: ateapi.Control.DeleteActor:output_type -> ateapi.Actor + 18, // 253: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy + 18, // 254: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 18, // 255: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 18, // 256: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy + 87, // 257: ateapi.Control.MintActorJwt:output_type -> ateapi.MintActorJwtResponse + 89, // 258: ateapi.Control.MintActorCertificate:output_type -> ateapi.MintActorCertificateResponse + 33, // 259: ateapi.Control.CreateTag:output_type -> ateapi.Tag + 33, // 260: ateapi.Control.GetTag:output_type -> ateapi.Tag + 91, // 261: ateapi.Control.ListTags:output_type -> ateapi.ListTagsResponse + 33, // 262: ateapi.Control.UpdateTag:output_type -> ateapi.Tag + 33, // 263: ateapi.Control.DeleteTag:output_type -> ateapi.Tag + 100, // 264: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse + 113, // 265: ateapi.Control.GetWorker:output_type -> ateapi.Worker + 113, // 266: ateapi.Control.CreateWorker:output_type -> ateapi.Worker + 113, // 267: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker + 113, // 268: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker + 106, // 269: ateapi.Control.DrainWorker:output_type -> ateapi.DrainWorkerResponse + 108, // 270: ateapi.Control.MarkWorkerAvailable:output_type -> ateapi.MarkWorkerAvailableResponse + 110, // 271: ateapi.Control.MarkWorkerUnavailable:output_type -> ateapi.MarkWorkerUnavailableResponse + 98, // 272: ateapi.Control.ListWorkerActorAssignments:output_type -> ateapi.ListWorkerActorAssignmentsResponse + 112, // 273: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse + 34, // 274: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace + 34, // 275: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace + 63, // 276: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse + 34, // 277: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace + 36, // 278: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate + 36, // 279: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate + 68, // 280: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse + 36, // 281: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate + 140, // 282: ateapi.Control.GetGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 140, // 283: ateapi.Control.CreateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 140, // 284: ateapi.Control.UpdateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 140, // 285: ateapi.Control.GetAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 140, // 286: ateapi.Control.CreateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 140, // 287: ateapi.Control.UpdateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 140, // 288: ateapi.Control.DeleteAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 123, // 289: ateapi.WorkerService.AcquireWorkerEpoch:output_type -> ateapi.AcquireWorkerEpochResponse + 125, // 290: ateapi.WorkerService.RegisterWorker:output_type -> ateapi.RegisterWorkerResponse + 127, // 291: ateapi.WorkerService.SetWorkerSandboxRuntimes:output_type -> ateapi.SetWorkerSandboxRuntimesResponse + 129, // 292: ateapi.WorkerService.MintWorkerActorCertificate:output_type -> ateapi.MintWorkerActorCertificateResponse + 131, // 293: ateapi.WorkerService.MintWorkerActorJwt:output_type -> ateapi.MintWorkerActorJwtResponse + 133, // 294: ateapi.WorkerService.RequestActorSuspend:output_type -> ateapi.RequestActorSuspendResponse + 137, // 295: ateapi.WorkerService.ReportActorSnapshotState:output_type -> ateapi.ReportActorSnapshotStateResponse + 135, // 296: ateapi.WorkerService.ReportActorCrashed:output_type -> ateapi.ReportActorCrashedResponse + 246, // [246:297] is the sub-list for method output_type + 195, // [195:246] is the sub-list for method input_type + 193, // [193:195] is the sub-list for extension type_name + 191, // [191:193] is the sub-list for extension extendee + 0, // [0:191] is the sub-list for field type_name } func init() { file_ateapi_proto_init() } @@ -10906,7 +11172,7 @@ func file_ateapi_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_ateapi_proto_rawDesc), len(file_ateapi_proto_rawDesc)), NumEnums: 10, - NumMessages: 140, + NumMessages: 144, NumExtensions: 2, NumServices: 2, }, diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index 794a0cbcf0..e453a318c9 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -906,8 +906,10 @@ message ActorStatus { // worker_assignment points at the worker currently hosting this Actor. // Unset whenever the Actor has no worker (SUSPENDED, CRASHED). // + // It can be set and cleared, but not pointed at another Worker in place: + // the fields that identify the Worker cannot change while it is set. + // // +k8s:optional - // +k8s:update=NoModify # can be set and cleared, but not changed in place WorkerAssignment worker_assignment = 2; // external_volumes is the state of the external volumes attached to the @@ -971,8 +973,11 @@ message ActorCrash { // // This is a denormalized snapshot, not merely a reference: atenet reads // worker_pod_ips on the request-routing path and must not need a second lookup, -// and kubectl-ate displays the pod name. The copies stay valid for the life of -// the assignment because a Worker's identity fields are immutable. +// and kubectl-ate displays the pod name. The copies of the Worker's identity +// fields stay valid for the life of the assignment because those fields are +// immutable, and cannot change in place. worker_pod_ips and worker_epoch +// describe the worker process the Actor runs under, and are updated when the +// Actor becomes RUNNING. // // Pass `worker` to GetWorker; none of the denormalized fields identify the // Worker to the API. @@ -985,33 +990,39 @@ message WorkerAssignment { // // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix // +k8s:required + // +k8s:update=NoModify ObjectRef worker = 6; // worker_namespace is the Kubernetes namespace of the WorkerPool and worker Pod. // // +k8s:required // +k8s:format=k8s-short-name + // +k8s:update=NoModify string worker_namespace = 1; // worker_pool is the name of the WorkerPool the worker belongs to. // // +k8s:required // +k8s:format=k8s-long-name + // +k8s:update=NoModify string worker_pool = 2; // worker_pod is the name of the Kubernetes pod hosting the Actor. // // +k8s:required // +k8s:format=k8s-long-name + // +k8s:update=NoModify string worker_pod = 3; // worker_pod_uid is the Kubernetes UID of worker_pod. // // +k8s:required // +k8s:format=k8s-uuid + // +k8s:update=NoModify string worker_pod_uid = 4; - // worker_pod_ips are the IPs of worker_pod, copied from Worker.ips. + // worker_pod_ips are the IPs of worker_pod, copied from Worker.status.ips + // when the Actor is placed, and again when it becomes RUNNING. // // +k8s:required // +k8s:maxItems=2 @@ -1023,10 +1034,12 @@ message WorkerAssignment { // // +k8s:required // +k8s:format=k8s-long-name + // +k8s:update=NoModify string node_name = 7; - // worker_epoch is the Worker's epoch when the Actor was placed on it. 0 if - // the Worker had not reported one. + // worker_epoch is the epoch of the worker process the Actor runs under, as + // that process reports it from Run or Restore. 0 until the Actor becomes + // RUNNING. // // +k8s:optional // +k8s:minimum=0 @@ -1036,6 +1049,7 @@ message WorkerAssignment { // was started or restored under on this Worker. // // +k8s:optional + // +k8s:update=NoModify ActorSandboxRuntime sandbox_runtime = 9; // worker_spiffe_id is the SPIFFE ID worker_pod's certificate carries, copied @@ -1044,6 +1058,7 @@ message WorkerAssignment { // +k8s:optional // +k8s:maxLength=2048 // +k8s:customValidation # a SPIFFE ID + // +k8s:update=NoModify string worker_spiffe_id = 10; } @@ -2332,15 +2347,6 @@ message Worker { // +k8s:format=k8s-long-name // +k8s:immutable string node_name = 6; - // ips are the worker pod's IPs, at most one per IP family, in the order - // Kubernetes reports them; the first is in the cluster's primary family. - // - // +k8s:required - // +k8s:maxItems=2 - // +k8s:listType=atomic - // +k8s:customValidation # until `format=k8s-ip` is supported - // +k8s:immutable - repeated string ips = 7; // worker_spiffe_id is the SPIFFE ID the worker pod's certificate carries. // atenet-router requires it on the certificate of the worker's atunnel. // @@ -2348,7 +2354,7 @@ message Worker { // +k8s:maxLength=2048 // +k8s:customValidation # a SPIFFE ID // +k8s:immutable - string worker_spiffe_id = 8; + string worker_spiffe_id = 7; // labels mirror the WorkerPool object's Kubernetes labels, which selectors // match against. @@ -2359,21 +2365,7 @@ message Worker { // +k8s:maxProperties=64 // +k8s:eachKey=+k8s:format=k8s-label-key // +k8s:eachVal=+k8s:format=k8s-label-value - map labels = 9; - - // epoch counts the runs of the worker pod's worker container: 1 for its first - // run and one more for each restart. 0 means it has not been reported. - // - // A restarted worker has lost the sandboxes of the Actors it was hosting, so - // once epoch rises the control plane crashes every Actor placed during an - // earlier epoch; status.observed_epoch reports when that is done. It may only - // increase. - // - // +k8s:optional - // +k8s:minimum=0 - // +k8s:update=NoUnset - // +k8s:customValidation - int64 epoch = 10; + map labels = 8; // Output-only server-managed state. Absent from Create/Update request // payloads; whatever a request carries here is ignored. A client moves @@ -2381,7 +2373,7 @@ message Worker { // MarkWorkerUnavailable, and the assignments are the scheduler's. // // +k8s:optional - WorkerStatus status = 11; + WorkerStatus status = 9; } enum WorkerState { @@ -2423,24 +2415,55 @@ message WorkerStatus { WorkerResources allocated = 3; // observed_epoch is the latest epoch whose earlier Actors the control plane - // has crashed and released. While it is below epoch, Actors placed before - // the worker's last restart may still be reported as running. + // has crashed and released. While it is below registered_epoch, Actors placed + // before the worker's last restart may still be reported as running. // // +k8s:optional // +k8s:minimum=0 int64 observed_epoch = 4; - // The sandbox classes this Worker runs and the runtimes it runs them - // under, as reported by the Worker at registration; empty until then, and - // nothing is placed on the Worker until then. One entry today, like - // WorkerPoolSpec.sandboxClasses. Its sandbox_class is the Worker's class, - // fixed by the first RegisterWorker call. + // The sandbox classes this Worker runs and the runtimes it runs them under, + // as the registered worker process reports them through + // WorkerService.SetWorkerSandboxRuntimes. Cleared when a new worker process + // registers, and nothing is placed on the Worker until it reports them. One + // entry today, like WorkerPoolSpec.sandboxClasses. Its sandbox_class is the + // Worker's class. // // +k8s:optional // +k8s:maxItems=1 # keep in sync with WorkerPoolSpec.sandboxClasses // +k8s:listType=map // +k8s:listMapKey=sandbox_class repeated WorkerSandboxClass sandbox_classes = 5; + + // ips are the worker pod's IPs, at most one per IP family, in the order + // Kubernetes reports them; the first is in the cluster's primary family. + // The worker process reports them when it registers, so they follow a pod + // sandbox that is recreated with new IPs. + // + // +k8s:optional + // +k8s:maxItems=2 + // +k8s:listType=atomic + // +k8s:customValidation # until `format=k8s-ip` is supported + repeated string ips = 6; + + // issued_epoch is the last epoch handed out by + // WorkerService.AcquireWorkerEpoch. Each worker process acquires one at + // startup. + // + // +k8s:optional + // +k8s:minimum=0 + int64 issued_epoch = 7; + + // registered_epoch is the epoch of the worker process that registered last, + // through WorkerService.RegisterWorker. Only one worker process runs in a pod + // at a time, and a process registers only an epoch it acquired, so a rise + // means every earlier process has exited and lost its sandboxes. The control + // plane then crashes the Actors that ran under an earlier epoch. 0 until the + // first registration; a Worker takes no Actors before then. + // + // +k8s:optional + // +k8s:minimum=0 + int64 registered_epoch = 8; } // WorkerSandboxClass is one sandbox class a Worker runs, with the runtimes @@ -2611,12 +2634,6 @@ message ActorAssignment { // // +k8s:optional Resources resources = 4; - - // worker_epoch is the Worker's epoch when the Actor was bound. - // - // +k8s:optional - // +k8s:minimum=0 - int64 worker_epoch = 5; } // WorkerService is how a Worker tells the control plane what it observes. It is @@ -2624,18 +2641,42 @@ message ActorAssignment { // authorization: Control is the client-facing API, while these RPCs are served // only to an atelet, and only for the Workers on its own node. service WorkerService { - // RegisterWorker records what a Worker can hold and its supported sandbox - // runtimes in one write, so a Worker is never schedulable without runtime - // compatibility information to match snapshots against. Capacity and - // runtimes are the Worker's to report rather than the control plane's to - // infer: they are what the worker can actually supply and expose, only its - // node can observe them, and a fleet may run mixed versions. + // AcquireWorkerEpoch hands out a new epoch for a worker process, the first + // step of its startup. Nothing becomes active until the process registers + // with the epoch through RegisterWorker, so an acquire from a process that + // exits before it sees the reply changes nothing the scheduler or the + // reconciler acts on. + // + // atelet calls this with its own client certificate, for a Worker on its own + // node. Not idempotent: each call hands out a new epoch. + rpc AcquireWorkerEpoch(AcquireWorkerEpochRequest) returns (AcquireWorkerEpochResponse); + + // RegisterWorker activates the epoch a worker process acquired and records + // its pod IPs and capacity, in one write. A worker process registers once. + // The epoch must be one AcquireWorkerEpoch handed out, and higher than the + // Worker's registered_epoch; once it is registered, the Actors that ran under + // an earlier epoch are crashed, and the Worker takes no Actors until the new + // worker process reports its sandbox runtimes. Capacity and IPs are the Worker's to report: + // only its node can observe them. // // atelet calls this with its own client certificate, as it does for - // MintCert. Idempotent: re-sending the same capacity and runtimes is not a - // write. + // MintCert. Re-sending the registered epoch with the same IPs and capacity is + // not a write. An older epoch, one that was never handed out, or the + // registered epoch with other IPs or capacity gets FailedPrecondition: an epoch's + // registration does not change once made. rpc RegisterWorker(RegisterWorkerRequest) returns (RegisterWorkerResponse); + // SetWorkerSandboxRuntimes records the sandbox runtimes a registered worker + // process can run. A worker process reports them after it registers, and + // again whenever they change. The epoch must be the Worker's registered_epoch, + // so a report from a worker process that has since been replaced gets + // FailedPrecondition, as does an entry of another class than the one this + // worker process reported first. + // + // atelet calls this with its own client certificate. Re-sending the same + // runtimes is not a write. + rpc SetWorkerSandboxRuntimes(SetWorkerSandboxRuntimesRequest) returns (SetWorkerSandboxRuntimesResponse); + // Create a Substrate-issued SPIFFE certificate that asserts a worker acting // on behalf of a particular actor. // @@ -2708,6 +2749,20 @@ service WorkerService { rpc ReportActorCrashed(ReportActorCrashedRequest) returns (ReportActorCrashedResponse); } +message AcquireWorkerEpochRequest { + // The Worker whose worker process is starting. atespace is always empty; + // Workers are global-scoped. + // + // +k8s:required + // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix + ObjectRef worker = 1; +} + +message AcquireWorkerEpochResponse { + // The epoch handed out, for the worker process to register with. + int64 epoch = 1; +} + message RegisterWorkerRequest { // The Worker being registered. atespace is always empty; Workers are // global-scoped. @@ -2716,17 +2771,52 @@ message RegisterWorkerRequest { // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix ObjectRef worker = 1; + // The epoch the worker process acquired through AcquireWorkerEpoch. + // + // +k8s:required + // +k8s:minimum=1 + int64 epoch = 3; + + // The worker pod's IPs, at most one per IP family, in the order Kubernetes + // reports them. They replace what is recorded. + // + // +k8s:required + // +k8s:maxItems=2 + // +k8s:listType=atomic + // +k8s:customValidation # until `format=k8s-ip` is supported + repeated string ips = 4; + // Everything the Worker can hold. This replaces what is recorded rather than // merging into it: a dimension left out is one the Worker no longer supplies, // and an Actor asking for that dimension will not be placed here. // // +k8s:required WorkerResources capacity = 2; +} + +message RegisterWorkerResponse { + // The Worker as recorded, so a caller sees what its report resolved to. + Worker worker = 1; +} + +message SetWorkerSandboxRuntimesRequest { + // The Worker whose runtimes these are. atespace is always empty; Workers are + // global-scoped. + // + // +k8s:required + // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix + ObjectRef worker = 1; + + // The epoch the worker process registered with. + // + // +k8s:required + // +k8s:minimum=1 + int64 epoch = 2; - // The sandbox classes this worker runs and the runtimes it runs them - // under. The first registration fixes the Worker's sandbox class; later - // reports replace the runtimes, but an entry of another class is refused. - // Like capacity, this replaces what is recorded on the Worker. + // The sandbox classes this worker process runs and the runtimes it runs them + // under. The first report of a worker process fixes the Worker's sandbox + // class; later reports replace the runtimes, but an entry of another class is + // refused. This replaces what is recorded on the Worker. // // +k8s:required // +k8s:minItems=1 @@ -2736,8 +2826,8 @@ message RegisterWorkerRequest { repeated WorkerSandboxClass sandbox_classes = 3; } -message RegisterWorkerResponse { - // The Worker as recorded, so a caller sees what its report resolved to. +message SetWorkerSandboxRuntimesResponse { + // The Worker as recorded. Worker worker = 1; } diff --git a/pkg/proto/ateapipb/ateapi_grpc.pb.go b/pkg/proto/ateapipb/ateapi_grpc.pb.go index 31545303e8..f8a5e5be72 100644 --- a/pkg/proto/ateapipb/ateapi_grpc.pb.go +++ b/pkg/proto/ateapipb/ateapi_grpc.pb.go @@ -1887,7 +1887,9 @@ var Control_ServiceDesc = grpc.ServiceDesc{ } const ( + WorkerService_AcquireWorkerEpoch_FullMethodName = "/ateapi.WorkerService/AcquireWorkerEpoch" WorkerService_RegisterWorker_FullMethodName = "/ateapi.WorkerService/RegisterWorker" + WorkerService_SetWorkerSandboxRuntimes_FullMethodName = "/ateapi.WorkerService/SetWorkerSandboxRuntimes" WorkerService_MintWorkerActorCertificate_FullMethodName = "/ateapi.WorkerService/MintWorkerActorCertificate" WorkerService_MintWorkerActorJwt_FullMethodName = "/ateapi.WorkerService/MintWorkerActorJwt" WorkerService_RequestActorSuspend_FullMethodName = "/ateapi.WorkerService/RequestActorSuspend" @@ -1904,17 +1906,39 @@ const ( // authorization: Control is the client-facing API, while these RPCs are served // only to an atelet, and only for the Workers on its own node. type WorkerServiceClient interface { - // RegisterWorker records what a Worker can hold and its supported sandbox - // runtimes in one write, so a Worker is never schedulable without runtime - // compatibility information to match snapshots against. Capacity and - // runtimes are the Worker's to report rather than the control plane's to - // infer: they are what the worker can actually supply and expose, only its - // node can observe them, and a fleet may run mixed versions. + // AcquireWorkerEpoch hands out a new epoch for a worker process, the first + // step of its startup. Nothing becomes active until the process registers + // with the epoch through RegisterWorker, so an acquire from a process that + // exits before it sees the reply changes nothing the scheduler or the + // reconciler acts on. + // + // atelet calls this with its own client certificate, for a Worker on its own + // node. Not idempotent: each call hands out a new epoch. + AcquireWorkerEpoch(ctx context.Context, in *AcquireWorkerEpochRequest, opts ...grpc.CallOption) (*AcquireWorkerEpochResponse, error) + // RegisterWorker activates the epoch a worker process acquired and records + // its pod IPs and capacity, in one write. A worker process registers once. + // The epoch must be one AcquireWorkerEpoch handed out, and higher than the + // Worker's registered_epoch; once it is registered, the Actors that ran under + // an earlier epoch are crashed, and the Worker takes no Actors until the new + // worker process reports its sandbox runtimes. Capacity and IPs are the Worker's to report: + // only its node can observe them. // // atelet calls this with its own client certificate, as it does for - // MintCert. Idempotent: re-sending the same capacity and runtimes is not a - // write. + // MintCert. Re-sending the registered epoch with the same IPs and capacity is + // not a write. An older epoch, one that was never handed out, or the + // registered epoch with other IPs or capacity gets FailedPrecondition: an epoch's + // registration does not change once made. RegisterWorker(ctx context.Context, in *RegisterWorkerRequest, opts ...grpc.CallOption) (*RegisterWorkerResponse, error) + // SetWorkerSandboxRuntimes records the sandbox runtimes a registered worker + // process can run. A worker process reports them after it registers, and + // again whenever they change. The epoch must be the Worker's registered_epoch, + // so a report from a worker process that has since been replaced gets + // FailedPrecondition, as does an entry of another class than the one this + // worker process reported first. + // + // atelet calls this with its own client certificate. Re-sending the same + // runtimes is not a write. + SetWorkerSandboxRuntimes(ctx context.Context, in *SetWorkerSandboxRuntimesRequest, opts ...grpc.CallOption) (*SetWorkerSandboxRuntimesResponse, error) // Create a Substrate-issued SPIFFE certificate that asserts a worker acting // on behalf of a particular actor. // @@ -1991,6 +2015,16 @@ func NewWorkerServiceClient(cc grpc.ClientConnInterface) WorkerServiceClient { return &workerServiceClient{cc} } +func (c *workerServiceClient) AcquireWorkerEpoch(ctx context.Context, in *AcquireWorkerEpochRequest, opts ...grpc.CallOption) (*AcquireWorkerEpochResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(AcquireWorkerEpochResponse) + err := c.cc.Invoke(ctx, WorkerService_AcquireWorkerEpoch_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *workerServiceClient) RegisterWorker(ctx context.Context, in *RegisterWorkerRequest, opts ...grpc.CallOption) (*RegisterWorkerResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(RegisterWorkerResponse) @@ -2001,6 +2035,16 @@ func (c *workerServiceClient) RegisterWorker(ctx context.Context, in *RegisterWo return out, nil } +func (c *workerServiceClient) SetWorkerSandboxRuntimes(ctx context.Context, in *SetWorkerSandboxRuntimesRequest, opts ...grpc.CallOption) (*SetWorkerSandboxRuntimesResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SetWorkerSandboxRuntimesResponse) + err := c.cc.Invoke(ctx, WorkerService_SetWorkerSandboxRuntimes_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *workerServiceClient) MintWorkerActorCertificate(ctx context.Context, in *MintWorkerActorCertificateRequest, opts ...grpc.CallOption) (*MintWorkerActorCertificateResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(MintWorkerActorCertificateResponse) @@ -2060,17 +2104,39 @@ func (c *workerServiceClient) ReportActorCrashed(ctx context.Context, in *Report // authorization: Control is the client-facing API, while these RPCs are served // only to an atelet, and only for the Workers on its own node. type WorkerServiceServer interface { - // RegisterWorker records what a Worker can hold and its supported sandbox - // runtimes in one write, so a Worker is never schedulable without runtime - // compatibility information to match snapshots against. Capacity and - // runtimes are the Worker's to report rather than the control plane's to - // infer: they are what the worker can actually supply and expose, only its - // node can observe them, and a fleet may run mixed versions. + // AcquireWorkerEpoch hands out a new epoch for a worker process, the first + // step of its startup. Nothing becomes active until the process registers + // with the epoch through RegisterWorker, so an acquire from a process that + // exits before it sees the reply changes nothing the scheduler or the + // reconciler acts on. + // + // atelet calls this with its own client certificate, for a Worker on its own + // node. Not idempotent: each call hands out a new epoch. + AcquireWorkerEpoch(context.Context, *AcquireWorkerEpochRequest) (*AcquireWorkerEpochResponse, error) + // RegisterWorker activates the epoch a worker process acquired and records + // its pod IPs and capacity, in one write. A worker process registers once. + // The epoch must be one AcquireWorkerEpoch handed out, and higher than the + // Worker's registered_epoch; once it is registered, the Actors that ran under + // an earlier epoch are crashed, and the Worker takes no Actors until the new + // worker process reports its sandbox runtimes. Capacity and IPs are the Worker's to report: + // only its node can observe them. // // atelet calls this with its own client certificate, as it does for - // MintCert. Idempotent: re-sending the same capacity and runtimes is not a - // write. + // MintCert. Re-sending the registered epoch with the same IPs and capacity is + // not a write. An older epoch, one that was never handed out, or the + // registered epoch with other IPs or capacity gets FailedPrecondition: an epoch's + // registration does not change once made. RegisterWorker(context.Context, *RegisterWorkerRequest) (*RegisterWorkerResponse, error) + // SetWorkerSandboxRuntimes records the sandbox runtimes a registered worker + // process can run. A worker process reports them after it registers, and + // again whenever they change. The epoch must be the Worker's registered_epoch, + // so a report from a worker process that has since been replaced gets + // FailedPrecondition, as does an entry of another class than the one this + // worker process reported first. + // + // atelet calls this with its own client certificate. Re-sending the same + // runtimes is not a write. + SetWorkerSandboxRuntimes(context.Context, *SetWorkerSandboxRuntimesRequest) (*SetWorkerSandboxRuntimesResponse, error) // Create a Substrate-issued SPIFFE certificate that asserts a worker acting // on behalf of a particular actor. // @@ -2147,9 +2213,15 @@ type WorkerServiceServer interface { // pointer dereference when methods are called. type UnimplementedWorkerServiceServer struct{} +func (UnimplementedWorkerServiceServer) AcquireWorkerEpoch(context.Context, *AcquireWorkerEpochRequest) (*AcquireWorkerEpochResponse, error) { + return nil, status.Error(codes.Unimplemented, "method AcquireWorkerEpoch not implemented") +} func (UnimplementedWorkerServiceServer) RegisterWorker(context.Context, *RegisterWorkerRequest) (*RegisterWorkerResponse, error) { return nil, status.Error(codes.Unimplemented, "method RegisterWorker not implemented") } +func (UnimplementedWorkerServiceServer) SetWorkerSandboxRuntimes(context.Context, *SetWorkerSandboxRuntimesRequest) (*SetWorkerSandboxRuntimesResponse, error) { + return nil, status.Error(codes.Unimplemented, "method SetWorkerSandboxRuntimes not implemented") +} func (UnimplementedWorkerServiceServer) MintWorkerActorCertificate(context.Context, *MintWorkerActorCertificateRequest) (*MintWorkerActorCertificateResponse, error) { return nil, status.Error(codes.Unimplemented, "method MintWorkerActorCertificate not implemented") } @@ -2186,6 +2258,24 @@ func RegisterWorkerServiceServer(s grpc.ServiceRegistrar, srv WorkerServiceServe s.RegisterService(&WorkerService_ServiceDesc, srv) } +func _WorkerService_AcquireWorkerEpoch_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AcquireWorkerEpochRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(WorkerServiceServer).AcquireWorkerEpoch(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: WorkerService_AcquireWorkerEpoch_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(WorkerServiceServer).AcquireWorkerEpoch(ctx, req.(*AcquireWorkerEpochRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _WorkerService_RegisterWorker_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(RegisterWorkerRequest) if err := dec(in); err != nil { @@ -2204,6 +2294,24 @@ func _WorkerService_RegisterWorker_Handler(srv interface{}, ctx context.Context, return interceptor(ctx, in, info, handler) } +func _WorkerService_SetWorkerSandboxRuntimes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SetWorkerSandboxRuntimesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(WorkerServiceServer).SetWorkerSandboxRuntimes(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: WorkerService_SetWorkerSandboxRuntimes_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(WorkerServiceServer).SetWorkerSandboxRuntimes(ctx, req.(*SetWorkerSandboxRuntimesRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _WorkerService_MintWorkerActorCertificate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(MintWorkerActorCertificateRequest) if err := dec(in); err != nil { @@ -2301,10 +2409,18 @@ var WorkerService_ServiceDesc = grpc.ServiceDesc{ ServiceName: "ateapi.WorkerService", HandlerType: (*WorkerServiceServer)(nil), Methods: []grpc.MethodDesc{ + { + MethodName: "AcquireWorkerEpoch", + Handler: _WorkerService_AcquireWorkerEpoch_Handler, + }, { MethodName: "RegisterWorker", Handler: _WorkerService_RegisterWorker_Handler, }, + { + MethodName: "SetWorkerSandboxRuntimes", + Handler: _WorkerService_SetWorkerSandboxRuntimes_Handler, + }, { MethodName: "MintWorkerActorCertificate", Handler: _WorkerService_MintWorkerActorCertificate_Handler, From e88ffcdb54d65eac54c3a0e064364029bd84f0af Mon Sep 17 00:00:00 2001 From: Chenyi Wang Date: Thu, 22 Oct 2026 15:14:22 +0000 Subject: [PATCH 12/12] benchmarking: rename --lifecycle-mode values to sync/async and default to async (#3097) Rename the `--lifecycle-mode` choices from `suspend` and `pause` to `sync` and `async`, and switch the default in `lifecycle_mode.py` and the boomer `agentsession`, `ateenv`, `glutton`, `durdir`, and `sweperf` user classes to `async` (`DURABILITY_MODE_ASYNCHRONOUS`) to match `Control.SuspendActor`'s default (#2813). - [x] Tests pass - [x] Docs updated --- benchmarking/README.md | 8 ++-- benchmarking/automation/tests.yaml | 32 ++++++++++---- benchmarking/locust/common/lifecycle_mode.py | 10 ++--- .../locust/unit_tests/test_boomer_config.py | 4 +- .../boomer/agentsession/agentsession.go | 19 +++++---- .../boomer/agentsession/agentsession_test.go | 24 +++++++---- .../boomer/agentsession/knobs_test.go | 4 +- .../benchmarking/boomer/ateenv/knobs_test.go | 4 +- .../benchmarking/boomer/ateenv/lifecycle.go | 12 +++--- .../boomer/ateenv/lifecycle_test.go | 36 ++++++++-------- .../boomer/dynconfig/dynconfig.go | 14 +++---- .../benchmarking/boomer/glutton/durdir.go | 12 +++--- .../boomer/glutton/durdir_test.go | 42 ++++++++++++------- .../benchmarking/boomer/glutton/knobs_test.go | 8 ++-- .../benchmarking/boomer/glutton/lifecycle.go | 12 +++--- .../boomer/glutton/lifecycle_test.go | 16 +++---- .../benchmarking/boomer/sweperf/knobs_test.go | 4 +- .../benchmarking/boomer/sweperf/sweperf.go | 16 +++---- .../boomer/sweperf/sweperf_test.go | 10 ++--- 19 files changed, 163 insertions(+), 124 deletions(-) diff --git a/benchmarking/README.md b/benchmarking/README.md index 5f17b2466b..1455a9653f 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -288,7 +288,7 @@ through `atenet-router` (`1 dns:///` gRPC channel per VU). Reuses `glutton`'s runtime and startup knobs: `--min-wait-time`, `--max-wait-time`, `--min-live-time`, `--max-live-time`, -`--max-pings-per-wake`, `--lifecycle-mode` (`suspend` | `pause` | `none`), +`--max-pings-per-wake`, `--lifecycle-mode` (`sync` | `async` | `none`), `--trace-probability`, and `--actors-per-user`. #### ate-env Reported Metrics @@ -373,9 +373,9 @@ deployment fails loudly instead of showing up as OOM-flaky steps. gap gets ±20% jitter so sessions don't move in lockstep. * `--resume-mode implicit|explicit` — implicit (default) lets the parked first request wake the actor; explicit issues ResumeActor before traffic. -* `--lifecycle-mode suspend|pause|none` — synchronous suspend (`suspend`, - default, uploads the snapshot before returning), asynchronous suspend - (`pause`, uploads the snapshot in the background), or `none` to keep actors +* `--lifecycle-mode sync|async|none` — synchronous suspend (`sync`, + uploads the snapshot before returning), asynchronous suspend (`async`, + default, uploads the snapshot in the background), or `none` to keep actors resident in memory between steps without hibernating. * `--agentsession-snapshot-steps` — comma-separated 1-based step numbers after which the actor is hibernated (for example, `3,10,20,30,40,50,61`); between diff --git a/benchmarking/automation/tests.yaml b/benchmarking/automation/tests.yaml index 945fdda3d1..678cbf6ec5 100644 --- a/benchmarking/automation/tests.yaml +++ b/benchmarking/automation/tests.yaml @@ -149,6 +149,8 @@ tests: users: 1 workerCount: 1 flags: + - "--lifecycle-mode" + - "sync" - "--min-wait-time" - "1.0" - "--max-wait-time" @@ -163,7 +165,7 @@ tests: workerCount: 1 flags: - "--lifecycle-mode" - - "pause" + - "async" - "--min-wait-time" - "1.0" - "--max-wait-time" @@ -191,6 +193,8 @@ tests: workerCount: 1 actorMemory: "1536Mi" flags: + - "--lifecycle-mode" + - "sync" - "--mem-target" - "1Gi" - "--mem-churn" @@ -212,7 +216,7 @@ tests: actorMemory: "1536Mi" flags: - "--lifecycle-mode" - - "pause" + - "async" - "--mem-target" - "1Gi" - "--mem-churn" @@ -254,6 +258,8 @@ tests: workerCount: 1 actorMemory: "2560Mi" flags: + - "--lifecycle-mode" + - "sync" - "--mem-target" - "2Gi" - "--mem-churn" @@ -275,7 +281,7 @@ tests: actorMemory: "2560Mi" flags: - "--lifecycle-mode" - - "pause" + - "async" - "--mem-target" - "2Gi" - "--mem-churn" @@ -316,6 +322,8 @@ tests: users: 5 workerCount: 5 flags: + - "--lifecycle-mode" + - "sync" - "--min-wait-time" - "1.0" - "--max-wait-time" @@ -330,7 +338,7 @@ tests: workerCount: 5 flags: - "--lifecycle-mode" - - "pause" + - "async" - "--min-wait-time" - "1.0" - "--max-wait-time" @@ -344,6 +352,8 @@ tests: users: 10 workerCount: 10 flags: + - "--lifecycle-mode" + - "sync" - "--min-wait-time" - "1.0" - "--max-wait-time" @@ -358,7 +368,7 @@ tests: workerCount: 10 flags: - "--lifecycle-mode" - - "pause" + - "async" - "--min-wait-time" - "1.0" - "--max-wait-time" @@ -456,6 +466,8 @@ tests: flags: - "--durdir-template" - "glutton-durdir-data" + - "--lifecycle-mode" + - "sync" - "--resume-mode" - "explicit" - "--durdir-file-size-bytes" @@ -476,7 +488,7 @@ tests: - "--durdir-template" - "glutton-durdir-data" - "--lifecycle-mode" - - "pause" + - "async" - "--resume-mode" - "explicit" - "--durdir-file-size-bytes" @@ -496,6 +508,8 @@ tests: flags: - "--durdir-template" - "glutton-durdir-full" + - "--lifecycle-mode" + - "sync" - "--resume-mode" - "explicit" - "--durdir-file-size-bytes" @@ -516,7 +530,7 @@ tests: - "--durdir-template" - "glutton-durdir-full" - "--lifecycle-mode" - - "pause" + - "async" - "--resume-mode" - "explicit" - "--durdir-file-size-bytes" @@ -597,6 +611,8 @@ tests: flags: - "--durdir-template" - "glutton-durdir-data" + - "--lifecycle-mode" + - "sync" - "--resume-mode" - "explicit" - "--durdir-read-mode" @@ -619,7 +635,7 @@ tests: - "--durdir-template" - "glutton-durdir-data" - "--lifecycle-mode" - - "pause" + - "async" - "--resume-mode" - "explicit" - "--durdir-read-mode" diff --git a/benchmarking/locust/common/lifecycle_mode.py b/benchmarking/locust/common/lifecycle_mode.py index 77e27de69a..c21035491e 100644 --- a/benchmarking/locust/common/lifecycle_mode.py +++ b/benchmarking/locust/common/lifecycle_mode.py @@ -24,9 +24,9 @@ def add_lifecycle_mode_arguments(parser: LocustArgumentParser) -> None: group.add_argument( "--lifecycle-mode", type=str, - default="suspend", - choices=["suspend", "pause", "none"], - help="Lifecycle mode: 'suspend' uses synchronous SuspendActor (uploads snapshot before returning); " - "'pause' uses asynchronous SuspendActor (uploads snapshot in background); " - "'none' keeps actors resident in memory between steps (default: suspend)", + default="async", + choices=["sync", "async", "none"], + help="Lifecycle mode: 'sync' uses synchronous SuspendActor (uploads snapshot before returning); " + "'async' uses asynchronous SuspendActor (uploads snapshot in background); " + "'none' keeps actors resident in memory between steps (default: async)", ) diff --git a/benchmarking/locust/unit_tests/test_boomer_config.py b/benchmarking/locust/unit_tests/test_boomer_config.py index 58e08fd594..56a547c515 100644 --- a/benchmarking/locust/unit_tests/test_boomer_config.py +++ b/benchmarking/locust/unit_tests/test_boomer_config.py @@ -39,12 +39,12 @@ def test_build_config_json_lifecycle_mode_none(self): def test_build_config_json_snapshot_steps(self): raw = boomer_config.build_config_json([ "--lifecycle-mode", - "suspend", + "sync", "--agentsession-snapshot-steps", "3,10,20,30,40,50,61", ]) parsed = json.loads(raw) - self.assertEqual(parsed["lifecycle_mode"], "suspend") + self.assertEqual(parsed["lifecycle_mode"], "sync") self.assertEqual( parsed["agentsession_snapshot_steps"], "3,10,20,30,40,50,61" ) diff --git a/internal/benchmarking/boomer/agentsession/agentsession.go b/internal/benchmarking/boomer/agentsession/agentsession.go index d539d39381..9f1dbc013d 100644 --- a/internal/benchmarking/boomer/agentsession/agentsession.go +++ b/internal/benchmarking/boomer/agentsession/agentsession.go @@ -1085,10 +1085,11 @@ func (u *sessionUser) resume(ctx context.Context) error { return err } -// hibernate suspends the actor at the end of a step, asynchronously in pause -// LifecycleMode. In LifecycleModeNone the actor stays resident and hibernate -// is a no-op. A failure leaves hibernatePending set so the next step -// re-drives it instead of waking a stranded actor. +// hibernate suspends the actor at the end of a step, asynchronously by default +// (async LifecycleMode) and synchronously in sync LifecycleMode. In +// LifecycleModeNone the actor stays resident and hibernate is a no-op. A +// failure leaves hibernatePending set so the next step re-drives it instead of +// waking a stranded actor. func (u *sessionUser) hibernate(ctx context.Context) { mode := dynconfig.Get[knobs](u.cfg.Dyn).LifecycleMode if mode == dynconfig.LifecycleModeNone { @@ -1096,14 +1097,14 @@ func (u *sessionUser) hibernate(ctx context.Context) { return } var err error - if mode == dynconfig.LifecycleModePause { - err = u.tracedCall(ctx, "SuspendActorAsync", func(callCtx context.Context, tr *metadata.MD) error { - _, err := u.cfg.APIStub.SuspendActor(callCtx, &ateapipb.SuspendActorRequest{Actor: u.ref(), Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}}, grpc.Trailer(tr)) + if mode == dynconfig.LifecycleModeSync { + err = u.tracedCall(ctx, "SuspendActor", func(callCtx context.Context, tr *metadata.MD) error { + _, err := u.cfg.APIStub.SuspendActor(callCtx, &ateapipb.SuspendActorRequest{Actor: u.ref(), Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}, grpc.Trailer(tr)) return err }) } else { - err = u.tracedCall(ctx, "SuspendActor", func(callCtx context.Context, tr *metadata.MD) error { - _, err := u.cfg.APIStub.SuspendActor(callCtx, &ateapipb.SuspendActorRequest{Actor: u.ref(), Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}, grpc.Trailer(tr)) + err = u.tracedCall(ctx, "SuspendActorAsync", func(callCtx context.Context, tr *metadata.MD) error { + _, err := u.cfg.APIStub.SuspendActor(callCtx, &ateapipb.SuspendActorRequest{Actor: u.ref(), Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}}, grpc.Trailer(tr)) return err }) } diff --git a/internal/benchmarking/boomer/agentsession/agentsession_test.go b/internal/benchmarking/boomer/agentsession/agentsession_test.go index 79c8aff2f0..23d0e76f18 100644 --- a/internal/benchmarking/boomer/agentsession/agentsession_test.go +++ b/internal/benchmarking/boomer/agentsession/agentsession_test.go @@ -819,8 +819,8 @@ func TestRunStep_RetriesStrandedHibernate(t *testing.T) { } } -// A step ends with an async SuspendActor in pause LifecycleMode and a sync -// one in suspend LifecycleMode. +// A step ends with an async SuspendActor in async LifecycleMode and a sync +// one in sync LifecycleMode. func TestRunStep_HibernateMode(t *testing.T) { ref := &ateapipb.ObjectRef{Atespace: "benchmark", Name: "agent-test"} for _, tt := range []struct { @@ -829,16 +829,24 @@ func TestRunStep_HibernateMode(t *testing.T) { want *ateapipb.SuspendActorRequest }{ { - name: "pause", - lifecycleMode: dynconfig.LifecycleModePause, + name: "default", + lifecycleMode: "", want: &ateapipb.SuspendActorRequest{ Actor: ref, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, }, }, { - name: "suspend", - lifecycleMode: dynconfig.LifecycleModeSuspend, + name: "async", + lifecycleMode: dynconfig.LifecycleModeAsync, + want: &ateapipb.SuspendActorRequest{ + Actor: ref, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, + }, + }, + { + name: "sync", + lifecycleMode: dynconfig.LifecycleModeSync, want: &ateapipb.SuspendActorRequest{ Actor: ref, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, @@ -1340,7 +1348,7 @@ func TestSnapshotSteps(t *testing.T) { ctl := &fakeControlClient{templateMemory: 1 << 30} u := newTestUser(t, &fake.Server{}, ctl, knobs{ Lifecycle: dynconfig.Lifecycle{ - LifecycleMode: dynconfig.LifecycleModePause, + LifecycleMode: dynconfig.LifecycleModeAsync, ResumeMode: dynconfig.ResumeModeExplicit, }, SnapshotSteps: "2,4", @@ -1391,7 +1399,7 @@ func TestSnapshotSteps(t *testing.T) { // A step number beyond len(loaded.Steps) is rejected before CreateActor. badCtl := &fakeControlClient{templateMemory: 1 << 30} badUser := newTestUser(t, &fake.Server{}, badCtl, knobs{ - Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModePause}, + Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeAsync}, SnapshotSteps: "2,99", }) badRT := &runtime{cfg: badUser.cfg} diff --git a/internal/benchmarking/boomer/agentsession/knobs_test.go b/internal/benchmarking/boomer/agentsession/knobs_test.go index 4182716888..804d9c1ba4 100644 --- a/internal/benchmarking/boomer/agentsession/knobs_test.go +++ b/internal/benchmarking/boomer/agentsession/knobs_test.go @@ -28,7 +28,7 @@ func TestKnobsCodec(t *testing.T) { t.Fatalf("fresh holder = %+v, want the defaults with a %v retry budget", got, defaultResumeRetryBudget) } if _, _, err := h.Apply([]byte(`{ - "resume_mode": "explicit", "lifecycle_mode": "pause", + "resume_mode": "explicit", "lifecycle_mode": "async", "agentsession_script": "coding-session", "agentsession_script_file": "/etc/agentsession/script.yaml", "agentsession_think_scale": 0.5, @@ -39,7 +39,7 @@ func TestKnobsCodec(t *testing.T) { t.Fatalf("Apply(valid): %v", err) } want := knobs{ - Lifecycle: dynconfig.Lifecycle{ResumeMode: dynconfig.ResumeModeExplicit, LifecycleMode: dynconfig.LifecycleModePause}, + Lifecycle: dynconfig.Lifecycle{ResumeMode: dynconfig.ResumeModeExplicit, LifecycleMode: dynconfig.LifecycleModeAsync}, Script: "coding-session", ScriptFile: "/etc/agentsession/script.yaml", ThinkScale: 0.5, diff --git a/internal/benchmarking/boomer/ateenv/knobs_test.go b/internal/benchmarking/boomer/ateenv/knobs_test.go index c5dced4cc1..0f705cd003 100644 --- a/internal/benchmarking/boomer/ateenv/knobs_test.go +++ b/internal/benchmarking/boomer/ateenv/knobs_test.go @@ -35,7 +35,7 @@ func TestAteEnvCodec(t *testing.T) { valid := `{ "min_wait_time": 0.2, "max_wait_time": 1.0, "min_live_time": 9, "max_live_time": 14, - "resume_mode": "explicit", "lifecycle_mode": "pause", + "resume_mode": "explicit", "lifecycle_mode": "async", "max_pings_per_wake": 2 }` if _, _, err := h.Apply([]byte(valid)); err != nil { @@ -43,7 +43,7 @@ func TestAteEnvCodec(t *testing.T) { } want := ateEnvKnobs{ WaitTime: dynconfig.WaitTime{MinWait: dynconfig.Seconds(200 * time.Millisecond), MaxWait: dynconfig.Seconds(time.Second)}, - Lifecycle: dynconfig.Lifecycle{ResumeMode: dynconfig.ResumeModeExplicit, LifecycleMode: dynconfig.LifecycleModePause}, + Lifecycle: dynconfig.Lifecycle{ResumeMode: dynconfig.ResumeModeExplicit, LifecycleMode: dynconfig.LifecycleModeAsync}, MinLive: dynconfig.Seconds(9 * time.Second), MaxLive: dynconfig.Seconds(14 * time.Second), MaxPingsPerWake: 2, diff --git a/internal/benchmarking/boomer/ateenv/lifecycle.go b/internal/benchmarking/boomer/ateenv/lifecycle.go index 86741bf58c..873b039e61 100644 --- a/internal/benchmarking/boomer/ateenv/lifecycle.go +++ b/internal/benchmarking/boomer/ateenv/lifecycle.go @@ -420,10 +420,10 @@ func (r *taskRuntime) shutdown(ctx context.Context) { u := val.(*ateEnvUser) for _, a := range u.snapshotActors() { if a.isRunning() { - if dynconfig.Get[ateEnvKnobs](r.cfg.Dyn).LifecycleMode == dynconfig.LifecycleModePause { - _ = a.pause(ctx) - } else { + if dynconfig.Get[ateEnvKnobs](r.cfg.Dyn).LifecycleMode == dynconfig.LifecycleModeSync { _ = a.suspend(ctx) + } else { + _ = a.pause(ctx) } } a.delete(ctx) @@ -655,10 +655,10 @@ func (u *ateEnvActor) hibernate(ctx context.Context) error { switch dynconfig.Get[ateEnvKnobs](u.cfg.Dyn).LifecycleMode { case dynconfig.LifecycleModeNone: return nil - case dynconfig.LifecycleModePause: - return u.pause(ctx) - default: + case dynconfig.LifecycleModeSync: return u.suspend(ctx) + default: + return u.pause(ctx) } } diff --git a/internal/benchmarking/boomer/ateenv/lifecycle_test.go b/internal/benchmarking/boomer/ateenv/lifecycle_test.go index 2a27b4b929..a3923c3c61 100644 --- a/internal/benchmarking/boomer/ateenv/lifecycle_test.go +++ b/internal/benchmarking/boomer/ateenv/lifecycle_test.go @@ -375,7 +375,7 @@ func TestRouterGRPCTarget(t *testing.T) { func TestAteEnvIterate_SuspendMode(t *testing.T) { ctrl := &fakeControlClient{} srv := &fakeProcessServer{} - rt, fix := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSuspend) + rt, fix := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSync) rt.iterate() @@ -424,7 +424,7 @@ func TestAteEnvIterate_SuspendMode(t *testing.T) { func TestAteEnvIterate_PauseMode(t *testing.T) { ctrl := &fakeControlClient{} srv := &fakeProcessServer{} - rt, _ := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModePause) + rt, _ := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeAsync) rt.iterate() @@ -455,8 +455,8 @@ func TestAteEnvIterate_NoneMode(t *testing.T) { rt.shutdown(context.Background()) afterShutdown := ctrl.recordedCalls() - if len(afterShutdown) < 2 || afterShutdown[len(afterShutdown)-2] != "SuspendActor" || afterShutdown[len(afterShutdown)-1] != "DeleteActor" { - t.Errorf("shutdown in none mode must end with [SuspendActor, DeleteActor], got %v", afterShutdown) + if len(afterShutdown) < 2 || afterShutdown[len(afterShutdown)-2] != "SuspendActor(async)" || afterShutdown[len(afterShutdown)-1] != "DeleteActor" { + t.Errorf("shutdown in none mode must end with [SuspendActor(async), DeleteActor], got %v", afterShutdown) } } @@ -475,7 +475,7 @@ func TestAteEnvExec_MetadataPropagation(t *testing.T) { srv := &fakeProcessServer{ stdoutChunks: []string{"hello ", "world\n"}, } - rt, fix := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSuspend) + rt, fix := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSync) rt.cfg.Tracer = tp.Tracer("test-sampled") rt.iterate() @@ -564,7 +564,7 @@ func TestAteEnvExec_ValidationFailures(t *testing.T) { ctrl := &fakeControlClient{} srv := &fakeProcessServer{} tt.setupSrv(srv) - rt, fix := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSuspend) + rt, fix := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSync) actor := &ateEnvActor{ cfg: rt.cfg, @@ -595,7 +595,7 @@ func TestAteEnvExec_ValidationFailures(t *testing.T) { func TestAteEnvExec_Timeout(t *testing.T) { ctrl := &fakeControlClient{} srv := &fakeProcessServer{blockStream: true} - rt, fix := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSuspend) + rt, fix := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSync) actor := &ateEnvActor{ cfg: rt.cfg, @@ -614,7 +614,7 @@ func TestAteEnvIterate_RetriesStrandedHibernate(t *testing.T) { suspendErrs: []error{status.Error(codes.Unavailable, "ate-api-server restarting")}, } srv := &fakeProcessServer{} - rt, _ := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSuspend) + rt, _ := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSync) rt.iterate() // resume, exec, suspend -> suspend fails before := ctrl.recordedCalls() @@ -636,7 +636,7 @@ func TestAteEnvIterate_ReplacesActorOnTerminalResumeFailure(t *testing.T) { resumeErrs: []error{status.Error(codes.FailedPrecondition, "AssignWorker prerequisite not met")}, } srv := &fakeProcessServer{} - rt, _ := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSuspend) + rt, _ := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSync) rt.iterate() @@ -654,7 +654,7 @@ func TestAteEnvIterate_ReplacesActorOnRepeatedExecFailures(t *testing.T) { srv := &fakeProcessServer{ startErr: status.Error(codes.Unavailable, "guest wedged"), } - rt, _ := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSuspend) + rt, _ := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSync) for i := 0; i < maxConsecutiveFailures; i++ { rt.iterate() @@ -677,9 +677,9 @@ func TestAteEnvIterate_CountsAtMostOneExecFailurePerWake(t *testing.T) { srv := &fakeProcessServer{ startErr: status.Error(codes.Unavailable, "transient router blip"), } - rt, _ := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSuspend) + rt, _ := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSync) rt.cfg.Dyn = dynconfig.Static(ateEnvKnobs{ - Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeSuspend}, + Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeSync}, MinLive: dynconfig.Seconds(5 * time.Second), MaxLive: dynconfig.Seconds(5 * time.Second), MaxPingsPerWake: maxConsecutiveFailures, @@ -704,7 +704,7 @@ func TestAteEnvIterate_DoesNotReplaceOnResourceExhaustedExecFailure(t *testing.T srv := &fakeProcessServer{ startErr: status.Error(codes.ResourceExhausted, "rate limit exceeded"), } - rt, _ := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSuspend) + rt, _ := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSync) for i := 0; i < maxConsecutiveFailures+1; i++ { rt.iterate() @@ -722,7 +722,7 @@ func TestAteEnvIterate_DoesNotReplaceOnResourceExhaustedExecFailure(t *testing.T func TestAteEnvShutdown_PauseModeRunningActor(t *testing.T) { ctrl := &fakeControlClient{} srv := &fakeProcessServer{} - rt, fix := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModePause) + rt, fix := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeAsync) u := &ateEnvUser{ routerConn: fix.closerTracker, @@ -749,7 +749,7 @@ func TestAteEnvShutdown_PauseModeRunningActor(t *testing.T) { func TestAteEnvShutdown_StoppedActor(t *testing.T) { ctrl := &fakeControlClient{} srv := &fakeProcessServer{} - rt, fix := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSuspend) + rt, fix := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSync) u := &ateEnvUser{ routerConn: fix.closerTracker, @@ -776,9 +776,9 @@ func TestAteEnvShutdown_StoppedActor(t *testing.T) { func TestAteEnvShutdown_WaitsForInFlightIterate(t *testing.T) { ctrl := &fakeControlClient{} srv := &fakeProcessServer{} - rt, fix := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSuspend) + rt, fix := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSync) rt.cfg.Dyn = dynconfig.Static(ateEnvKnobs{ - Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeSuspend}, + Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeSync}, WaitTime: dynconfig.WaitTime{MinWait: dynconfig.Seconds(5 * time.Second), MaxWait: dynconfig.Seconds(5 * time.Second)}, MinLive: dynconfig.Seconds(5 * time.Second), MaxLive: dynconfig.Seconds(5 * time.Second), @@ -831,7 +831,7 @@ func TestAteEnvShutdown_WaitsForInFlightIterate(t *testing.T) { func TestAteEnvShutdown_RespectsContextCancellation(t *testing.T) { ctrl := &fakeControlClient{} srv := &fakeProcessServer{blockStream: true} - rt, fix := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSuspend) + rt, fix := newTestRuntime(t, ctrl, srv, dynconfig.LifecycleModeSync) iterDone := make(chan struct{}) go func() { diff --git a/internal/benchmarking/boomer/dynconfig/dynconfig.go b/internal/benchmarking/boomer/dynconfig/dynconfig.go index a107ced68d..f84c3137fa 100644 --- a/internal/benchmarking/boomer/dynconfig/dynconfig.go +++ b/internal/benchmarking/boomer/dynconfig/dynconfig.go @@ -240,16 +240,16 @@ func Uniform(lo, hi time.Duration) time.Duration { // Resume and lifecycle modes. Explicit issues a ResumeActor RPC before // sending traffic; implicit issues no wake request at all: the actor stays // suspended until a request reaches the atenet router, which wakes it -// while the request is parked. Suspend runs a synchronous SuspendActor -// (uploads the snapshot before returning); pause runs an asynchronous +// while the request is parked. Sync runs a synchronous SuspendActor +// (uploads the snapshot before returning); async runs an asynchronous // SuspendActor (uploads the snapshot in the background). const ( ResumeModeExplicit = "explicit" ResumeModeImplicit = "implicit" - LifecycleModeSuspend = "suspend" - LifecycleModePause = "pause" - LifecycleModeNone = "none" + LifecycleModeSync = "sync" + LifecycleModeAsync = "async" + LifecycleModeNone = "none" ) // Lifecycle is how a class takes its actors off a worker and brings them @@ -263,8 +263,8 @@ func (l Lifecycle) Validate() error { if l.ResumeMode != "" && l.ResumeMode != ResumeModeExplicit && l.ResumeMode != ResumeModeImplicit { return fmt.Errorf("invalid resume_mode %q: must be %q or %q", l.ResumeMode, ResumeModeExplicit, ResumeModeImplicit) } - if l.LifecycleMode != "" && l.LifecycleMode != LifecycleModeSuspend && l.LifecycleMode != LifecycleModePause && l.LifecycleMode != LifecycleModeNone { - return fmt.Errorf("invalid lifecycle_mode %q: must be %q, %q, or %q", l.LifecycleMode, LifecycleModeSuspend, LifecycleModePause, LifecycleModeNone) + if l.LifecycleMode != "" && l.LifecycleMode != LifecycleModeSync && l.LifecycleMode != LifecycleModeAsync && l.LifecycleMode != LifecycleModeNone { + return fmt.Errorf("invalid lifecycle_mode %q: must be %q, %q, or %q", l.LifecycleMode, LifecycleModeSync, LifecycleModeAsync, LifecycleModeNone) } return nil } diff --git a/internal/benchmarking/boomer/glutton/durdir.go b/internal/benchmarking/boomer/glutton/durdir.go index d6cbc1ee7e..dd67511518 100644 --- a/internal/benchmarking/boomer/glutton/durdir.go +++ b/internal/benchmarking/boomer/glutton/durdir.go @@ -229,10 +229,10 @@ func (u *durDirUser) suspend(ctx context.Context) { } func (u *durDirUser) hibernate(ctx context.Context, dynCfg durDirKnobs) { - if dynCfg.LifecycleMode == dynconfig.LifecycleModePause { - u.pause(ctx) - } else { + if dynCfg.LifecycleMode == dynconfig.LifecycleModeSync { u.suspend(ctx) + } else { + u.pause(ctx) } } @@ -240,15 +240,15 @@ func (u *durDirUser) hibernate(ctx context.Context, dynCfg durDirKnobs) { // The hibernate call is unmetered (teardown precondition, not benchmark latency), // while the delete is metered so true leaks still surface in failures.csv. func (u *durDirUser) hibernateAndDelete(ctx context.Context, dynCfg durDirKnobs) { - if dynCfg.LifecycleMode == dynconfig.LifecycleModePause { + if dynCfg.LifecycleMode == dynconfig.LifecycleModeSync { _, _ = u.cfg.APIStub.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ Actor: u.ref(), - Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }) } else { _, _ = u.cfg.APIStub.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ Actor: u.ref(), - Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, }) } u.delete(ctx) diff --git a/internal/benchmarking/boomer/glutton/durdir_test.go b/internal/benchmarking/boomer/glutton/durdir_test.go index a0b7ce763b..6b8159b256 100644 --- a/internal/benchmarking/boomer/glutton/durdir_test.go +++ b/internal/benchmarking/boomer/glutton/durdir_test.go @@ -44,20 +44,28 @@ func TestDurDirLoopSequence(t *testing.T) { { name: "explicit resume mode", resumeMode: dynconfig.ResumeModeExplicit, - wantGRPCCall: []string{"SuspendActor", "ResumeActor"}, + wantGRPCCall: []string{"SuspendActor(async)", "ResumeActor"}, wantHTTPCall: []string{fake.ReadDiskRoute, fake.ReadDiskRoute, fake.WriteDiskRoute}, - wantMode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS, + wantMode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS, }, { name: "implicit resume mode", resumeMode: dynconfig.ResumeModeImplicit, - wantGRPCCall: []string{"SuspendActor"}, // No ResumeActor RPC! + wantGRPCCall: []string{"SuspendActor(async)"}, // No ResumeActor RPC! wantHTTPCall: []string{fake.ReadDiskRoute, fake.ReadDiskRoute, fake.WriteDiskRoute}, - wantMode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS, + wantMode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS, }, { - name: "pause lifecycle mode", - lifecycleMode: dynconfig.LifecycleModePause, + name: "sync lifecycle mode", + lifecycleMode: dynconfig.LifecycleModeSync, + resumeMode: dynconfig.ResumeModeExplicit, + wantGRPCCall: []string{"SuspendActor", "ResumeActor"}, + wantHTTPCall: []string{fake.ReadDiskRoute, fake.ReadDiskRoute, fake.WriteDiskRoute}, + wantMode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS, + }, + { + name: "async lifecycle mode", + lifecycleMode: dynconfig.LifecycleModeAsync, resumeMode: dynconfig.ResumeModeExplicit, wantGRPCCall: []string{"SuspendActor(async)", "ResumeActor"}, wantHTTPCall: []string{fake.ReadDiskRoute, fake.ReadDiskRoute, fake.WriteDiskRoute}, @@ -242,14 +250,14 @@ func TestDurDirBootstrapFailureSuspendsBeforeDelete(t *testing.T) { } calls := fakeCtrl.recordedCalls() - if len(calls) < 2 || calls[len(calls)-2] != "SuspendActor" || calls[len(calls)-1] != "DeleteActor" { - t.Errorf("recordedCalls must end with [SuspendActor, DeleteActor], got %v", calls) + if len(calls) < 2 || calls[len(calls)-2] != "SuspendActor(async)" || calls[len(calls)-1] != "DeleteActor" { + t.Errorf("recordedCalls must end with [SuspendActor(async), DeleteActor], got %v", calls) } suspends := fakeCtrl.recordedSuspendRequests() if len(suspends) == 0 { t.Fatalf("got no SuspendActor requests") } - if got, want := suspends[len(suspends)-1].GetOptions().GetMode(), ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS; got != want { + if got, want := suspends[len(suspends)-1].GetOptions().GetMode(), ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS; got != want { t.Errorf("suspend mode = %v, want %v", got, want) } } @@ -262,13 +270,19 @@ func TestDurDirShutdownHibernatesBeforeDelete(t *testing.T) { wantMode ateapipb.DurabilityMode }{ { - name: "suspend lifecycle mode", - wantCalls: []string{"SuspendActor", "DeleteActor"}, - wantMode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS, + name: "default lifecycle mode", + wantCalls: []string{"SuspendActor(async)", "DeleteActor"}, + wantMode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS, + }, + { + name: "sync lifecycle mode", + lifecycleMode: dynconfig.LifecycleModeSync, + wantCalls: []string{"SuspendActor", "DeleteActor"}, + wantMode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS, }, { - name: "pause lifecycle mode", - lifecycleMode: dynconfig.LifecycleModePause, + name: "async lifecycle mode", + lifecycleMode: dynconfig.LifecycleModeAsync, wantCalls: []string{"SuspendActor(async)", "DeleteActor"}, wantMode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS, }, diff --git a/internal/benchmarking/boomer/glutton/knobs_test.go b/internal/benchmarking/boomer/glutton/knobs_test.go index f5f28071a8..389ee9196c 100644 --- a/internal/benchmarking/boomer/glutton/knobs_test.go +++ b/internal/benchmarking/boomer/glutton/knobs_test.go @@ -41,14 +41,14 @@ func TestClassCodecs(t *testing.T) { valid: `{ "min_wait_time": 0.1, "max_wait_time": 0.5, "min_live_time": 9, "max_live_time": 14, - "resume_mode": "explicit", "lifecycle_mode": "pause", + "resume_mode": "explicit", "lifecycle_mode": "async", "mem_target": "2Gi", "mem_churn": "512Mi", "mem_read": "all", "cpu_cores": 2, "cpu_duty_cycle": 0.1, "max_pings_per_wake": 3 }`, want: gluttonKnobs{ WaitTime: window, - Lifecycle: dynconfig.Lifecycle{ResumeMode: dynconfig.ResumeModeExplicit, LifecycleMode: dynconfig.LifecycleModePause}, + Lifecycle: dynconfig.Lifecycle{ResumeMode: dynconfig.ResumeModeExplicit, LifecycleMode: dynconfig.LifecycleModeAsync}, MinLive: dynconfig.Seconds(9 * time.Second), MaxLive: dynconfig.Seconds(14 * time.Second), MemTarget: "2Gi", @@ -81,14 +81,14 @@ func TestClassCodecs(t *testing.T) { initial: durDirKnobs{}, valid: `{ "min_wait_time": 0.1, "max_wait_time": 0.5, - "resume_mode": "implicit", "lifecycle_mode": "suspend", + "resume_mode": "implicit", "lifecycle_mode": "sync", "durdir_file_size_bytes": 1048576, "durdir_read_mode": "digest", "durdir_template": "glutton-durdir-data" }`, want: durDirKnobs{ WaitTime: window, - Lifecycle: dynconfig.Lifecycle{ResumeMode: dynconfig.ResumeModeImplicit, LifecycleMode: dynconfig.LifecycleModeSuspend}, + Lifecycle: dynconfig.Lifecycle{ResumeMode: dynconfig.ResumeModeImplicit, LifecycleMode: dynconfig.LifecycleModeSync}, FileSize: 1048576, ReadMode: readModeDigest, Template: "glutton-durdir-data", diff --git a/internal/benchmarking/boomer/glutton/lifecycle.go b/internal/benchmarking/boomer/glutton/lifecycle.go index 985ef5e7f1..812272601a 100644 --- a/internal/benchmarking/boomer/glutton/lifecycle.go +++ b/internal/benchmarking/boomer/glutton/lifecycle.go @@ -252,10 +252,10 @@ func (r *taskRuntime) shutdown(ctx context.Context) { u := val.(*gluttonUser) for _, a := range u.actors { if a.actorRunning { - if dynconfig.Get[gluttonKnobs](r.cfg.Dyn).LifecycleMode == dynconfig.LifecycleModePause { - _ = a.pause(ctx) - } else { + if dynconfig.Get[gluttonKnobs](r.cfg.Dyn).LifecycleMode == dynconfig.LifecycleModeSync { _ = a.suspend(ctx) + } else { + _ = a.pause(ctx) } } a.delete(ctx) @@ -461,10 +461,10 @@ func (u *gluttonActor) hibernate(ctx context.Context) error { switch dynconfig.Get[gluttonKnobs](u.cfg.Dyn).LifecycleMode { case dynconfig.LifecycleModeNone: return nil - case dynconfig.LifecycleModePause: - return u.pause(ctx) - default: + case dynconfig.LifecycleModeSync: return u.suspend(ctx) + default: + return u.pause(ctx) } } diff --git a/internal/benchmarking/boomer/glutton/lifecycle_test.go b/internal/benchmarking/boomer/glutton/lifecycle_test.go index 8a6d71c517..63df97697c 100644 --- a/internal/benchmarking/boomer/glutton/lifecycle_test.go +++ b/internal/benchmarking/boomer/glutton/lifecycle_test.go @@ -35,7 +35,7 @@ func TestGluttonIterate_SuspendMode(t *testing.T) { APIStub: fakeCtrl, Atespace: "bench-test", Dyn: dynconfig.Static(gluttonKnobs{ - Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeSuspend}, + Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeSync}, }), }) @@ -63,7 +63,7 @@ func TestGluttonIterate_PauseMode(t *testing.T) { APIStub: fakeCtrl, Atespace: "bench-test", Dyn: dynconfig.Static(gluttonKnobs{ - Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModePause}, + Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeAsync}, }), }) @@ -104,8 +104,8 @@ func TestGluttonIterate_NoneMode(t *testing.T) { rt.shutdown(context.Background()) afterShutdown := fakeCtrl.recordedCalls() - if len(afterShutdown) < 2 || afterShutdown[len(afterShutdown)-2] != "SuspendActor" || afterShutdown[len(afterShutdown)-1] != "DeleteActor" { - t.Errorf("shutdown in none mode must end with [SuspendActor, DeleteActor], got %v", afterShutdown) + if len(afterShutdown) < 2 || afterShutdown[len(afterShutdown)-2] != "SuspendActor(async)" || afterShutdown[len(afterShutdown)-1] != "DeleteActor" { + t.Errorf("shutdown in none mode must end with [SuspendActor(async), DeleteActor], got %v", afterShutdown) } } @@ -116,7 +116,7 @@ func TestGluttonShutdown_PauseModeRunningActor(t *testing.T) { APIStub: fakeCtrl, Atespace: "bench-test", Dyn: dynconfig.Static(gluttonKnobs{ - Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModePause}, + Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeAsync}, }), }) @@ -143,7 +143,7 @@ func TestGluttonShutdown_StoppedActorOnlyDeleted(t *testing.T) { APIStub: fakeCtrl, Atespace: "bench-test", Dyn: dynconfig.Static(gluttonKnobs{ - Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeSuspend}, + Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeSync}, }), }) @@ -172,7 +172,7 @@ func newReplacementRuntime(t *testing.T, resumeErrs ...error) (*taskRuntime, *fa APIStub: fakeCtrl, Atespace: "bench-test", Dyn: dynconfig.Static(gluttonKnobs{ - Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeSuspend}, + Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeSync}, }), }) return &taskRuntime{cfg: cfg}, fakeCtrl @@ -260,7 +260,7 @@ func TestGluttonIterate_RetriesStrandedHibernate(t *testing.T) { APIStub: fakeCtrl, Atespace: "bench-test", Dyn: dynconfig.Static(gluttonKnobs{ - Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeSuspend}, + Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeSync}, }), }) rt := &taskRuntime{cfg: cfg} diff --git a/internal/benchmarking/boomer/sweperf/knobs_test.go b/internal/benchmarking/boomer/sweperf/knobs_test.go index 4c353cc8e3..4ff06c802d 100644 --- a/internal/benchmarking/boomer/sweperf/knobs_test.go +++ b/internal/benchmarking/boomer/sweperf/knobs_test.go @@ -33,7 +33,7 @@ func TestKnobsCodec(t *testing.T) { "sweperf_total_steps": 21, "sweperf_num_cycles": 4, "sweperf_poll_interval_ms": 100, - "lifecycle_mode": "pause", + "lifecycle_mode": "async", "resume_mode": "explicit" }`)); err != nil { t.Fatalf("Apply(valid): %v", err) @@ -44,7 +44,7 @@ func TestKnobsCodec(t *testing.T) { TotalSteps: 21, NumCycles: 4, PollIntervalMs: 100, - Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModePause, ResumeMode: dynconfig.ResumeModeExplicit}, + Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeAsync, ResumeMode: dynconfig.ResumeModeExplicit}, } if got := dynconfig.Get[knobs](h); got != want { t.Errorf("decoded =\n %+v, want\n %+v", got, want) diff --git a/internal/benchmarking/boomer/sweperf/sweperf.go b/internal/benchmarking/boomer/sweperf/sweperf.go index 0bb7fba77c..07b102eed6 100644 --- a/internal/benchmarking/boomer/sweperf/sweperf.go +++ b/internal/benchmarking/boomer/sweperf/sweperf.go @@ -97,8 +97,8 @@ func init() { // --lifecycle-mode flag (common/lifecycle_mode.py). type knobs struct { dynconfig.WaitTime - // Lifecycle picks how each cycle ends: lifecycle_mode "suspend" (or - // empty) sends a synchronous SuspendActor, "pause" an asynchronous one. + // Lifecycle picks how each cycle ends: lifecycle_mode "async" (or + // empty) sends an asynchronous SuspendActor, "sync" a synchronous one. // "none" and resume_mode "implicit" are refused, since every sweperf // cycle ends in a suspend and resumes with ResumeActor. dynconfig.Lifecycle @@ -616,13 +616,13 @@ func (u *sweperfUser) resume(ctx context.Context) (time.Duration, bool) { } // suspend snapshots the actor and puts it to sleep, ending a cycle. It reports -// whether the suspend succeeded, like resume. In pause lifecycle mode the -// suspend is asynchronous and recorded as SuspendActorAsync, as in the other -// user classes. +// whether the suspend succeeded, like resume. By default (async lifecycle mode) +// the suspend is asynchronous and recorded as SuspendActorAsync; in sync +// lifecycle mode it is synchronous and recorded as SuspendActor. func (u *sweperfUser) suspend(ctx context.Context) bool { - name, mode := "SuspendActor", ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS - if dynconfig.Get[knobs](u.cfg.Dyn).LifecycleMode == dynconfig.LifecycleModePause { - name, mode = "SuspendActorAsync", ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS + name, mode := "SuspendActorAsync", ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS + if dynconfig.Get[knobs](u.cfg.Dyn).LifecycleMode == dynconfig.LifecycleModeSync { + name, mode = "SuspendActor", ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS } err := u.tracedCall(ctx, name, func(callCtx context.Context, tr *metadata.MD) error { _, err := u.cfg.APIStub.SuspendActor(callCtx, &ateapipb.SuspendActorRequest{ diff --git a/internal/benchmarking/boomer/sweperf/sweperf_test.go b/internal/benchmarking/boomer/sweperf/sweperf_test.go index bcd5bff90c..28e68176e4 100644 --- a/internal/benchmarking/boomer/sweperf/sweperf_test.go +++ b/internal/benchmarking/boomer/sweperf/sweperf_test.go @@ -407,8 +407,8 @@ func TestSweperfUserCycleSequence(t *testing.T) { suspendReqs := fakeCtrl.suspendReqs fakeCtrl.mu.Unlock() for _, req := range suspendReqs { - if got := req.GetOptions().GetMode(); got != ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS { - t.Errorf("suspend mode = %v, want %v", got, ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS) + if got := req.GetOptions().GetMode(); got != ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS { + t.Errorf("suspend mode = %v, want %v", got, ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS) } } @@ -428,9 +428,9 @@ func TestSuspendLifecycleMode(t *testing.T) { want ateapipb.DurabilityMode wantMetric string }{ - {"", ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS, "SuspendActor"}, - {dynconfig.LifecycleModeSuspend, ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS, "SuspendActor"}, - {dynconfig.LifecycleModePause, ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS, "SuspendActorAsync"}, + {"", ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS, "SuspendActorAsync"}, + {dynconfig.LifecycleModeSync, ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS, "SuspendActor"}, + {dynconfig.LifecycleModeAsync, ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS, "SuspendActorAsync"}, } { t.Run("mode="+tt.mode, func(t *testing.T) { cfg, _, fakeCtrl := newTestConfig(t, http.NotFoundHandler())