Sync upstream worker registration and snapshot updates

benchmarking/README.md

@@ -288,7 +288,7 @@ through `atenet-router` (`1 dns:///` gRPC channel per VU).
 
 Reuses `glutton`'s runtime and startup knobs:
 `--min-wait-time`, `--max-wait-time`, `--min-live-time`, `--max-live-time`,
-`--max-pings-per-wake`, `--lifecycle-mode` (`suspend` | `pause` | `none`),
+`--max-pings-per-wake`, `--lifecycle-mode` (`sync` | `async` | `none`),
 `--trace-probability`, and `--actors-per-user`.
 
 #### ate-env Reported Metrics
@@ -373,9 +373,9 @@ deployment fails loudly instead of showing up as OOM-flaky steps.
   gap gets ±20% jitter so sessions don't move in lockstep.
 * `--resume-mode implicit|explicit` — implicit (default) lets the parked
   first request wake the actor; explicit issues ResumeActor before traffic.
-* `--lifecycle-mode suspend|pause|none` — synchronous suspend (`suspend`,
-  default, uploads the snapshot before returning), asynchronous suspend
-  (`pause`, uploads the snapshot in the background), or `none` to keep actors
+* `--lifecycle-mode sync|async|none` — synchronous suspend (`sync`,
+  uploads the snapshot before returning), asynchronous suspend (`async`,
+  default, uploads the snapshot in the background), or `none` to keep actors
   resident in memory between steps without hibernating.
 * `--agentsession-snapshot-steps` — comma-separated 1-based step numbers after
   which the actor is hibernated (for example, `3,10,20,30,40,50,61`); between
@@ -658,7 +658,14 @@ fakes from `cmd/benchmarking/isolate/` instead of worker pods:
   time, so it adds to `--delay`, the data plane's share of the call.
   It also relays fake-workersync's capacity reports, as atelet relays ateworker's:
   ate-api-server accepts a report only from the atelet on the Worker's node.
-  The relay admits only fake-workersync's pod identity.
+  The relay admits only fake-workersync's pod identity. fake-atelet remembers
+  the epoch each Worker registers with and reports it from every run and
+  restore, as a worker process does; ate-api-server commits a resumed actor
+  only under its Worker's current epoch. It keeps the epochs in memory only,
+  and fake-workersync registers a Worker once, so a fake-atelet that restarts
+  after its Workers registered, in a DaemonSet rollout for example, fails
+  every resume on those Workers with Unavailable until they are replaced,
+  such as by redeploying the benchmark.
 
 The install also scales ate-controller to zero, whose worker syncer would
 delete the fake Workers. Its other controllers stop with it: no NetworkPolicy

benchmarking/automation/tests.yaml

@@ -149,6 +149,8 @@ tests:
     users: 1
     workerCount: 1
     flags:
+      - "--lifecycle-mode"
+      - "sync"
       - "--min-wait-time"
       - "1.0"
       - "--max-wait-time"
@@ -163,7 +165,7 @@ tests:
     workerCount: 1
     flags:
       - "--lifecycle-mode"
-      - "pause"
+      - "async"
       - "--min-wait-time"
       - "1.0"
       - "--max-wait-time"
@@ -191,6 +193,8 @@ tests:
     workerCount: 1
     actorMemory: "1536Mi"
     flags:
+      - "--lifecycle-mode"
+      - "sync"
       - "--mem-target"
       - "1Gi"
       - "--mem-churn"
@@ -212,7 +216,7 @@ tests:
     actorMemory: "1536Mi"
     flags:
       - "--lifecycle-mode"
-      - "pause"
+      - "async"
       - "--mem-target"
       - "1Gi"
       - "--mem-churn"
@@ -254,6 +258,8 @@ tests:
     workerCount: 1
     actorMemory: "2560Mi"
     flags:
+      - "--lifecycle-mode"
+      - "sync"
       - "--mem-target"
       - "2Gi"
       - "--mem-churn"
@@ -275,7 +281,7 @@ tests:
     actorMemory: "2560Mi"
     flags:
       - "--lifecycle-mode"
-      - "pause"
+      - "async"
       - "--mem-target"
       - "2Gi"
       - "--mem-churn"
@@ -316,6 +322,8 @@ tests:
     users: 5
     workerCount: 5
     flags:
+      - "--lifecycle-mode"
+      - "sync"
       - "--min-wait-time"
       - "1.0"
       - "--max-wait-time"
@@ -330,7 +338,7 @@ tests:
     workerCount: 5
     flags:
       - "--lifecycle-mode"
-      - "pause"
+      - "async"
       - "--min-wait-time"
       - "1.0"
       - "--max-wait-time"
@@ -344,6 +352,8 @@ tests:
     users: 10
     workerCount: 10
     flags:
+      - "--lifecycle-mode"
+      - "sync"
       - "--min-wait-time"
       - "1.0"
       - "--max-wait-time"
@@ -358,7 +368,7 @@ tests:
     workerCount: 10
     flags:
       - "--lifecycle-mode"
-      - "pause"
+      - "async"
       - "--min-wait-time"
       - "1.0"
       - "--max-wait-time"
@@ -456,6 +466,8 @@ tests:
     flags:
       - "--durdir-template"
       - "glutton-durdir-data"
+      - "--lifecycle-mode"
+      - "sync"
       - "--resume-mode"
       - "explicit"
       - "--durdir-file-size-bytes"
@@ -476,7 +488,7 @@ tests:
       - "--durdir-template"
       - "glutton-durdir-data"
       - "--lifecycle-mode"
-      - "pause"
+      - "async"
       - "--resume-mode"
       - "explicit"
       - "--durdir-file-size-bytes"
@@ -496,6 +508,8 @@ tests:
     flags:
       - "--durdir-template"
       - "glutton-durdir-full"
+      - "--lifecycle-mode"
+      - "sync"
       - "--resume-mode"
       - "explicit"
       - "--durdir-file-size-bytes"
@@ -516,7 +530,7 @@ tests:
       - "--durdir-template"
       - "glutton-durdir-full"
       - "--lifecycle-mode"
-      - "pause"
+      - "async"
       - "--resume-mode"
       - "explicit"
       - "--durdir-file-size-bytes"
@@ -597,6 +611,8 @@ tests:
     flags:
       - "--durdir-template"
       - "glutton-durdir-data"
+      - "--lifecycle-mode"
+      - "sync"
       - "--resume-mode"
       - "explicit"
       - "--durdir-read-mode"
@@ -619,7 +635,7 @@ tests:
       - "--durdir-template"
       - "glutton-durdir-data"
       - "--lifecycle-mode"
-      - "pause"
+      - "async"
       - "--resume-mode"
       - "explicit"
       - "--durdir-read-mode"

benchmarking/locust/common/lifecycle_mode.py

@@ -24,9 +24,9 @@ def add_lifecycle_mode_arguments(parser: LocustArgumentParser) -> None:
     group.add_argument(
         "--lifecycle-mode",
         type=str,
-        default="suspend",
-        choices=["suspend", "pause", "none"],
-        help="Lifecycle mode: 'suspend' uses synchronous SuspendActor (uploads snapshot before returning); "
-        "'pause' uses asynchronous SuspendActor (uploads snapshot in background); "
-        "'none' keeps actors resident in memory between steps (default: suspend)",
+        default="async",
+        choices=["sync", "async", "none"],
+        help="Lifecycle mode: 'sync' uses synchronous SuspendActor (uploads snapshot before returning); "
+        "'async' uses asynchronous SuspendActor (uploads snapshot in background); "
+        "'none' keeps actors resident in memory between steps (default: async)",
     )

benchmarking/locust/unit_tests/test_boomer_config.py

@@ -39,12 +39,12 @@ def test_build_config_json_lifecycle_mode_none(self):
     def test_build_config_json_snapshot_steps(self):
         raw = boomer_config.build_config_json([
             "--lifecycle-mode",
-            "suspend",
+            "sync",
             "--agentsession-snapshot-steps",
             "3,10,20,30,40,50,61",
         ])
         parsed = json.loads(raw)
-        self.assertEqual(parsed["lifecycle_mode"], "suspend")
+        self.assertEqual(parsed["lifecycle_mode"], "sync")
         self.assertEqual(
             parsed["agentsession_snapshot_steps"], "3,10,20,30,40,50,61"
         )

charts/substrate/README.md

@@ -122,6 +122,12 @@ verifying and publishing them. `ATE_STORAGE_BACKEND` must not be set on the
 API server or atelet itself; select it through `atelet.storageBackend`.
 RustFS is wired automatically. Private S3 asset URLs must use `s3://`.
 
+GCS snapshot transfers now require `objectStorePlugin.transferServiceAccount`,
+the email of a dedicated Google service account. Grant it object access to the
+snapshot bucket, and let the API server's identity impersonate it. See
+[the GCP setup guide](../../tools/setup-gcp/README.md#what-create-iam-actually-grants)
+for the required IAM grants. S3 and RustFS do not need this setting.
+
 The default gVisor config now uses the 2026-10-17 nightly (`v4`) for separate
 container-rootfs and durable-directory checkpoint bundles. Upgrade the sandbox config together with ateworker-gvisor;
 custom configs must pin a release with the same checkpoint support and supply

charts/substrate/templates/ate-api-server-envvars.yaml

@@ -34,5 +34,6 @@ metadata:
   namespace: {{ .Release.Namespace }}
 data:
   ATE_API_SNAPSHOT_LOCATION: {{ $snapshotLocation | quote }}
+  ATE_API_SNAPSHOT_TRANSFER_SERVICE_ACCOUNT: {{ .Values.objectStorePlugin.transferServiceAccount | quote }}
   ATE_API_POSTGRES_SCHEMA: {{ .Values.postgres.schema | quote }}
   ATE_API_ACTOR_JWT_ISSUER: {{ include "substrate.actorJWTIssuer" . | quote }}

charts/substrate/templates/ate-api-server.yaml

@@ -94,6 +94,7 @@ spec:
         - --socket=/run/objectstore-plugin/control.sock
         - --exchange-listen=:8445
         - --actor-jwt-issuer=$(ATE_API_ACTOR_JWT_ISSUER)
+        - --transfer-service-account=$(ATE_API_SNAPSHOT_TRANSFER_SERVICE_ACCOUNT)
         - '--ateapi-server-name={{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc'
         ports:
         - name: exchange
@@ -117,6 +118,11 @@ spec:
             configMapKeyRef:
               name: {{ .Values.ateApiServerEnvVarsConfigMap }}
               key: ATE_API_ACTOR_JWT_ISSUER
+        - name: ATE_API_SNAPSHOT_TRANSFER_SERVICE_ACCOUNT
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Values.ateApiServerEnvVarsConfigMap }}
+              key: ATE_API_SNAPSHOT_TRANSFER_SERVICE_ACCOUNT
         - name: POD_NAMESPACE
           valueFrom:
             fieldRef:

charts/substrate/tests/snapshot_exchange_test.yaml

@@ -30,15 +30,22 @@ tests:
   - equal:
       path: data.ATE_API_SNAPSHOT_LOCATION
       value: gs://custom-snapshots/
+  - equal:
+      path: data.ATE_API_SNAPSHOT_TRANSFER_SERVICE_ACCOUNT
+      value: ""
 - it: uses an explicit external snapshot location
   template: ate-api-server-envvars.yaml
   set:
     rustfs.enabled: false
     ateApi.snapshotLocation: gs://external/prefix/
+    objectStorePlugin.transferServiceAccount: snapshot-transfer@example.iam.gserviceaccount.com
   asserts:
   - equal:
       path: data.ATE_API_SNAPSHOT_LOCATION
       value: gs://external/prefix/
+  - equal:
+      path: data.ATE_API_SNAPSHOT_TRANSFER_SERVICE_ACCOUNT
+      value: snapshot-transfer@example.iam.gserviceaccount.com
 - it: requires a snapshot location for external storage
   template: ate-api-server-envvars.yaml
   set:
@@ -73,6 +80,9 @@ tests:
   - contains:
       path: spec.template.spec.initContainers[0].args
       content: --actor-jwt-issuer=$(ATE_API_ACTOR_JWT_ISSUER)
+  - contains:
+      path: spec.template.spec.initContainers[0].args
+      content: --transfer-service-account=$(ATE_API_SNAPSHOT_TRANSFER_SERVICE_ACCOUNT)
   - contains:
       path: spec.template.spec.initContainers[0].args
       content: --ateapi-server-name=custom-api.custom-system.svc
@@ -84,6 +94,14 @@ tests:
           configMapKeyRef:
             name: ate-api-server-envvars
             key: ATE_API_ACTOR_JWT_ISSUER
+  - contains:
+      path: spec.template.spec.initContainers[0].env
+      content:
+        name: ATE_API_SNAPSHOT_TRANSFER_SERVICE_ACCOUNT
+        valueFrom:
+          configMapKeyRef:
+            name: ate-api-server-envvars
+            key: ATE_API_SNAPSHOT_TRANSFER_SERVICE_ACCOUNT
   - contains:
       path: spec.template.spec.initContainers[0].volumeMounts
       content:

charts/substrate/values.yaml

@@ -96,6 +96,9 @@ atelet:
 
 # Object storage credentials/settings for snapshot transfers and private sandbox assets.
 objectStorePlugin:
+  # Required for GCS: the Google service account the exchange impersonates
+  # for actor-scoped snapshot transfers. Unused by S3/RustFS.
+  transferServiceAccount: ""
   extraEnv: []
   control:
     resources: {}

cmd/ate-setup/internal/config/config.go

@@ -139,6 +139,11 @@ type Config struct {
 	// template's snapshotConfig.storageLocation override it.
 	SnapshotLocation string
 
+	// TransferGSAName is the name of the Google service account snapshot
+	// transfers run as on GCS, in ProjectID (SNAPSHOT_TRANSFER_GSA_NAME).
+	// Only the in-tree GCS backend uses it. See TransferServiceAccount.
+	TransferGSAName string
+
 	// KODockerRepo is where ko pushes images (KO_DOCKER_REPO).
 	KODockerRepo string
 	// KODefaultPlatforms constrains ko's build platforms.
@@ -283,6 +288,33 @@ func (c *Config) RequireIngressAllowedSPIFFEIDs() error {
 	return c.Resolved().Require("ingressAllowedSPIFFEIDs")
 }
 
+// TransferServiceAccount is the email of the snapshot transfer service
+// account, built from TransferGSAName and ProjectID as tools/setup-gcp builds
+// it. It is empty when either is.
+func (c *Config) TransferServiceAccount() string {
+	if c.TransferGSAName == "" || c.ProjectID == "" {
+		return ""
+	}
+	return c.TransferGSAName + "@" + c.ProjectID + ".iam.gserviceaccount.com"
+}
+
+// RequireTransferServiceAccount fails when the install would deploy
+// ate-api-server on GCS without a transfer service account. Its credential
+// exchange impersonates the account and refuses to start without one.
+// ate-setup installs GCS everywhere but kind, which uses S3.
+func (c *Config) RequireTransferServiceAccount() error {
+	if c.Kind {
+		return nil
+	}
+	if c.TransferGSAName == "" {
+		return c.Resolved().Require("storage.transferGSAName")
+	}
+	if c.ProjectID == "" {
+		return c.Resolved().Require("gcp.projectID")
+	}
+	return nil
+}
+
 // SetResolved attaches resolved settings to a Config built directly. Load
 // does this itself; the callers here are tests and the demo test harness.
 func (c *Config) SetResolved(r *Resolved) { c.resolved = r }