From d77cfeb3638378af01bbf2ffce82f9d1eaa080d3 Mon Sep 17 00:00:00 2001 From: Chenyi Wang Date: Tue, 20 Oct 2026 16:31:37 +0000 Subject: [PATCH 01/63] objectstorage: size small snapshot upload buffers to actual data (#2945) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Follow-up to #2848 (and #2847). While #2848 fixed upload memory accounting for multi-part ranged sparse files, small single-request files (`< 64 MiB`) still allocated and reserved worst-case `64 MiB` buffers at three layers (`64 MiB` GCS lookahead + `64 MiB` GCS chunk buffer + `64 MiB` zstd worker = `192 MiB` per file, and `128 MiB` for `manifest.json`). Under the default `--upload-memory-budget=512Mi`, a small gVisor snapshot (~7 files + `manifest.json`, ~17 MB total) still required `> 1.2 GiB` of budget reservations, forcing concurrent `SuspendActor` uploads to queue behind the semaphore (2–3 files at a time). ### Changes - **Proportional GCS single-request buffers**: When the upload stream has a known size bound below `64 MiB`, size both the `PutObject` lookahead buffer and the single-request `Writer.ChunkSize` to the actual object size (rounded up to `256 KiB` multiples, `512 KiB` minimum total instead of `128 MiB`). - **Right-sized zstd compression chunks**: For sparse files with `< 8 MiB` of populated data, scale the per-frame compression chunk and zstd window down to a power of two in `[1 MiB, 8 MiB]`, reducing per-worker memory from `64 MiB` down to as low as `8 MiB`. - **Budget accounting**: Update `Uploader` planning to reserve these proportional buffer sizes (`8.5 MiB` minimum for small sparse files instead of `192 MiB`), allowing all files in a snapshot burst to upload concurrently within the default `512Mi` budget. Large files (`>= 64 MiB`) and ranged sparse uploads (`parts >= 2`) are unchanged. ## Test Plan ### 1. GKE Cluster Benchmark (`glutton_baseline_10_users`, 10 gVisor workers, default `512Mi` budget) | Metric (`grpc_SuspendActor`) | Oct 15 Baseline (pre-#2708) | `main` (`90de075f`, with #2848) | This PR | Change vs `main` | | :--- | :---: | :---: | :---: | :---: | | **p50** | 400 ms | 850 ms | **390 ms** | **-54.1%** | | **p75** | — | 1,000 ms | **420 ms** | **-58.0%** | | **p90** | 460 ms | 1,400 ms | **450 ms** | **-67.9%** | | **p95** | — | 1,600 ms | **520 ms** | **-67.5%** | | **p99** | 610 ms | 1,700 ms | **640 ms** | **-62.4%** | | **Average** | — | 905.4 ms | **414.3 ms** | **-54.2%** | | **Throughput** | — | 4.39 req/s | **5.73 req/s** | **+30.5%** | ### 2. Microbenchmarks (`pkg/objectstorage/upload_bench_test.go`) | Benchmark (`512 MiB` budget, 50 ms RTT) | `main` (#2848) | This PR | Speedup / Memory | | :--- | :---: | :---: | :---: | | `BenchmarkConcurrentSnapshotUploads/gcs_shared_512MiB` (80 small files) | 2,085 ms / 5,375 MiB | **73.6 ms / 83.2 MiB** | **28.4x faster, -98.5% B/op** | | `BenchmarkLargeSparseUpload/4_actors_gvisor_7files_each` (4× [256 MiB ranged + 6× 64 KiB]) | 905.5 ms / 1,613 MiB | **358.4 ms / 138 MiB** | **2.5x faster, -91.4% B/op** | | `BenchmarkLargeSparseUpload/1_actor_256MiB_ranged` (large file only) | 115.6 ms | **114.6 ms** | Unchanged | Part of #2553 --- cmd/objectstore-plugin/README.md | 12 +- internal/objectstoreplugin/upload_test.go | 15 +- pkg/objectstorage/gcs.go | 70 ++++++-- pkg/objectstorage/objects.go | 23 ++- pkg/objectstorage/parzstd.go | 61 ++++++- pkg/objectstorage/sparsezstd.go | 50 +++--- pkg/objectstorage/upload.go | 42 +++-- pkg/objectstorage/upload_bench_test.go | 190 +++++++++++++++++++++- pkg/objectstorage/upload_gcs_test.go | 178 ++++++++++++++++++++ pkg/objectstorage/upload_test.go | 16 +- 10 files changed, 561 insertions(+), 96 deletions(-) diff --git a/cmd/objectstore-plugin/README.md b/cmd/objectstore-plugin/README.md index a81ca91f2e..ce02d4c5c1 100644 --- a/cmd/objectstore-plugin/README.md +++ b/cmd/objectstore-plugin/README.md @@ -68,11 +68,13 @@ in memory. The sparse format and download readers are unchanged. With the current buffer sizes, the minimum budget is `384Mi` for GCS and `224Mi` for S3. Startup rejects smaller values. Each upload reserves for the path it -takes: on GCS, a file split into range parts reserves only those parts, and an -object whose sparse stream is bounded below one request (`64Mi`) reserves that -request's buffers instead of the composite-upload buffers. Other streamed -objects still reserve for a composite upload, even when they ultimately fit in -one request. Raise the budget to trade memory for more simultaneous uploads. +takes: on GCS, a file split into range parts reserves only those parts, sparse +files with less than `8Mi` of populated data scale their compression chunk +buffers down to `1Mi`, and an object whose stream is bounded below one request +(`64Mi`) sizes both its lookahead and single-request chunk buffer to that +bound instead of the composite-upload buffers. Unbounded streamed objects still +reserve for a composite upload, even when they ultimately fit in one request. +Raise the budget to trade memory for more simultaneous uploads. The budget is **not a process memory limit**. The Go runtime, garbage awaiting collection, connections, downloads, and asset transfers need additional memory. diff --git a/internal/objectstoreplugin/upload_test.go b/internal/objectstoreplugin/upload_test.go index cbec1a68d8..2fb8f19151 100644 --- a/internal/objectstoreplugin/upload_test.go +++ b/internal/objectstoreplugin/upload_test.go @@ -91,8 +91,9 @@ func TestUploadBudgetSharedAcrossSnapshots(t *testing.T) { synctest.Test(t, func(t *testing.T) { gate := make(chan struct{}) store := &slowSnapshotStore{memObjects: newMemObjects(), release: gate} - // This backend has no SDK buffers. Two compression reservations fit. - node, err := NewNodePlugin(store, root, 128<<20) + // This backend has no SDK buffers. Eight small-file compression + // reservations (8 MiB each) fit in the 64 MiB minimum budget. + node, err := NewNodePlugin(store, root, 64<<20) if err != nil { t.Fatal(err) } @@ -101,8 +102,8 @@ func TestUploadBudgetSharedAcrossSnapshots(t *testing.T) { go func() { _, err := node.UploadSnapshot(t.Context(), req); done <- err }() } synctest.Wait() - if got := store.active.Load(); got != 2 { - t.Errorf("active uploads = %d, want 2 across all requests", got) + if got := store.active.Load(); got != 8 { + t.Errorf("active uploads = %d, want 8 across all requests", got) } if got := store.manifests.Load(); got != 0 { t.Errorf("committed %d manifests before data finished", got) @@ -114,8 +115,8 @@ func TestUploadBudgetSharedAcrossSnapshots(t *testing.T) { t.Fatal(err) } } - if got := store.peak.Load(); got != 2 { - t.Errorf("peak uploads = %d, want 2", got) + if got := store.peak.Load(); got != 8 { + t.Errorf("peak uploads = %d, want 8", got) } if got := len(store.keys()); got != snapshots*(filesPerSnapshot+1) { t.Fatalf("uploaded %d objects", got) @@ -170,7 +171,7 @@ func TestUploadFailureDoesNotCommitManifest(t *testing.T) { func TestUploadCanceledWaiterReturnsCanceled(t *testing.T) { root := t.TempDir() dir := filepath.Join(root, "src") - files := snapshotUploadFiles(t, dir, 1) + files := snapshotUploadFiles(t, dir, 8) synctest.Test(t, func(t *testing.T) { gate := make(chan struct{}) store := &slowSnapshotStore{memObjects: newMemObjects(), release: gate} diff --git a/pkg/objectstorage/gcs.go b/pkg/objectstorage/gcs.go index 327af424f5..1445f406a1 100644 --- a/pkg/objectstorage/gcs.go +++ b/pkg/objectstorage/gcs.go @@ -105,38 +105,74 @@ func setRetry(c *storage.Client) { // the signal. func (g *gcsClient) supportsStreamingPut() {} -// uploadChunkSize is how much of a streamed object the GCS client buffers before -// starting a request. Each chunk costs a round trip, so a snapshot that fits in one -// chunk pays only one: measured on a GKE worker node, a 24 MiB object took 425-530ms -// at the 16 MiB default versus 258-314ms at 64 MiB. The buffer is capped by the object -// size, so small objects still cost only their own bytes. -const uploadChunkSize = 64 << 20 +// boundedReader wraps an io.Reader whose remaining length is at most max bytes. +type boundedReader struct { + io.Reader + max int64 +} + +func (b boundedReader) sizeBound() int64 { return b.max } +func (b boundedReader) Unwrap() io.Reader { return b.Reader } + +// readerSizeBound returns the upper bound on r's remaining length if r or an +// unwrapped inner reader reports one via sizeBound(). +func readerSizeBound(r io.Reader) (int64, bool) { + for r != nil { + if b, ok := r.(interface{ sizeBound() int64 }); ok { + return b.sizeBound(), true + } + u, ok := r.(interface{ Unwrap() io.Reader }) + if !ok { + return 0, false + } + r = u.Unwrap() + } + return 0, false +} + +// singleUploadChunkSize returns the Writer.ChunkSize that sends an object of at +// most size bytes in one request. googleapi rounds ChunkSize up to a multiple of +// MinUploadChunkSize (256 KiB), allocates a retry buffer of that size, and sends +// media strictly smaller than ChunkSize in a single request. +func singleUploadChunkSize(size int64) int64 { + const unit = int64(googleapi.MinUploadChunkSize) + return ((max(size, 0) + unit) / unit) * unit +} + +// singleUploadLookahead returns the PutObject lookahead size for reader. +func singleUploadLookahead(reader io.Reader) int64 { + if bound, ok := readerSizeBound(reader); ok && bound >= 0 && bound < uploadCompositeMin { + return singleUploadChunkSize(bound) + } + return uploadCompositeMin +} // PutObject writes reader to the object, in a single request below -// uploadCompositeMin and as parallel parts above it. The size is not known up front, -// so this reads that many bytes to find out which case it is, then hands them on. +// uploadCompositeMin and as parallel parts above it. When reader carries an +// upper bound below uploadCompositeMin, only that request's chunk size is +// allocated for the lookahead instead of the full uploadCompositeMin. func (g *gcsClient) PutObject(ctx context.Context, bucket, object string, reader io.Reader) error { - head := make([]byte, uploadCompositeMin) + head := make([]byte, singleUploadLookahead(reader)) n, err := io.ReadFull(reader, head) switch { case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF): // The whole object is in hand and fits in one request. - return g.putSingle(ctx, bucket, object, bytes.NewReader(head[:n])) + return g.putSingle(ctx, bucket, object, head[:n]) case err != nil: return fmt.Errorf("while reading object body: %w", err) } return g.putComposite(ctx, bucket, object, bytes.NewReader(head[:n]), reader) } -// putSingle writes the whole body in one resumable request, on a pooled client so -// the small objects a node uploads concurrently do not queue on one connection. -func (g *gcsClient) putSingle(ctx context.Context, bucket, object string, reader io.Reader) error { +// putSingle writes body in one request, on a pooled client so the small objects +// a node uploads concurrently do not queue on one connection. +func (g *gcsClient) putSingle(ctx context.Context, bucket, object string, body []byte) error { wc := g.nextClient().Bucket(bucket).Object(object).NewWriter(ctx) - wc.ChunkSize = uploadChunkSize - // io.Copy reports local read errors; wc.Close() reports the actual - // GCS upload (auth, permissions, transient). Join both so the caller + wc.ChunkSize = int(singleUploadChunkSize(int64(len(body)))) + // Write reports local pipe errors; wc.Close() reports the actual GCS + // upload (auth, permissions, transient). Join both so the caller // doesn't lose either. - _, copyErr := io.Copy(wc, reader) + _, copyErr := wc.Write(body) closeErr := wc.Close() if err := errors.Join(copyErr, closeErr); err != nil { return fmt.Errorf("while putting GCS object: %w", err) diff --git a/pkg/objectstorage/objects.go b/pkg/objectstorage/objects.go index c7c696df97..e1827b32b3 100644 --- a/pkg/objectstorage/objects.go +++ b/pkg/objectstorage/objects.go @@ -184,6 +184,13 @@ type writeContentResult struct { // plain zstd stream otherwise. It touches only io, so it is unit-testable without // an object store, and is shared by the buffered and streaming upload paths. func writeContent(ctx context.Context, out io.Writer, content io.Reader, workers int) (writeContentResult, error) { + if sf, ok := content.(*sparseFile); ok { + logical, populated, err := writeSparseExtents(ctx, out, sf.File, sf.size, sf.exts, sf.populated, workers) + if err != nil { + return writeContentResult{}, err + } + return writeContentResult{logicalBytes: logical, populatedBytes: populated, sparse: true}, nil + } if f, ok := content.(*os.File); ok { logical, populated, err := writeSparseZstd(ctx, out, f, workers) if err != nil { @@ -223,7 +230,7 @@ func (u *Uploader) sendZstd(ctx context.Context, gsURL string, content io.Reader if err != nil { return fmt.Errorf("while parsing URL: %w", err) } - plan, err := u.plan(content) + plan, streamMax, plannedContent, err := u.planStream(content) if err != nil { return err } @@ -232,7 +239,7 @@ func (u *Uploader) sendZstd(ctx context.Context, gsURL string, content io.Reader } defer u.budget.reserved.Release(plan.memory) tStart := time.Now() - if sp, ok := client.(sparseFilePutter); ok { + if sp, ok := client.(sparseFilePutter); ok && plan.parts >= 2 { if f, isFile := content.(*os.File); isFile { res, err := sp.PutSparseFile(ctx, bucket, object, f, plan.parts) switch { @@ -254,9 +261,9 @@ func (u *Uploader) sendZstd(ctx context.Context, gsURL string, content io.Reader } } if _, ok := client.(streamingPutter); ok { - return sendStreamingZstd(ctx, client, bucket, object, content, tStart, plan.workers) + return sendStreamingZstd(ctx, client, bucket, object, plannedContent, tStart, plan.workers, streamMax) } - return sendBufferedZstd(ctx, client, bucket, object, content, tStart, plan.workers) + return sendBufferedZstd(ctx, client, bucket, object, plannedContent, tStart, plan.workers) } // sendBufferedZstd compresses content to a seekable temp file, then uploads it. @@ -295,7 +302,7 @@ func sendBufferedZstd(ctx context.Context, client ObjectStorage, bucket, object // PutObject streams the read end to the object store. No seekable temp file, and // the compress runs concurrently with the network PUT. Used only for streaming // backends (GCS and S3); see sendZstd. -func sendStreamingZstd(ctx context.Context, client ObjectStorage, bucket, object string, content io.Reader, tStart time.Time, workers int) error { +func sendStreamingZstd(ctx context.Context, client ObjectStorage, bucket, object string, content io.Reader, tStart time.Time, workers int, streamMax int64) error { ctx, cancel := context.WithCancel(ctx) defer cancel() type result struct { @@ -311,7 +318,11 @@ func sendStreamingZstd(ctx context.Context, client ObjectStorage, bucket, object ch <- result{res: res, err: err} }() - putErr := client.PutObject(ctx, bucket, object, pr) + var body io.Reader = pr + if streamMax >= 0 { + body = boundedReader{Reader: pr, max: streamMax} + } + putErr := client.PutObject(ctx, bucket, object, body) if putErr != nil { cancel() // PutObject bailed (e.g. mid-stream); unblock the compressor goroutine so it diff --git a/pkg/objectstorage/parzstd.go b/pkg/objectstorage/parzstd.go index 7802703d36..4a0157b683 100644 --- a/pkg/objectstorage/parzstd.go +++ b/pkg/objectstorage/parzstd.go @@ -35,8 +35,11 @@ import ( // Independent chunks cost a little ratio (no matches across a boundary), which // parZstdChunk is sized to keep negligible. const ( - // parZstdChunk is the per-frame chunk size. + // parZstdChunk is the maximum per-frame chunk size. parZstdChunk = 8 << 20 + // parZstdMinChunk is the smallest per-frame chunk size, covering a fast + // encoder's fixed tables, block buffers, and minimum history. + parZstdMinChunk = 1 << 20 // parZstdQueue is how many chunks may be outstanding per worker, so the producer // can run ahead while workers compress. parZstdQueue = 2 @@ -57,18 +60,43 @@ var ( parZstdFramePool = sync.Pool{New: func() any { b := make([]byte, 0, parZstdFrameCap); return &b }} ) +// parZstdChunkSize returns the per-frame chunk size for a sparse file with +// populated data bytes, as a power of two between parZstdMinChunk and parZstdChunk. +func parZstdChunkSize(populated int64) int64 { + chunk := int64(parZstdMinChunk) + for chunk < parZstdChunk && chunk < populated { + chunk <<= 1 + } + return chunk +} + +// parZstdWorkerMemory returns the memory reservation per worker for a given +// frame chunk size. +func parZstdWorkerMemory(chunk int64) int64 { + return (parZstdQueue*3 + 2) * chunk +} + // newChunkEncoder returns an encoder for EncodeAll use. Single-segment frames always // declare their content size, even the short last frame of a stream (zstd otherwise // leaves it out under 256 bytes), so a reader can size every frame without having // to bound it; a chunk is at most parZstdChunk, so the frame never needs a larger // window than that. func newChunkEncoder() *zstd.Encoder { - enc, err := zstd.NewWriter(nil, + return newChunkEncoderSized(parZstdChunk) +} + +func newChunkEncoderSized(chunk int64) *zstd.Encoder { + opts := []zstd.EOption{ zstd.WithEncoderLevel(zstd.SpeedFastest), zstd.WithEncoderConcurrency(1), - zstd.WithSingleSegment(true)) + zstd.WithSingleSegment(true), + } + if chunk > 0 && chunk < 4<<20 { + opts = append(opts, zstd.WithWindowSize(int(max(chunk, parZstdMinChunk)))) + } + enc, err := zstd.NewWriter(nil, opts...) if err != nil { - // NewWriter only fails on bad options, which are compile-time constants here. + // NewWriter only fails on bad options, which are bounded powers of two here. panic(err) } return enc @@ -77,6 +105,13 @@ func newChunkEncoder() *zstd.Encoder { func getChunk() []byte { return (*parZstdChunkPool.Get().(*[]byte))[:0] } func getFrame() []byte { return (*parZstdFramePool.Get().(*[]byte))[:0] } +func getChunkSized(chunk int64) []byte { + if chunk == parZstdChunk { + return getChunk() + } + return make([]byte, 0, chunk) +} + func putChunk(b []byte) { if cap(b) == parZstdChunk { b = b[:0] @@ -107,6 +142,7 @@ func putFrame(b []byte) { type parZstd struct { dst io.Writer workers int + chunk int64 buf []byte free chan []byte @@ -131,28 +167,37 @@ type parZstdJob struct { // One worker, after the cap, means inline mode with no goroutines, so with // GOMAXPROCS=1 every caller compresses inline. func newParZstd(dst io.Writer, workers int) *parZstd { + return newParZstdChunk(dst, workers, parZstdChunk) +} + +func newParZstdChunk(dst io.Writer, workers int, chunk int64) *parZstd { if workers <= 0 || workers > runtime.GOMAXPROCS(0) { workers = runtime.GOMAXPROCS(0) } + if chunk <= 0 || chunk > parZstdChunk { + chunk = parZstdChunk + } if workers == 1 { return &parZstd{ dst: dst, workers: 1, + chunk: chunk, inline: true, - enc: newChunkEncoder(), - buf: getChunk(), + enc: newChunkEncoderSized(chunk), + buf: getChunkSized(chunk), } } p := &parZstd{ dst: dst, workers: workers, + chunk: chunk, free: make(chan []byte, workers*parZstdQueue), jobs: make(chan parZstdJob, workers), ordered: make(chan chan []byte, workers*parZstdQueue), done: make(chan struct{}), } for range workers * parZstdQueue { - p.free <- getChunk() + p.free <- getChunkSized(chunk) } for range workers { p.workerWG.Go(p.worker) @@ -165,7 +210,7 @@ func newParZstd(dst io.Writer, workers int) *parZstd { // worker compresses whole chunks. Each holds its own encoder: the encoders are // single-shot EncodeAll users, so one per worker keeps their state private. func (p *parZstd) worker() { - enc := newChunkEncoder() + enc := newChunkEncoderSized(p.chunk) defer enc.Close() for j := range p.jobs { // Only a full chunk takes a pooled frame. A partial one (a small file, or diff --git a/pkg/objectstorage/sparsezstd.go b/pkg/objectstorage/sparsezstd.go index 05b18643e0..d5e59f0ae6 100644 --- a/pkg/objectstorage/sparsezstd.go +++ b/pkg/objectstorage/sparsezstd.go @@ -23,7 +23,6 @@ import ( "os" "github.com/klauspost/compress/zstd" - "golang.org/x/sys/unix" ) // sparseMagic marks the sparse-extent snapshot format (see writeSparseZstd). It is @@ -44,6 +43,16 @@ const sparseVersion uint32 = 2 // and the reader stops when it sees the sentinel. const sparseEndOffset int64 = -1 +// sparseFile holds a pre-scanned sparse file's logical size, extents, and +// populated byte count so planning and compression share one SEEK_DATA/SEEK_HOLE +// pass and use the same chunk size. +type sparseFile struct { + *os.File + size int64 + exts []extentRange + populated int64 +} + // writeSparseZstd encodes a sparse file src to dst in the sparse-extent format: // // magic[8] | version:u32 | zstd( totalSize:i64 | (off:i64, len:i64, data[len])* | -1:i64 ) @@ -66,7 +75,17 @@ func writeSparseZstd(ctx context.Context, dst io.Writer, src *os.File, workers i return 0, 0, err } size := fi.Size() + exts, populated, err := sparseExtents(src, size) + if err != nil { + return 0, 0, err + } + if _, err := src.Seek(0, io.SeekStart); err != nil { + return 0, 0, err + } + return writeSparseExtents(ctx, dst, src, size, exts, populated, workers) +} +func writeSparseExtents(ctx context.Context, dst io.Writer, src *os.File, size int64, exts []extentRange, populated int64, workers int) (logical, dataBytes int64, err error) { // magic + version in the clear (buffered: a couple of tiny writes). bw := bufio.NewWriter(dst) if _, err := bw.WriteString(sparseMagic); err != nil { @@ -79,7 +98,7 @@ func writeSparseZstd(ctx context.Context, dst io.Writer, src *os.File, workers i return 0, 0, err } - zw := newParZstd(dst, workers) + zw := newParZstdChunk(dst, workers, parZstdChunkSize(populated)) cw := contextWriter{ctx: ctx, dst: zw} // fail closes the encoder before returning err (Close flushes/frees state). fail := func(e error) (int64, int64, error) { @@ -90,36 +109,21 @@ func writeSparseZstd(ctx context.Context, dst io.Writer, src *os.File, workers i return fail(err) } - fd := int(src.Fd()) - off := int64(0) - for off < size { - ds, serr := unix.Seek(fd, off, unix.SEEK_DATA) - if serr != nil { - if serr == unix.ENXIO { // no more data: the rest is a hole - break - } - return fail(fmt.Errorf("SEEK_DATA: %w", serr)) - } - de, serr := unix.Seek(fd, ds, unix.SEEK_HOLE) - if serr != nil { - return fail(fmt.Errorf("SEEK_HOLE: %w", serr)) - } - length := de - ds - if err := binary.Write(cw, binary.LittleEndian, ds); err != nil { + for _, e := range exts { + if err := binary.Write(cw, binary.LittleEndian, e.off); err != nil { return fail(err) } - if err := binary.Write(cw, binary.LittleEndian, length); err != nil { + if err := binary.Write(cw, binary.LittleEndian, e.length); err != nil { return fail(err) } - if _, err := src.Seek(ds, io.SeekStart); err != nil { + if _, err := src.Seek(e.off, io.SeekStart); err != nil { return fail(err) } - n, cerr := io.CopyN(cw, src, length) + n, cerr := io.CopyN(cw, src, e.length) dataBytes += n if cerr != nil { - return fail(fmt.Errorf("reading extent @%d+%d: %w", ds, length, cerr)) + return fail(fmt.Errorf("reading extent @%d+%d: %w", e.off, e.length, cerr)) } - off = de } if err := binary.Write(cw, binary.LittleEndian, sparseEndOffset); err != nil { return fail(err) diff --git a/pkg/objectstorage/upload.go b/pkg/objectstorage/upload.go index bca454f9d1..247d1e1572 100644 --- a/pkg/objectstorage/upload.go +++ b/pkg/objectstorage/upload.go @@ -97,25 +97,32 @@ type uploadPlan struct { } func (u *Uploader) plan(content io.Reader) (uploadPlan, error) { + plan, _, _, err := u.planStream(content) + return plan, err +} + +func (u *Uploader) planStream(content io.Reader) (uploadPlan, int64, io.Reader, error) { buffers := uploadBufferMemory(u.client) workers := min(int64(runtime.GOMAXPROCS(0)), (u.budget.capacity-buffers)/parallelWorkerMemory) + workerMemory := parallelWorkerMemory parts := 1 + streamMax := int64(-1) if f, ok := content.(*os.File); ok { info, err := f.Stat() if err != nil { - return uploadPlan{}, err + return uploadPlan{}, 0, nil, err } position, err := f.Seek(0, io.SeekCurrent) if err != nil { - return uploadPlan{}, err + return uploadPlan{}, 0, nil, err } exts, populated, err := sparseExtents(f, info.Size()) _, seekErr := f.Seek(position, io.SeekStart) if err != nil { - return uploadPlan{}, err + return uploadPlan{}, 0, nil, err } if seekErr != nil { - return uploadPlan{}, seekErr + return uploadPlan{}, 0, nil, seekErr } // Holes never enter the compressor or range uploads. workers = min(workers, populated/parZstdChunk+1) @@ -124,17 +131,15 @@ func (u *Uploader) plan(content io.Reader) (uploadPlan, error) { // PutSparseFile splits a file into parts whenever it has two or // more (see errSparseTooSmall), so the streaming fallback never // runs: reserve only what the parts hold. - return uploadPlan{workers: int(workers), parts: parts, memory: int64(parts) * sparseUploadMemory}, nil + return uploadPlan{workers: int(workers), parts: parts, memory: int64(parts) * sparseUploadMemory}, -1, content, nil } - buffers = streamBufferMemory(u.client, sparseEncodedBound(populated, len(exts))) - } - memory := buffers + workers*parallelWorkerMemory - if _, ok := u.client.(sparseFilePutter); ok { - // The range-parallel path and streaming fallback are alternatives, - // so reserve the larger requirement rather than adding both. - memory = max(memory, int64(parts)*sparseUploadMemory) + streamMax = sparseEncodedBound(populated, len(exts)) + buffers = streamBufferMemory(u.client, streamMax) + workerMemory = parZstdWorkerMemory(parZstdChunkSize(populated)) + content = &sparseFile{File: f, size: info.Size(), exts: exts, populated: populated} } - return uploadPlan{workers: int(workers), parts: parts, memory: memory}, nil + memory := buffers + workers*workerMemory + return uploadPlan{workers: int(workers), parts: parts, memory: memory}, streamMax, content, nil } // sparseEncodedBound bounds the size of the sparse-extent stream for a file @@ -147,11 +152,12 @@ func sparseEncodedBound(populated int64, extents int) int64 { // streamBufferMemory is the backend buffering for a streamed object of at // most size bytes. GCS sends an object smaller than uploadCompositeMin in one -// request: it holds PutObject's lookahead and the client's chunk buffer for -// that request (see putSingle), never the composite parts. +// request: PutObject allocates singleUploadChunkSize(size) for the lookahead +// and putSingle sets the same ChunkSize on the writer, never allocating the +// composite parts. func streamBufferMemory(client ObjectStorage, size int64) int64 { if _, ok := client.(*gcsClient); ok && size < uploadCompositeMin { - return uploadCompositeMin + uploadChunkSize + return 2 * singleUploadChunkSize(size) } return uploadBufferMemory(client) } @@ -179,7 +185,9 @@ func (u *Uploader) Send(ctx context.Context, url string, content io.Reader) erro // A manifest is a few KiB: size its reservation by the file rather // than by the largest object the backend could buffer. if info, err := f.Stat(); err == nil && info.Mode().IsRegular() { - memory = streamBufferMemory(u.client, info.Size()) + size := info.Size() + memory = streamBufferMemory(u.client, size) + content = boundedReader{Reader: content, max: size} } } if err := u.budget.reserved.Acquire(ctx, memory); err != nil { diff --git a/pkg/objectstorage/upload_bench_test.go b/pkg/objectstorage/upload_bench_test.go index 40beb8ae93..c1ef431041 100644 --- a/pkg/objectstorage/upload_bench_test.go +++ b/pkg/objectstorage/upload_bench_test.go @@ -15,17 +15,24 @@ package objectstorage import ( + "bytes" "context" + "encoding/json" "fmt" "io" "log/slog" + "net/http" + "net/url" "os" "path/filepath" "runtime" + "strings" "sync" "sync/atomic" "testing" "time" + + "google.golang.org/api/option" ) type delayedSnapshotStore struct{} @@ -49,9 +56,54 @@ func (delayedSnapshotStore) PutObject(ctx context.Context, _, _ string, r io.Rea return err } +type delayedGCSTransport struct{} + +func (delayedGCSTransport) RoundTrip(r *http.Request) (*http.Response, error) { + reply := func(body string) *http.Response { + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": {"application/json"}}, + Body: io.NopCloser(strings.NewReader(body)), + Request: r, + } + } + switch { + case r.URL.Query().Get("uploadType") == "resumable": + var metadata struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil { + return nil, err + } + response := reply("") + response.Header.Set("Location", "https://upload.test/session/"+url.PathEscape(metadata.Name)) + return response, nil + case strings.HasPrefix(r.URL.Path, "/session/"), r.URL.Query().Get("uploadType") == "multipart": + name := strings.TrimPrefix(r.URL.Path, "/session/") + if r.URL.Query().Get("uploadType") == "multipart" { + name = r.URL.Query().Get("name") + } + if _, err := io.Copy(io.Discard, r.Body); err != nil { + return nil, err + } + select { + case <-time.After(20 * time.Millisecond): + case <-r.Context().Done(): + return nil, r.Context().Err() + } + return reply(fmt.Sprintf(`{"name":%q,"bucket":"bucket"}`, name)), nil + case strings.HasSuffix(r.URL.Path, "/compose"): + return reply(`{"name":"composed","bucket":"bucket"}`), nil + case r.Method == http.MethodDelete: + return reply(""), nil + default: + return nil, fmt.Errorf("unexpected GCS request: %s %s", r.Method, r.URL) + } +} + // BenchmarkConcurrentSnapshotUploads compares the shared compression budget with -// admitting all 80 uploads at once. It samples peak heap, not cumulative allocation. -// SDK buffering is covered separately by the slow GCS and S3 tests. +// admitting all 80 uploads at once, and measures the full GCS single-upload path +// under the default 512 MiB budget. It samples peak heap, not cumulative allocation. func BenchmarkConcurrentSnapshotUploads(b *testing.B) { const files = 80 path := filepath.Join(b.TempDir(), "metadata") @@ -61,15 +113,32 @@ func BenchmarkConcurrentSnapshotUploads(b *testing.B) { logger := slog.Default() slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil))) defer slog.SetDefault(logger) + + gcsStore, err := NewGCSClient(b.Context(), + option.WithoutAuthentication(), + option.WithHTTPClient(&http.Client{Transport: delayedGCSTransport{}})) + if err != nil { + b.Fatal(err) + } + g := gcsStore.(*gcsClient) + defer func() { + g.client.Close() + for _, c := range g.pool { + c.Close() + } + }() + for _, tc := range []struct { name string + client ObjectStorage capacity int64 }{ - {"shared_512MiB", DefaultUploadMemory}, - {"all_80_admitted", files * parallelWorkerMemory}, + {"shared_512MiB", delayedSnapshotStore{}, DefaultUploadMemory}, + {"all_80_admitted", delayedSnapshotStore{}, files * parallelWorkerMemory}, + {"gcs_shared_512MiB", gcsStore, DefaultUploadMemory}, } { b.Run(tc.name, func(b *testing.B) { - u, err := NewUploader(delayedSnapshotStore{}, tc.capacity) + u, err := NewUploader(tc.client, tc.capacity) if err != nil { b.Fatal(err) } @@ -122,3 +191,114 @@ func BenchmarkConcurrentSnapshotUploads(b *testing.B) { }) } } + +// BenchmarkLargeSparseUpload verifies that large sparse memory images (both +// multi-chunk streaming and multi-part GCS ranged uploads) keep their full +// chunk sizes and throughput, and measures a full 7-file gVisor snapshot +// (6 small files + 1 large pages.img) under the 512 MiB default budget. +func BenchmarkLargeSparseUpload(b *testing.B) { + dir := b.TempDir() + largePath := filepath.Join(dir, "pages.img") + file, err := os.Create(largePath) + if err != nil { + b.Fatal(err) + } + block := bytes.Repeat([]byte{0x5a}, 1<<20) + const populatedMiB = 256 + for range populatedMiB { + if _, err := file.Write(block); err != nil { + b.Fatal(err) + } + } + if err := file.Truncate(2 << 30); err != nil { + b.Fatal(err) + } + if err := file.Close(); err != nil { + b.Fatal(err) + } + + var smallPaths []string + for i := range 6 { + p := filepath.Join(dir, fmt.Sprintf("small-%d.img", i)) + if err := os.WriteFile(p, bytes.Repeat([]byte{byte(i + 1)}, 64<<10), 0o600); err != nil { + b.Fatal(err) + } + smallPaths = append(smallPaths, p) + } + + logger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil))) + defer slog.SetDefault(logger) + + gcsStore, err := NewGCSClient(b.Context(), + option.WithoutAuthentication(), + option.WithHTTPClient(&http.Client{Transport: delayedGCSTransport{}})) + if err != nil { + b.Fatal(err) + } + g := gcsStore.(*gcsClient) + defer func() { + g.client.Close() + for _, c := range g.pool { + c.Close() + } + }() + + b.Run("large_256MiB_only", func(b *testing.B) { + u, err := NewUploader(gcsStore, DefaultUploadMemory) + if err != nil { + b.Fatal(err) + } + b.SetBytes(populatedMiB << 20) + b.ResetTimer() + for range b.N { + f, err := os.Open(largePath) + if err != nil { + b.Fatal(err) + } + err = u.SendFile(b.Context(), "gs://bucket/pages.img", f) + f.Close() + if err != nil { + b.Fatal(err) + } + } + }) + + b.Run("4_actors_gvisor_7files_each", func(b *testing.B) { + u, err := NewUploader(gcsStore, DefaultUploadMemory) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for range b.N { + var wg sync.WaitGroup + for actor := range 4 { + for i, p := range smallPaths { + wg.Go(func() { + f, err := os.Open(p) + if err != nil { + b.Error(err) + return + } + defer f.Close() + if err := u.SendFile(b.Context(), fmt.Sprintf("gs://bucket/a%d-small-%d.img", actor, i), f); err != nil { + b.Error(err) + } + }) + } + wg.Go(func() { + f, err := os.Open(largePath) + if err != nil { + b.Error(err) + return + } + defer f.Close() + if err := u.SendFile(b.Context(), fmt.Sprintf("gs://bucket/a%d-pages.img", actor), f); err != nil { + b.Error(err) + } + }) + } + wg.Wait() + } + }) +} diff --git a/pkg/objectstorage/upload_gcs_test.go b/pkg/objectstorage/upload_gcs_test.go index ea865f978b..830706c16d 100644 --- a/pkg/objectstorage/upload_gcs_test.go +++ b/pkg/objectstorage/upload_gcs_test.go @@ -29,6 +29,7 @@ import ( "testing" "testing/synctest" + "google.golang.org/api/googleapi" "google.golang.org/api/option" ) @@ -176,3 +177,180 @@ func TestGCSSparseUploadHonorsReservedParts(t *testing.T) { } }) } + +func TestUploaderGCSSnapshotBurstFitsDefaultBudget(t *testing.T) { + dir := t.TempDir() + var smallFiles []*os.File + for i := range 6 { + path := filepath.Join(dir, fmt.Sprintf("small-%d.img", i)) + writeSparseSource(t, path, 2<<30, []region{{0, 64 << 10, byte(i + 1)}}) + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + smallFiles = append(smallFiles, f) + } + pagesPath := filepath.Join(dir, "pages.img") + writeSparseSource(t, pagesPath, 2<<30, []region{{0, 2 * sparsePartTarget, 0x7f}}) + pages, err := os.Open(pagesPath) + if err != nil { + t.Fatal(err) + } + defer pages.Close() + + synctest.Test(t, func(t *testing.T) { + gate := make(chan struct{}) + transport := &slowGCSTransport{parts: make(chan string, 16), release: gate} + client, err := NewGCSClient(t.Context(), option.WithoutAuthentication(), option.WithHTTPClient(&http.Client{Transport: transport})) + if err != nil { + t.Fatal(err) + } + g := client.(*gcsClient) + defer func() { + g.client.Close() + for _, c := range g.pool { + c.Close() + } + }() + u, err := NewUploader(client, DefaultUploadMemory) + if err != nil { + t.Fatal(err) + } + done := make(chan error, len(smallFiles)+1) + for i, f := range smallFiles { + go func() { + done <- u.SendFile(t.Context(), fmt.Sprintf("gs://bucket/small-%d.img", i), f) + }() + } + go func() { + done <- u.SendFile(t.Context(), "gs://bucket/pages.img", pages) + }() + synctest.Wait() + // All 6 small files and both ranged parts of pages.img must fit in + // the 512 MiB default budget simultaneously without queueing. + if got, want := len(transport.parts), len(smallFiles)+2; got != want { + t.Errorf("started %d concurrent GCS uploads, want %d", got, want) + } + close(gate) + synctest.Wait() + for range len(smallFiles) + 1 { + if err := <-done; err != nil { + t.Fatal(err) + } + } + }) +} + +func TestSingleUploadChunkSize(t *testing.T) { + const unit = int64(googleapi.MinUploadChunkSize) + for _, tc := range []struct { + size int64 + want int64 + }{ + {-1, unit}, + {0, unit}, + {1, unit}, + {unit - 1, unit}, + {unit, 2 * unit}, + {unit + 1, 2 * unit}, + {uploadCompositeMin - 1, uploadCompositeMin}, + } { + got := singleUploadChunkSize(tc.size) + if got != tc.want { + t.Errorf("singleUploadChunkSize(%d) = %d, want %d", tc.size, got, tc.want) + } + if got <= max(tc.size, 0) { + t.Errorf("singleUploadChunkSize(%d) = %d, must be strictly greater than size", tc.size, got) + } + if got%unit != 0 { + t.Errorf("singleUploadChunkSize(%d) = %d, must be a multiple of %d", tc.size, got, unit) + } + } + + // An object whose size is an exact multiple of MinUploadChunkSize (256 KiB) + // must still hit EOF inside PutObject's lookahead and take the single-request + // path rather than spilling into composite parts. + closed := make(chan struct{}) + close(closed) + transport := &slowGCSTransport{parts: make(chan string, 4), release: closed} + client, err := NewGCSClient(t.Context(), option.WithoutAuthentication(), option.WithHTTPClient(&http.Client{Transport: transport})) + if err != nil { + t.Fatal(err) + } + g := client.(*gcsClient) + defer func() { + g.client.Close() + for _, c := range g.pool { + c.Close() + } + }() + payload := bytes.Repeat([]byte{0xab}, int(unit)) + body := boundedReader{Reader: bytes.NewReader(payload), max: unit} + if err := client.PutObject(t.Context(), "bucket", "exact-256k", body); err != nil { + t.Fatal(err) + } + if got := len(transport.parts); got != 1 { + t.Errorf("256 KiB object sent %d requests, want 1", got) + } +} + +type wrappedReader struct { + r io.Reader +} + +func (w wrappedReader) Read(p []byte) (int, error) { return w.r.Read(p) } +func (w wrappedReader) Unwrap() io.Reader { return w.r } + +type lookaheadCaptureStore struct { + lookaheads []int64 +} + +func (*lookaheadCaptureStore) supportsStreamingPut() {} +func (*lookaheadCaptureStore) GetObject(context.Context, string, string) (io.ReadCloser, error) { + return nil, fmt.Errorf("unused") +} +func (s *lookaheadCaptureStore) PutObject(_ context.Context, _, _ string, r io.Reader) error { + // Wrap the reader before checking singleUploadLookahead to verify the size + // bound survives decorator unwrapping through the public upload paths. + s.lookaheads = append(s.lookaheads, singleUploadLookahead(wrappedReader{r: r})) + _, err := io.Copy(io.Discard, r) + return err +} + +func TestSingleUploadLookaheadThroughPublicUploaders(t *testing.T) { + dir := t.TempDir() + sparsePath := filepath.Join(dir, "small.img") + writeSparseSource(t, sparsePath, 2<<30, []region{{0, 64 << 10, 0x33}}) + sparseFile, err := os.Open(sparsePath) + if err != nil { + t.Fatal(err) + } + defer sparseFile.Close() + + manifestPath := filepath.Join(dir, "manifest.json") + if err := os.WriteFile(manifestPath, []byte(`{"version":1}`), 0o600); err != nil { + t.Fatal(err) + } + manifestFile, err := os.Open(manifestPath) + if err != nil { + t.Fatal(err) + } + defer manifestFile.Close() + + store := &lookaheadCaptureStore{} + u, err := NewUploader(store, DefaultUploadMemory) + if err != nil { + t.Fatal(err) + } + if err := u.SendFile(t.Context(), "gs://bucket/small.img", sparseFile); err != nil { + t.Fatal(err) + } + if err := u.Send(t.Context(), "gs://bucket/manifest.json", manifestFile); err != nil { + t.Fatal(err) + } + const want = int64(googleapi.MinUploadChunkSize) + if len(store.lookaheads) != 2 || store.lookaheads[0] != want || store.lookaheads[1] != want { + t.Errorf("captured lookaheads = %v, want [%d %d]", store.lookaheads, want, want) + } +} diff --git a/pkg/objectstorage/upload_test.go b/pkg/objectstorage/upload_test.go index 199dc8b366..84d8356b03 100644 --- a/pkg/objectstorage/upload_test.go +++ b/pkg/objectstorage/upload_test.go @@ -60,7 +60,7 @@ func TestUploaderBudgetFitsBothPaths(t *testing.T) { continue } // An all-holes file streams a few bytes. - if plan.memory < streamBufferMemory(client, sparseEncodedBound(0, 0))+int64(plan.workers)*parallelWorkerMemory { + if plan.memory < streamBufferMemory(client, sparseEncodedBound(0, 0))+int64(plan.workers)*parZstdWorkerMemory(parZstdMinChunk) { t.Fatalf("%T: streaming pipeline under-reserved: %+v", client, plan) } } @@ -75,9 +75,9 @@ func TestUploaderPlansFromPopulatedBytes(t *testing.T) { populatedMiB int gcs, s3 uploadPlan }{ - {"all_holes", 0, uploadPlan{1, 1, 192 << 20}, uploadPlan{1, 1, 224 << 20}}, - {"small", 1, uploadPlan{1, 1, 192 << 20}, uploadPlan{1, 1, 224 << 20}}, - {"two_chunks", 9, uploadPlan{2, 1, 256 << 20}, uploadPlan{2, 1, 288 << 20}}, + {"all_holes", 0, uploadPlan{1, 1, 8<<20 + 512<<10}, uploadPlan{1, 1, 168 << 20}}, + {"small", 1, uploadPlan{1, 1, 8<<20 + 2560<<10}, uploadPlan{1, 1, 168 << 20}}, + {"two_chunks", 9, uploadPlan{2, 1, 146<<20 + 512<<10}, uploadPlan{2, 1, 288 << 20}}, {"multiple_ranges", 192, uploadPlan{3, 3, 144 << 20}, uploadPlan{5, 3, 480 << 20}}, } { t.Run(tc.name, func(t *testing.T) { @@ -140,13 +140,13 @@ func TestUploaderReservesOnlyTheChosenPath(t *testing.T) { if got.parts != 4 || got.memory != 4*sparseUploadMemory { t.Errorf("capacity %d: ranged plan = %+v, want 4 parts reserving %d", capacity, got, 4*sparseUploadMemory) } - // 1 MiB populated: one request, so PutObject's lookahead, the client's - // chunk buffer, and one worker. + // 1 MiB populated: one request, sized to the bounded stream and 1 MiB + // compression chunks. got, err = u.plan(small) if err != nil { t.Fatal(err) } - if want := uploadCompositeMin + uploadChunkSize + parallelWorkerMemory; got.memory != want { + if want := streamBufferMemory(&gcsClient{}, sparseEncodedBound(1<<20, 1)) + parZstdWorkerMemory(parZstdMinChunk); got.memory != want { t.Errorf("capacity %d: small plan = %+v, want %d", capacity, got, want) } } @@ -162,7 +162,7 @@ func TestUploaderSmallSparseFilesShareBudget(t *testing.T) { synctest.Test(t, func(t *testing.T) { gate := make(chan struct{}) store := &gatedUploadStore{started: make(chan string, len(files)), release: gate} - u, err := NewUploader(store, 2*parallelWorkerMemory) + u, err := NewUploader(store, parallelWorkerMemory) if err != nil { t.Fatal(err) } From 87a3d9a286a283c6dbf336bb6dd3ac40485611df Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Tue, 20 Oct 2026 16:36:30 +0000 Subject: [PATCH 02/63] atelet: Wait for a terminated actor's upload or detach to stop (#2943) Terminate now cancels the actor's upload or detach in progress and waits for it to stop before it forgets the actor, and keeps new ones from starting while it runs. Once the control plane has terminated the actor, atelet is no longer uploading its snapshots in the background, so the control plane can delete their objects. If atelet restarts while Terminate waits, it resumes the upload or detach from the state file until the control plane retries Terminate. Part of https://github.com/agent-substrate/substrate/issues/2069 and https://github.com/agent-substrate/substrate/issues/2571 AI use: AI-assisted, I reviewed and tested it - [x] Tests pass - [x] Docs updated --- cmd/atelet/internal/snapshottracker/actor.go | 26 ++-- .../internal/snapshottracker/background.go | 105 +++++-------- .../snapshottracker/background_test.go | 141 +++++++++++++----- cmd/atelet/internal/snapshottracker/detach.go | 11 +- .../internal/snapshottracker/detach_test.go | 34 +++-- .../internal/snapshottracker/lifecycle.go | 48 ++++-- .../internal/snapshottracker/metrics_test.go | 23 ++- cmd/atelet/internal/snapshottracker/report.go | 11 +- .../internal/snapshottracker/report_test.go | 16 +- .../internal/snapshottracker/tracker.go | 52 +++++-- .../internal/snapshottracker/tracker_test.go | 53 +++++-- cmd/atelet/internal/snapshottracker/upload.go | 44 +++--- .../internal/snapshottracker/upload_test.go | 101 +++++++++++-- cmd/atelet/lifecycle_test.go | 122 ++++++++++++++- cmd/atelet/main.go | 9 +- 15 files changed, 574 insertions(+), 222 deletions(-) diff --git a/cmd/atelet/internal/snapshottracker/actor.go b/cmd/atelet/internal/snapshottracker/actor.go index b72a8af303..d97a6e86d6 100644 --- a/cmd/atelet/internal/snapshottracker/actor.go +++ b/cmd/atelet/internal/snapshottracker/actor.go @@ -161,9 +161,9 @@ type actorSnapshotsRecord struct { // // It is not updated, but the actor is queued again, when a worker pulls // the actor out of the workqueue and: - // - A Run, Restore, Checkpoint, or PromoteSnapshot of the actor is in - // progress (BeginStart, BeginCheckpoint, or BeginPromote). Its end - // queues the actor again. + // - A Run, Restore, Checkpoint, PromoteSnapshot, or Terminate of the + // actor is in progress (BeginStart, BeginCheckpoint, BeginPromote, or + // BeginTerminate). Its end queues the actor again. // - Another operation on the actor, such as a Detach call, is in // progress. Its end queues the actor again. // - The detach is not due yet. The worker queues the actor again for @@ -189,8 +189,16 @@ const ( holdStart // Run or Restore holdCheckpoint // Checkpoint holdPromote // PromoteSnapshot + holdTerminate // Terminate ) +// blocksUploads reports whether no background upload of the actor starts while +// an RPC of kind k holds it: PromoteSnapshot, which makes a snapshot durable +// itself, or Terminate, which forgets the actor. +func (k holdKind) blocksUploads() bool { + return k == holdPromote || k == holdTerminate +} + // actorSnapshots is what atelet tracks of one actor's snapshots on this node. // Its fields are guarded by the actor's lock in Tracker.locks. type actorSnapshots struct { @@ -204,10 +212,10 @@ type actorSnapshots struct { // A background worker or an RPC call such as Detach runs it; an actor has // at most one. The entry stays in memory while op is set. op *actorOp - // hold records the Run, Restore, Checkpoint, or PromoteSnapshot of the - // actor in progress on this node. While set, atelet does not detach the - // actor or report its state (nor start a background upload during a - // promote), and the entry stays in memory. + // hold records the Run, Restore, Checkpoint, PromoteSnapshot, or Terminate + // of the actor in progress on this node. While set, atelet does not detach + // the actor or report its state (nor start a background upload when + // hold.blocksUploads), and the entry stays in memory. hold holdKind // reportDeferred is set when a report of the actor's state is held back // while hold is set, so holdLocked queues it once the RPC ends. @@ -487,8 +495,8 @@ func (a *actorSnapshots) markStarted() bool { // detachScheduled reports whether a detach of the actor is scheduled: atelet // holds a local snapshot of the actor, and no Run or Restore of the actor has // begun since that snapshot was committed. A scheduled detach runs once it is -// due (see detachDue) and no Run, Restore, Checkpoint, or PromoteSnapshot of -// the actor is in progress. +// due (see detachDue) and no Run, Restore, Checkpoint, PromoteSnapshot, or +// Terminate of the actor is in progress. func (a *actorSnapshots) detachScheduled() bool { return a.LatestLocal != nil && !a.DetachAt.IsZero() } diff --git a/cmd/atelet/internal/snapshottracker/background.go b/cmd/atelet/internal/snapshottracker/background.go index eef10454c6..963ca4e759 100644 --- a/cmd/atelet/internal/snapshottracker/background.go +++ b/cmd/atelet/internal/snapshottracker/background.go @@ -19,7 +19,6 @@ import ( "log/slog" "time" - "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/proto/ateletpb" ) @@ -121,7 +120,9 @@ func logOp(ctx context.Context, actorUID string, kind opKind, target SnapshotRef // // The operation holds the actor's lock only while it begins and ends, not // while it runs: -// - Terminate never waits for it. +// - Forget cancels any operation in progress and waits for it to end before +// it drops the actor's state. No operation begins during the Terminate +// that calls it. // - Checkpoint never waits for an upload. // - Run, Restore, and Checkpoint wait for a detach to end, so it never // deletes the actor's local snapshots underneath them. @@ -213,8 +214,10 @@ func (t *Tracker) syncActor(actorUID string, kind opKind) { // beginDueWork begins the actor's background operation of kind if one is due, // and returns the func that runs it. Otherwise it returns nil, and queues the // actor for when its next operations are due, unless an operation, Run, -// Restore, Checkpoint, or PromoteSnapshot in progress on the actor queues it -// again when it ends. +// Restore, Checkpoint, PromoteSnapshot, or Terminate in progress on the actor +// queues it again when it ends. An upload worker that finds no upload due +// clears a superseded Uploading snapshot instead, unless Terminate holds the +// actor. func (t *Tracker) beginDueWork(actorUID string, kind opKind) func() { if !t.locks.Lock(t.ctx, actorUID) { // atelet is shutting down. @@ -233,8 +236,13 @@ func (t *Tracker) beginDueWork(actorUID string, kind opKind) func() { // requeueLocked queues the actor again. return nil } + if a.hold == holdTerminate { + // A Terminate of the actor is in progress: Forget drops the actor, or + // the terminate's end queues it again if it fails first (see + // holdLocked). + return nil + } now := t.clock.Now() - var cleanup func() switch kind { case opDetach: if due, ok := t.detachDueLocked(a); ok && !due.After(now) { @@ -251,7 +259,7 @@ func (t *Tracker) beginDueWork(actorUID string, kind opKind) func() { } case opUpload: if due, ok := t.uploadDueLocked(a); ok && !due.After(now) { - if a.hold == holdPromote { + if a.hold.blocksUploads() { // A PromoteSnapshot of the actor is in progress, and queues // the actor again when it ends (see holdLocked). return nil @@ -264,10 +272,10 @@ func (t *Tracker) beginDueWork(actorUID string, kind opKind) func() { // starting the next upload. When no upload runs (for example after an // immediate checkpoint or promote made a newer snapshot durable, // leaving no upload due), clear a superseded Uploading snapshot here. - cleanup = t.cleanupSupersededUploadingLocked(a) + t.cleanupSupersededUploadingLocked(a) } t.requeueLocked(a) - return cleanup + return nil } // nextDueLocked returns when the actor's next background detach or upload is @@ -283,13 +291,18 @@ func (t *Tracker) nextDueLocked(a *actorSnapshots) (time.Time, bool) { // requeueLocked queues the actor on each queue for when its next background // operation of that kind is due, if it has one, then drops the actor's entry // from memory unless the entry has an operation, or a Run, Restore, -// Checkpoint, or PromoteSnapshot, in progress, a retry hold, a failed upload -// kept to retry, or a change that failed to save. An entry whose Uploading is -// superseded is queued at once on the upload queue and kept in memory until an -// upload worker cleans it up. A save that finds the actor's directory gone -// leaves nothing unsaved. requeueLocked runs after a change to the actor is -// saved, so it must not change what the snapshot state file keeps. +// Checkpoint, PromoteSnapshot, or Terminate, in progress, a retry hold, a +// failed upload kept to retry, or a change that failed to save. An entry whose +// Uploading is superseded is queued at once on the upload queue and kept in +// memory until an upload worker cleans it up. A save that finds the actor's +// directory gone leaves nothing unsaved. requeueLocked runs after a change to +// the actor is saved, so it must not change what the snapshot state file keeps. func (t *Tracker) requeueLocked(a *actorSnapshots) { + if a.hold == holdTerminate { + // Forget drops the actor, or the terminate's end queues it again if it + // fails first (see holdLocked). + return + } now := t.clock.Now() // CommitDurable and CommitPromoted leave a superseded Uploading snapshot // for an upload worker to clear from the state file and delete from disk, @@ -334,63 +347,32 @@ func (t *Tracker) beginOpLocked(ctx context.Context, a *actorSnapshots, kind opK return ctx, a.op } -// opEntryLocked returns the actor's entry while op is still its operation in -// progress, or nil once Forget has abandoned op. -func (t *Tracker) opEntryLocked(actorUID string, op *actorOp) *actorSnapshots { - a := t.findLocked(actorUID) - if a == nil || a.op != op { - return nil - } - return a -} - -// abandonedError returns what an operation on the actor fails with once Forget -// has abandoned it: opErr if the operation failed already, and otherwise -// Aborted. -func abandonedError(actorUID string, opErr error) error { - if opErr != nil { - return opErr - } - return apierror.Aborted("actor %s was forgotten during the operation", actorUID) -} - -// updateOp runs fn under the actor's lock and saves the actor's state while op -// is still the actor's operation in progress. Once Forget has abandoned op, it -// leaves the actor alone and fails with abandonedError. -func (t *Tracker) updateOp(actorUID string, op *actorOp, opErr error, fn func(a *actorSnapshots)) error { +// updateOp runs fn on the actor's entry under the actor's lock, while an +// operation of the actor runs, and saves the actor's state. The operation keeps +// the entry in memory (see beginOpLocked). +func (t *Tracker) updateOp(actorUID string, fn func(a *actorSnapshots)) { t.lock(actorUID) defer t.unlock(actorUID) - a := t.opEntryLocked(actorUID, op) - if a == nil { - return abandonedError(actorUID, opErr) - } + a := t.findLocked(actorUID) fn(a) t.saveLocked(a) - return nil } -// endOpLocked ends op and returns the actor's entry, or nil once Forget has -// abandoned op. +// endOpLocked ends op, the actor's operation in progress, and returns the +// actor's entry. func (t *Tracker) endOpLocked(actorUID string, op *actorOp) *actorSnapshots { op.cancel() - a := t.opEntryLocked(actorUID, op) - if a != nil { - a.op = nil - } + a := t.findLocked(actorUID) + a.op = nil return a } // endOp ends op, queues the actor for its next background operation and any -// report due, and returns the actor's state. Once Forget has abandoned op, it -// leaves the actor alone and fails with abandonedError; otherwise it fails -// with opErr if non-nil. +// report due, and returns the actor's state, or opErr if non-nil. func (t *Tracker) endOp(actorUID string, op *actorOp, opErr error) (*ateletpb.ActorSnapshotState, error) { t.lock(actorUID) defer t.unlock(actorUID) a := t.endOpLocked(actorUID, op) - if a == nil { - return nil, abandonedError(actorUID, opErr) - } t.requeueLocked(a) t.queueReportLocked(a) if opErr != nil { @@ -486,19 +468,6 @@ func (t *Tracker) waitOpLocked(ctx context.Context, actorUID string, op *actorOp } } -// abandonOpLocked cancels the actor's operation in progress, if any, without -// waiting for it to end. -func (t *Tracker) abandonOpLocked(actorUID string) { - // An entry with an operation in progress is always in memory. - t.mu.Lock() - a := t.actors[actorUID] - t.mu.Unlock() - if a != nil && a.op != nil { - a.op.cancel() - a.op = nil - } -} - // backOffLocked records a failure of the actor's background work, and holds // that work back until the time it returns: later with each consecutive // failure. diff --git a/cmd/atelet/internal/snapshottracker/background_test.go b/cmd/atelet/internal/snapshottracker/background_test.go index cba3521552..a9687a389c 100644 --- a/cmd/atelet/internal/snapshottracker/background_test.go +++ b/cmd/atelet/internal/snapshottracker/background_test.go @@ -71,18 +71,13 @@ func beginOp(t *testing.T, tr *Tracker, target SnapshotRef) (context.Context, *a return ctx, op } -// endOp ends op, as a worker does, and reports whether it was still the -// tracked actor's operation in progress. It then finishes like a worker -// whose operation changed nothing, calling requeueLocked. -func endOp(tr *Tracker, op *actorOp) bool { +// endOp ends op, as a worker does whose operation changed nothing: it calls +// requeueLocked and closes op.done. +func endOp(tr *Tracker, op *actorOp) { + defer close(op.done) tr.lock(trackedActorUID) defer tr.unlock(trackedActorUID) - a := tr.endOpLocked(trackedActorUID, op) - if a == nil { - return false - } - tr.requeueLocked(a) - return true + tr.requeueLocked(tr.endOpLocked(trackedActorUID, op)) } // inMemory reports whether the tracker holds the tracked actor's entry in @@ -144,9 +139,7 @@ func TestEntryStaysInMemoryDuringOp(t *testing.T) { t.Error("entry dropped from memory during an operation, want it kept") } - if !endOp(tr, op) { - t.Fatal("endOpLocked = nil, want the operation still the actor's") - } + endOp(tr, op) if inMemory(tr) { t.Error("entry in memory after the operation ended, want it dropped") } @@ -217,34 +210,114 @@ func TestStartQueuesActorsWithSnapshotState(t *testing.T) { } } -// Forget does not wait for the actor's background operation in progress. It -// cancels the operation, which finds itself abandoned when it ends, so it -// leaves alone both the forgotten actor's state and an operation that began -// for the actor after Forget. -func TestForgetAbandonsInFlightOp(t *testing.T) { +// Forget cancels the actor's background operation in progress and waits for it +// to end, releasing the actor's lock meanwhile, before it drops the actor's +// state in memory and on disk. +func TestForgetWaitsForInFlightOp(t *testing.T) { useActorDir(t) tr := newTracker(t) s1 := testRef("s1", 1, time.Now()) tr.CommitLocal(trackedActorUID, s1) opCtx, op := beginOp(t, tr, s1) - returnsPromptly(t, "Forget during a background operation", func() { - if err := tr.Forget(trackedActorUID); err != nil { + forgetDone := make(chan error, 1) + go func() { forgetDone <- tr.Forget(trackedActorUID) }() + select { + case <-opCtx.Done(): + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for Forget to cancel the operation") + } + returnsPromptly(t, "lock while Forget waits", func() { + tr.lock(trackedActorUID) + tr.unlock(trackedActorUID) + }) + if !inMemory(tr) { + t.Error("entry dropped from memory while Forget waits, want it kept") + } + if got := uuidOf(newTracker(t).State(trackedActorUID).GetLatestLocal()); got != "s1" { + t.Errorf("latest_local a restarted atelet reads while Forget waits = %q, want %q", got, "s1") + } + select { + case err := <-forgetDone: + t.Fatalf("Forget returned (%v) before the canceled operation ended", err) + case <-time.After(50 * time.Millisecond): + } + endOp(tr, op) + select { + case err := <-forgetDone: + if err != nil { t.Errorf("Forget: %v", err) } - }) - if opCtx.Err() == nil { - t.Error("the operation's context is live after Forget, want it cancelled") + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for Forget to return") } - _, next := beginOp(t, tr, s1) - if endOp(tr, op) { - t.Error("endOpLocked = true after Forget, want the operation abandoned") + if inMemory(tr) { + t.Error("entry in memory after Forget, want it dropped") } - if got, want := tr.State(trackedActorUID), (&ateletpb.ActorSnapshotState{Uploading: s1.report()}); !proto.Equal(got, want) { - t.Errorf("state after forget = %v, want %v: only the operation begun after Forget", got, want) + if st := newTracker(t).State(trackedActorUID); !proto.Equal(st, &ateletpb.ActorSnapshotState{}) { + t.Errorf("state a restarted atelet reads after Forget = %v, want empty", st) } - if !endOp(tr, next) { - t.Error("endOpLocked = false for the operation begun after Forget, want the abandoned one's end to leave it alone") +} + +// From BeginTerminate on, no background upload or detach of the actor begins, +// though both are due. If the terminate fails before Forget, its end lets them +// begin again; once Forget has dropped the actor, its end leaves the actor +// alone and queues nothing for it. +func TestTrackerTerminateHoldsOffBackgroundWork(t *testing.T) { + for _, tc := range []struct { + name string + forget bool + }{ + {name: "terminate fails before Forget"}, + {name: "Forget drops the actor", forget: true}, + } { + t.Run(tc.name, func(t *testing.T) { + tr := newIdleTestTracker(t, newFakeSnapshotStore()) + t0 := tr.clock.Now() + tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + stepTo(tr, t0.Add(testDetachDelay)) // An upload and a detach are both due. + eventually(t, "upload and detach are queued", func() bool { + return queuedNow(tr) == 2 + }) + for _, q := range tr.queues { + uid, _ := q.Get() + q.Done(uid) + } + + end := tr.BeginTerminate(trackedActorUID) + for _, kind := range []opKind{opUpload, opDetach} { + if run := tr.beginDueWork(trackedActorUID, kind); run != nil { + t.Errorf("beginDueWork(%v) began work during Terminate, want none", kind) + run() + } + } + if tc.forget { + opCtx, op := beginOp(t, tr, testRef("s1", 1, t0)) + go func() { + <-opCtx.Done() + endOp(tr, op) + }() + if err := tr.Forget(trackedActorUID); err != nil { + t.Fatalf("Forget: %v", err) + } + } + end() + + if tc.forget { + if inMemory(tr) { + t.Error("entry in memory after Forget and the terminate's end, want it dropped") + } + if n := queuedNow(tr); n != 0 { + t.Errorf("queued = %d after Forget and the terminate's end, want nothing queued for the forgotten actor", n) + } + return + } + run := tr.beginDueWork(trackedActorUID, opUpload) + if run == nil { + t.Fatal("beginDueWork(upload) began nothing once the terminate failed, want the due upload") + } + run() + }) } } @@ -274,9 +347,7 @@ func TestCommitsDoNotWaitForInFlightOp(t *testing.T) { if err := opCtx.Err(); err != nil { t.Errorf("the operation's context ended with %v, want it still running", err) } - if !endOp(tr, op) { - t.Error("endOpLocked = false, want the operation still the actor's") - } + endOp(tr, op) } // An actor has at most one background operation in progress, and another @@ -302,9 +373,7 @@ func TestBeginOpLockedAllowsOneOpPerActor(t *testing.T) { t.Error("beginOpLocked for another actor = nil, want an operation") } - if !endOp(tr, op) { - t.Fatal("endOpLocked = false, want the operation still the actor's") - } + endOp(tr, op) _, op = beginOp(t, tr, s1) s1.Snapshot.SnapshotUuid = "changed" if got := op.target.Snapshot.GetSnapshotUuid(); got != "s1" { diff --git a/cmd/atelet/internal/snapshottracker/detach.go b/cmd/atelet/internal/snapshottracker/detach.go index 85a2c8d59c..dc5c8a6f03 100644 --- a/cmd/atelet/internal/snapshottracker/detach.go +++ b/cmd/atelet/internal/snapshottracker/detach.go @@ -88,8 +88,7 @@ func (t *Tracker) Detach(ctx context.Context, actorUID string) (*ateletpb.ActorS // uploads the snapshot unless already durable, then deletes the actor's local // snapshots. background reports whether a worker runs it, rather than a Detach // call; a worker's detach makes a report of what it changed due, while a -// Detach call returns the new state and cancels any pending report. A detach -// Forget abandons leaves the actor alone, and fails. +// Detach call returns the new state and cancels any pending report. func (t *Tracker) finishDetach(ctx context.Context, actorUID string, op *actorOp, alreadyDurable, background bool) (*ateletpb.ActorSnapshotState, error) { defer close(op.done) uploaded := false @@ -102,7 +101,7 @@ func (t *Tracker) finishDetach(ctx context.Context, actorUID string, op *actorOp uploaded, _, err = t.uploadSnapshot(ctx, actorUID, target, background) } if err == nil { - err = t.updateOp(actorUID, op, nil, func(a *actorSnapshots) { + t.updateOp(actorUID, func(a *actorSnapshots) { if t.recordUploadLocked(a, target, uploaded) && background { t.markReportDueLocked(a) } @@ -126,7 +125,7 @@ func (t *Tracker) finishDetach(ctx context.Context, actorUID string, op *actorOp if err == nil { err = PruneLocalCheckpoints(ctx, actorUID) } - if abErr := t.updateOp(actorUID, op, err, func(a *actorSnapshots) { + t.updateOp(actorUID, func(a *actorSnapshots) { switch { case err == nil: t.clearLatestLocalLocked(a) @@ -140,9 +139,7 @@ func (t *Tracker) finishDetach(ctx context.Context, actorUID string, op *actorOp // than the detach was due. a.DetachAt = later(a.DetachAt, t.backOffLocked(a)) } - }); abErr != nil { - err = abErr - } + }) // Log and record before endOp, which ends ctx. elapsed := t.clock.Since(detachStart) diff --git a/cmd/atelet/internal/snapshottracker/detach_test.go b/cmd/atelet/internal/snapshottracker/detach_test.go index 5c01ad25af..e53d88eaae 100644 --- a/cmd/atelet/internal/snapshottracker/detach_test.go +++ b/cmd/atelet/internal/snapshottracker/detach_test.go @@ -710,12 +710,12 @@ func TestTrackerLiftsBackoffWithDetach(t *testing.T) { } } -// Forget does not wait for a detach in progress: it cancels the detach, which -// fails and leaves the forgotten actor's local snapshots for Terminate to -// prune. -func TestTrackerForgetAbandonsDetach(t *testing.T) { +// Forget cancels a detach in progress and waits for it to end, leaving the +// forgotten actor's local snapshots for Terminate to prune. +func TestTrackerForgetCancelsDetach(t *testing.T) { store := newFakeSnapshotStore() store.setBlocking(true) + store.setIgnoreCancel(true) tr := newTestTracker(t, store) if err := os.MkdirAll(ateletpath.LocalSnapshotDir(trackedActorUID, "s1"), 0o700); err != nil { t.Fatal(err) @@ -724,14 +724,30 @@ func TestTrackerForgetAbandonsDetach(t *testing.T) { detached := startDetach(t, tr) waitStarted(t, store) - returnsPromptly(t, "Forget during a detach", func() { - if err := tr.Forget(trackedActorUID); err != nil { - t.Errorf("Forget: %v", err) - } - }) + forgetDone := make(chan error, 1) + go func() { + forgetDone <- tr.Forget(trackedActorUID) + }() + if got := waitCanceled(t, store); got != "s1" { + t.Fatalf("canceled detach upload = %q, want s1", got) + } + select { + case err := <-forgetDone: + t.Fatalf("Forget returned (%v) before the canceled detach ended", err) + case <-time.After(20 * time.Millisecond): + } + store.release <- context.Canceled if err := <-detached; !errors.Is(err, context.Canceled) { t.Errorf("detach = %v, want %v", err, context.Canceled) } + select { + case err := <-forgetDone: + if err != nil { + t.Errorf("Forget: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for Forget to return") + } if st := tr.State(trackedActorUID); !proto.Equal(st, &ateletpb.ActorSnapshotState{}) { t.Errorf("state after forget = %v, want empty", st) } diff --git a/cmd/atelet/internal/snapshottracker/lifecycle.go b/cmd/atelet/internal/snapshottracker/lifecycle.go index 07c6c6dcd8..69bd53ac8f 100644 --- a/cmd/atelet/internal/snapshottracker/lifecycle.go +++ b/cmd/atelet/internal/snapshottracker/lifecycle.go @@ -233,12 +233,24 @@ func (t *Tracker) BeginPromote(ctx context.Context, actorUID string, snap Snapsh return alreadyDurable, t.holdLocked(actorUID, a, holdPromote), nil } -// holdLocked records a Run, Restore, Checkpoint, or PromoteSnapshot of the -// actor in progress on its entry a, which holds detaches and state reports of -// the actor off (and background uploads during a promote), and returns the -// func that ends it and queues the actor again. An actor atelet tracks nothing -// of yet (a nil a) is held too, so no detach or report can begin on a snapshot -// its first Checkpoint commits before it ends. +// BeginTerminate records that a Terminate of the actor is in progress on this +// node, and returns the func to call when the terminate ends, whether or not it +// succeeded. Until then, atelet starts no background work on the actor; Forget +// cancels an upload or detach already in progress. If the terminate fails +// before Forget, its end lets the actor's background work resume. +func (t *Tracker) BeginTerminate(actorUID string) (end func()) { + t.lock(actorUID) + defer t.unlock(actorUID) + return t.holdLocked(actorUID, t.findLocked(actorUID), holdTerminate) +} + +// holdLocked records a Run, Restore, Checkpoint, PromoteSnapshot, or Terminate +// of the actor in progress on its entry a, which holds detaches and state +// reports of the actor off (and background uploads when kind.blocksUploads), +// and returns the func that ends it and queues the actor again, unless Forget +// has dropped the actor meanwhile. An actor atelet tracks nothing of yet (a +// nil a) is held too, so no detach or report can begin on a snapshot its first +// Checkpoint commits before it ends. func (t *Tracker) holdLocked(actorUID string, a *actorSnapshots, kind holdKind) (end func()) { if a == nil { a = &actorSnapshots{actorUID: actorUID} @@ -248,6 +260,11 @@ func (t *Tracker) holdLocked(actorUID string, a *actorSnapshots, kind holdKind) return func() { t.lock(actorUID) defer t.unlock(actorUID) + if t.findLocked(actorUID) != a { + // Forget dropped the actor, as a Terminate does while it holds + // the actor. + return + } a.hold = holdNone if a.reportDeferred { t.queueReportLocked(a) @@ -256,14 +273,23 @@ func (t *Tracker) holdLocked(actorUID string, a *actorSnapshots, kind holdKind) } } -// Forget drops the actor's state, in memory and on disk. It does not wait for -// the actor's operation in progress, an upload or a detach: it cancels it, and -// the operation finds itself abandoned when it ends. +// Forget drops the actor's state, in memory and on disk. It first cancels any +// background upload or detach of the actor in progress and waits for it to +// end, releasing the actor's lock meanwhile. func (t *Tracker) Forget(actorUID string) error { t.lock(actorUID) defer t.unlock(actorUID) - t.abandonOpLocked(actorUID) - // Start the actor's backoff over; its entry, and any hold, goes below. + if a := t.findLocked(actorUID); a != nil { + t.addLocked(a) + a.hold = holdTerminate + for a.op != nil { + op := a.op + op.cancel() + t.unlock(actorUID) + <-op.done + t.lock(actorUID) + } + } t.retries.Forget(actorUID) t.reporting.retries.Forget(actorUID) t.dropLocked(actorUID) diff --git a/cmd/atelet/internal/snapshottracker/metrics_test.go b/cmd/atelet/internal/snapshottracker/metrics_test.go index 0accc34b07..3bbc97cbaa 100644 --- a/cmd/atelet/internal/snapshottracker/metrics_test.go +++ b/cmd/atelet/internal/snapshottracker/metrics_test.go @@ -421,20 +421,29 @@ func TestDetachDurationOutcomes(t *testing.T) { t.Run("canceled", func(t *testing.T) { inst, reader := newTestInstruments(t) store := newFakeSnapshotStore() + store.setBlocking(true) + store.setIgnoreCancel(true) useActorDir(t) tr := newStoreTracker(t, store, WithInstruments(inst)) - ref := testRef("s1", 1, tr.clock.Now()) - tr.CommitDurable(trackedActorUID, ref) - tr.CommitLocal(trackedActorUID, ref) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) stepTo(tr, detachDueOf(tr, trackedActorUID)) run := tr.beginDueWork(trackedActorUID, opDetach) if run == nil { t.Fatal("beginDueWork returned nil, want due detach") } - if err := tr.Forget(trackedActorUID); err != nil { - t.Fatalf("Forget: %v", err) - } - run() + go run() + waitStarted(t, store) + // Forget cancels the detach as its upload finishes, and waits for it + // to end. + forgetDone := make(chan error, 1) + go func() { forgetDone <- tr.Forget(trackedActorUID) }() + waitCanceled(t, store) + store.release <- nil + returnsPromptly(t, "Forget", func() { + if err := <-forgetDone; err != nil { + t.Errorf("Forget: %v", err) + } + }) if got, want := errorTypesOf(t, reader, detachDurationMetric), []string{codes.Canceled.String()}; !slices.Equal(got, want) { t.Errorf("error types = %v, want %v", got, want) } diff --git a/cmd/atelet/internal/snapshottracker/report.go b/cmd/atelet/internal/snapshottracker/report.go index 6778908e3f..4297b7080c 100644 --- a/cmd/atelet/internal/snapshottracker/report.go +++ b/cmd/atelet/internal/snapshottracker/report.go @@ -118,8 +118,8 @@ func (t *Tracker) cancelReportLocked(a *actorSnapshots) { } // queueReportLocked queues the actor's report, if one is due. While a Run, -// Restore, Checkpoint, or PromoteSnapshot holds the actor, it defers the -// report until the hold ends. +// Restore, Checkpoint, PromoteSnapshot, or Terminate holds the actor, it +// defers the report until the hold ends. func (t *Tracker) queueReportLocked(a *actorSnapshots) { if t.reporting.reporter == nil || !a.reportDue() { return @@ -243,7 +243,7 @@ func jitteredReportDelay(d time.Duration) time.Duration { // beginReport returns the actor, the state to report for it, and when the // report became due, or false when no report is due or a Run, Restore, -// Checkpoint, or PromoteSnapshot of the actor is in progress. +// Checkpoint, PromoteSnapshot, or Terminate of the actor is in progress. func (t *Tracker) beginReport(actorUID string) (resources.ActorRef, *ateletpb.ActorSnapshotState, time.Time, bool) { if !t.locks.Lock(t.ctx, actorUID) { // atelet is shutting down. @@ -256,8 +256,9 @@ func (t *Tracker) beginReport(actorUID string) (resources.ActorRef, *ateletpb.Ac return resources.ActorRef{}, nil, time.Time{}, false } if a.hold != holdNone { - // A Run, Restore, Checkpoint, or PromoteSnapshot of the actor is in - // progress, and queues the report when it ends (see holdLocked). + // A Run, Restore, Checkpoint, PromoteSnapshot, or Terminate of the + // actor is in progress, and queues the report when it ends (see + // holdLocked). a.reportDeferred = true return resources.ActorRef{}, nil, time.Time{}, false } diff --git a/cmd/atelet/internal/snapshottracker/report_test.go b/cmd/atelet/internal/snapshottracker/report_test.go index f82b8c3f8a..7fd6d29794 100644 --- a/cmd/atelet/internal/snapshottracker/report_test.go +++ b/cmd/atelet/internal/snapshottracker/report_test.go @@ -1112,10 +1112,11 @@ func TestTrackerReportStaysDueAtShutdown(t *testing.T) { } } -// A report waits while a Run, Restore, Checkpoint, or PromoteSnapshot holds -// the actor, since the control plane holds the actor's lease throughout that -// call and would refuse the report with Aborted. Once the hold ends without a -// commit canceling the report, the report is queued. +// A report waits while a Run, Restore, Checkpoint, PromoteSnapshot, or +// Terminate holds the actor, since the control plane holds the actor's lease +// throughout that call and would refuse the report with Aborted. Once the hold +// ends without a commit canceling the report, or a Terminate forgetting the +// actor, the report is queued. func TestTrackerDefersReportDuringHold(t *testing.T) { holds := []struct { name string @@ -1153,6 +1154,13 @@ func TestTrackerDefersReportDuringHold(t *testing.T) { return end }, }, + { + // A Terminate that fails before Forget. + name: "BeginTerminate", + begin: func(t *testing.T, tr *Tracker) func() { + return tr.BeginTerminate(trackedActorUID) + }, + }, } for _, h := range holds { diff --git a/cmd/atelet/internal/snapshottracker/tracker.go b/cmd/atelet/internal/snapshottracker/tracker.go index 3bf91e6291..88c61e0fd8 100644 --- a/cmd/atelet/internal/snapshottracker/tracker.go +++ b/cmd/atelet/internal/snapshottracker/tracker.go @@ -83,9 +83,9 @@ type Store interface { // does not detach an actor whose start has begun, even after a restart or // a failed start; only the actor's next checkpoint schedules a detach // again. A background detach that comes due while a Run, Restore, -// Checkpoint, or PromoteSnapshot is in progress (BeginStart, -// BeginCheckpoint, or BeginPromote) waits for it to end, which queues the -// actor again. +// Checkpoint, PromoteSnapshot, or Terminate is in progress (BeginStart, +// BeginCheckpoint, BeginPromote, or BeginTerminate) waits for it to end, +// which queues the actor again unless Terminate forgot it. // 7. A background upload that records a new LatestDurable, or a background // detach that records one or clears LatestLocal, makes a report of the // actor's state due (ReportDueSince), saved with the change so that it @@ -96,18 +96,42 @@ type Store interface { // startup window, send the actor's whole state through the Reporter, and // retry with per-actor and queue-wide backoff until the control plane // acknowledges or refuses it, a call cancels it, or reporting.giveUpAfter -// passes. A report waits while a Run, Restore, Checkpoint, or -// PromoteSnapshot is in progress, and a background change made while a -// report ran is reported again. +// passes. A report waits while a Run, Restore, Checkpoint, +// PromoteSnapshot, or Terminate is in progress, and a background change +// made while a report ran is reported again. // // Detach runs in the calling goroutine rather than on a worker, so an explicit -// control-plane call does not wait behind other actors' background work. It -// does wait for the actor's own background operation in progress, even an -// upload of an older snapshot; Run, Restore, and Checkpoint wait for a -// background detach in progress; and PromoteSnapshot waits for a background -// detach or a background upload of the same snapshot. A worker leaves an actor -// with an operation in progress to that operation's end, which queues the -// actor again. +// control-plane call does not wait behind other actors' background work. +// +// The first table shows how each RPC on an actor treats the actor's upload or +// detach in progress, and its report due. The RPC waits for the operation to +// end as it begins (BeginStart, BeginCheckpoint, BeginPromote, or Detach), +// cancels it and waits for it to stop, or leaves it alone (-). Terminate +// cancels in Forget, once the workload is gone. A canceled report is neither +// sent nor retried, though an attempt under way finishes. +// +// RPC upload in progress detach in progress report due +// Run, Restore - waits - +// Checkpoint - waits cancels +// PromoteSnapshot waits if the same waits cancels if a +// snapshot local snapshot +// is kept +// DetachActor waits waits cancels unless +// it fails +// Terminate cancels cancels cancels +// +// The second table shows whether an upload, detach, or report of the actor +// that comes due while an RPC or background work of the actor runs begins, or +// waits for it to end. An upload or detach is the actor's one operation in +// progress, whether a worker or DetachActor runs it. +// +// running upload due detach due report due +// Run, Restore begins waits waits +// Checkpoint begins waits waits +// PromoteSnapshot waits waits waits +// Terminate waits waits waits +// upload or detach waits waits begins +// report begins begins waits type Tracker struct { // ctx bounds background work, and shuts the queues down when it ends. ctx context.Context @@ -364,7 +388,7 @@ func (t *Tracker) heldLocked(actorUID string) bool { t.mu.Lock() defer t.mu.Unlock() a := t.actors[actorUID] - return a != nil && a.hold != holdNone + return a != nil && a.hold != holdNone && a.hold != holdTerminate } // uploadDueLocked returns when the actor's background upload is due, as diff --git a/cmd/atelet/internal/snapshottracker/tracker_test.go b/cmd/atelet/internal/snapshottracker/tracker_test.go index cb44505c7d..52ab4d4d36 100644 --- a/cmd/atelet/internal/snapshottracker/tracker_test.go +++ b/cmd/atelet/internal/snapshottracker/tracker_test.go @@ -121,22 +121,31 @@ const ( type fakeSnapshotStore struct { // started receives each call's snapshot uuid as the call starts. started chan string + // canceled receives each call's snapshot uuid once its ctx ends while + // ignoreCancel is set. + canceled chan string // release ends a blocked call with the error it receives. release chan error - mu sync.Mutex - calls []string - actors map[string]resources.ActorRef - uploaded map[string]bool - blocking bool - err error + mu sync.Mutex + calls []string + actors map[string]resources.ActorRef + uploaded map[string]bool + blocking bool + ignoreCancel bool + err error // deadline is the deadline of the latest call's context: zero if it has // none. deadline time.Time } func newFakeSnapshotStore() *fakeSnapshotStore { - return &fakeSnapshotStore{started: make(chan string, 100), release: make(chan error), actors: map[string]resources.ActorRef{}} + return &fakeSnapshotStore{ + started: make(chan string, 100), + canceled: make(chan string, 100), + release: make(chan error), + actors: map[string]resources.ActorRef{}, + } } // setBlocking makes later calls wait for release, or their context. @@ -146,6 +155,14 @@ func (f *fakeSnapshotStore) setBlocking(blocking bool) { f.blocking = blocking } +// setIgnoreCancel makes blocked calls wait for release even after their +// context ends, sending their snapshot uuid on canceled first. +func (f *fakeSnapshotStore) setIgnoreCancel(ignore bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.ignoreCancel = ignore +} + // setErr makes later calls fail with err, or succeed when it is nil. func (f *fakeSnapshotStore) setErr(err error) { f.mu.Lock() @@ -179,14 +196,18 @@ func (f *fakeSnapshotStore) UploadLocalSnapshot(ctx context.Context, _ string, r f.calls = append(f.calls, ref.uuid()) f.actors[ref.uuid()] = ref.Actor f.deadline, _ = ctx.Deadline() - blocking, err := f.blocking, f.err + blocking, ignoreCancel, err := f.blocking, f.ignoreCancel, f.err f.mu.Unlock() f.started <- ref.uuid() if blocking { select { case err = <-f.release: case <-ctx.Done(): - return false, ctx.Err() + if !ignoreCancel { + return false, ctx.Err() + } + f.canceled <- ref.uuid() + err = <-f.release } } if err != nil { @@ -325,6 +346,20 @@ func waitStarted(t *testing.T, store *fakeSnapshotStore) string { } } +// waitCanceled waits for the next blocked store call with ignoreCancel set to +// see its context canceled, and returns the uuid of the snapshot it was +// uploading. +func waitCanceled(t *testing.T, store *fakeSnapshotStore) string { + t.Helper() + select { + case uuid := <-store.canceled: + return uuid + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for an upload to be canceled") + return "" + } +} + // opDone returns a channel closed once the actor's operation in progress has // ended, deleting the local snapshots it no longer needs included. func opDone(t *testing.T, tr *Tracker, actorUID string) <-chan struct{} { diff --git a/cmd/atelet/internal/snapshottracker/upload.go b/cmd/atelet/internal/snapshottracker/upload.go index 3fad1c546b..42066c2809 100644 --- a/cmd/atelet/internal/snapshottracker/upload.go +++ b/cmd/atelet/internal/snapshottracker/upload.go @@ -25,7 +25,7 @@ import ( // under a context derived from ctx, and returns the func that runs it once the // caller releases the actor's lock. Any superseded snapshot in a.Uploading is // replaced in the same state-file save, and its local copy removed (see -// removeSupersededLocalCopy) before the upload runs. +// removeSupersededLocalCopyLocked). func (t *Tracker) beginUploadLocked(ctx context.Context, a *actorSnapshots) func() { superseded, _ := a.clearSupersededUploading() target := *a.nextUploadTarget() @@ -37,31 +37,28 @@ func (t *Tracker) beginUploadLocked(ctx context.Context, a *actorSnapshots) func if !wasUploading { t.saveLocked(a) } + if superseded != nil { + t.removeSupersededLocalCopyLocked(a.actorUID, *superseded) + } actorUID := a.actorUID return func() { - if superseded != nil { - t.removeSupersededLocalCopy(actorUID, *superseded) - } t.backgroundUpload(opCtx, actorUID, op) } } // cleanupSupersededUploadingLocked clears a.Uploading when no upload is // running and a newer commit has superseded it, so backgroundUpload will never -// run for it. It saves the change and returns a func that removes the local -// copy kept for that upload (see removeSupersededLocalCopy), or nil. -func (t *Tracker) cleanupSupersededUploadingLocked(a *actorSnapshots) func() { +// run for it. It saves the change and removes the local copy kept for that +// upload (see removeSupersededLocalCopyLocked) without releasing the actor's +// lock, so no Terminate can begin in between. +func (t *Tracker) cleanupSupersededUploadingLocked(a *actorSnapshots) { superseded, cleared := a.clearSupersededUploading() if !cleared { - return nil + return } t.saveLocked(a) - if superseded == nil { - return nil - } - actorUID := a.actorUID - return func() { - t.removeSupersededLocalCopy(actorUID, *superseded) + if superseded != nil { + t.removeSupersededLocalCopyLocked(a.actorUID, *superseded) } } @@ -118,9 +115,8 @@ func uploadAllowance(size, bytesPerSecond int64) time.Duration { // backgroundUpload runs op, a background upload of the actor's nextUploadTarget // when it began, on a worker. Once the upload succeeds or a newer snapshot // replaces target, it removes target's local copy if LatestLocal moved on (see -// removeSupersededLocalCopy); when the upload fails and LatestLocal cannot -// replace target, it keeps target in Uploading to retry. An upload Forget -// abandoned leaves the actor alone. +// removeSupersededLocalCopyLocked); when the upload fails and LatestLocal +// cannot replace target, it keeps target in Uploading to retry. func (t *Tracker) backgroundUpload(ctx context.Context, actorUID string, op *actorOp) { defer close(op.done) target := op.target @@ -129,10 +125,6 @@ func (t *Tracker) backgroundUpload(ctx context.Context, actorUID string, op *act t.lock(actorUID) a := t.endOpLocked(actorUID, op) - if a == nil { - t.unlock(actorUID) - return - } removeOld := false switch { case err == nil: @@ -166,10 +158,10 @@ func (t *Tracker) backgroundUpload(ctx context.Context, actorUID string, op *act } t.requeueLocked(a) } - t.unlock(actorUID) if removeOld { - t.removeSupersededLocalCopy(actorUID, target) + t.removeSupersededLocalCopyLocked(actorUID, target) } + t.unlock(actorUID) } // recordUploadLocked records that an upload of target succeeded, and reports @@ -189,16 +181,14 @@ func (t *Tracker) recordUploadLocked(a *actorSnapshots, target SnapshotRef, uplo return true } -// removeSupersededLocalCopy deletes the local copy of target, an upload's +// removeSupersededLocalCopyLocked deletes the local copy of target, an upload's // snapshot that a newer snapshot superseded, once no upload needs it: the copy // was kept only for the upload. It leaves the copy while a Run, Restore, // Checkpoint, or PromoteSnapshot of the actor is in progress, since a Restore // or PromoteSnapshot may be reading it, and deletes it under the actor's lock // so that none begins reading it meanwhile. Best-effort: the actor's next // checkpoint, detach, or terminate prunes a copy left behind. -func (t *Tracker) removeSupersededLocalCopy(actorUID string, target SnapshotRef) { - t.lock(actorUID) - defer t.unlock(actorUID) +func (t *Tracker) removeSupersededLocalCopyLocked(actorUID string, target SnapshotRef) { if t.heldLocked(actorUID) { return } diff --git a/cmd/atelet/internal/snapshottracker/upload_test.go b/cmd/atelet/internal/snapshottracker/upload_test.go index 698993091f..f5d8318e0d 100644 --- a/cmd/atelet/internal/snapshottracker/upload_test.go +++ b/cmd/atelet/internal/snapshottracker/upload_test.go @@ -285,14 +285,14 @@ func TestTrackerFailedUploadKeepsUndurableSince(t *testing.T) { } } -// Forget does not wait for a background upload in progress: it cancels the -// upload, which then leaves the forgotten actor alone. The upload records no -// state for the actor, and deletes none of its local snapshots, not even one -// a newer checkpoint superseded: Terminate prunes them. Its metric records -// error.type Canceled. +// Forget cancels a background upload in progress and waits for it to end +// before it drops the actor's state. The upload ends as a canceled one does, +// deleting the local copy of s1, which a newer checkpoint superseded. Its +// metric records error.type Canceled. func TestTrackerForgetCancelsUpload(t *testing.T) { store := newFakeSnapshotStore() store.setBlocking(true) + store.setIgnoreCancel(true) tr := newIdleTestTracker(t, store) inst, reader := newTestInstruments(t) tr.instruments = inst @@ -304,20 +304,38 @@ func TestTrackerForgetCancelsUpload(t *testing.T) { done := startUpload(t, tr, store) tr.CommitLocal(trackedActorUID, testRef("s2", 2, tr.clock.Now())) - returnsPromptly(t, "Forget during an upload", func() { - if err := tr.Forget(trackedActorUID); err != nil { + forgetDone := make(chan error, 1) + go func() { forgetDone <- tr.Forget(trackedActorUID) }() + if got := waitCanceled(t, store); got != "s1" { + t.Fatalf("canceled upload = %q, want s1", got) + } + select { + case err := <-forgetDone: + t.Fatalf("Forget returned (%v) before the canceled upload ended", err) + case <-time.After(50 * time.Millisecond): + } + store.release <- context.Canceled + select { + case err := <-forgetDone: + if err != nil { t.Errorf("Forget: %v", err) } - }) - waitClosed(t, done, "the cancelled upload") + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for Forget to return") + } + select { + case <-done: + default: + t.Error("Forget returned before the canceled upload ended") + } if st := tr.State(trackedActorUID); !proto.Equal(st, &ateletpb.ActorSnapshotState{}) { t.Errorf("state after forget = %v, want empty", st) } if _, err := os.Stat(ateletpath.ActorSnapshotStateFile(trackedActorUID)); !os.IsNotExist(err) { - t.Errorf("state file once the cancelled upload ended: stat = %v, want none", err) + t.Errorf("state file after Forget: stat = %v, want none", err) } - if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s1")); err != nil { - t.Errorf("s1 once the cancelled upload ended: %v, want it left for Terminate to prune", err) + if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s1")); !os.IsNotExist(err) { + t.Errorf("s1 after Forget: stat = %v, want the canceled upload to delete it", err) } if got, want := recordedErrorTypes(t, reader), []string{codes.Canceled.String()}; !slices.Equal(got, want) { t.Errorf("recorded upload error types = %v, want %v", got, want) @@ -947,6 +965,65 @@ func TestTrackerKeepsSupersededCopyWhileHeld(t *testing.T) { }) } +// While Terminate holds the actor, no upload worker clears a superseded +// Uploading snapshot or deletes its local copy: Terminate prunes the actor's +// local snapshots once Forget drops it. If the terminate fails before Forget, +// its end lets a worker clear the snapshot, which deletes the copy before +// releasing the actor's lock. +func TestTrackerTerminateHoldsOffSupersededUploadingCleanup(t *testing.T) { + for _, tc := range []struct { + name string + forget bool + }{ + {name: "terminate fails before Forget"}, + {name: "Forget drops the actor", forget: true}, + } { + t.Run(tc.name, func(t *testing.T) { + store := newFakeSnapshotStore() + store.setErr(errors.New("boom")) + tr := newIdleTestTracker(t, store) + s1Dir := ateletpath.LocalSnapshotDir(trackedActorUID, "s1") + if err := os.MkdirAll(s1Dir, 0o700); err != nil { + t.Fatal(err) + } + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now().Add(-testUploadDelay))) + processNext(t, tr, opUpload) // s1's upload fails, and s1 waits in Uploading to retry. + tr.CommitDurable(trackedActorUID, testRef("s2", 2, tr.clock.Now())) + + end := tr.BeginTerminate(trackedActorUID) + processNext(t, tr, opUpload) + if rec := readSnapshotState(trackedActorUID); rec == nil || rec.Uploading == nil || rec.Uploading.uuid() != "s1" { + t.Errorf("state file during Terminate = %+v, want s1 still Uploading", rec) + } + if _, err := os.Stat(s1Dir); err != nil { + t.Errorf("s1 during Terminate: stat = %v, want it left for Terminate to prune", err) + } + + if tc.forget { + if err := tr.Forget(trackedActorUID); err != nil { + t.Fatalf("Forget: %v", err) + } + end() + if n := queuedNow(tr); n != 0 { + t.Errorf("queued = %d after Forget and the terminate's end, want nothing queued for the forgotten actor", n) + } + return + } + end() + if run := tr.beginDueWork(trackedActorUID, opUpload); run != nil { + t.Error("beginDueWork left work to run after releasing the actor's lock, want s1 deleted under it") + run() + } + if rec := readSnapshotState(trackedActorUID); rec == nil || rec.Uploading != nil { + t.Errorf("state file once the terminate failed = %+v, want Uploading cleared", rec) + } + if _, err := os.Stat(s1Dir); !os.IsNotExist(err) { + t.Errorf("s1 once the terminate failed: stat = %v, want it deleted", err) + } + }) + } +} + // Uploading is saved to the snapshot state file, so an atelet restart mid-upload // resumes s1 unless a newer snapshot was already durable before the restart. func TestTrackerResumesUploadingAfterRestart(t *testing.T) { diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index e813929a54..de49f5692d 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -87,6 +87,9 @@ type fakeWorker struct { // onCheckpoint, if non-nil, runs inside CheckpointWorkload after snapshot // files are written. onCheckpoint func() + // onTerminate, if non-nil, runs inside TerminateWorkload, which fails with + // the error it returns. + onTerminate func() error } func (f *fakeWorker) recordFidelity(rpc string, fidelity ateworkerpb.SnapshotFidelity) { @@ -147,6 +150,11 @@ func (f *fakeWorker) RestoreWorkload(_ context.Context, req *ateworkerpb.Restore func (f *fakeWorker) TerminateWorkload(_ context.Context, req *ateworkerpb.TerminateWorkloadRequest) (*ateworkerpb.TerminateWorkloadResponse, error) { f.recordActorDirs("TerminateWorkload", req.GetActorDirs()) + if f.onTerminate != nil { + if err := f.onTerminate(); err != nil { + return nil, err + } + } return &ateworkerpb.TerminateWorkloadResponse{}, nil } @@ -2028,7 +2036,8 @@ func runtimeFor(t *testing.T, rec *sandboxAssetsRecord) *ateletpb.SandboxRuntime } type gatingTrackerStore struct { - inner snapshottracker.Store + inner snapshottracker.Store + // objects, if set, fails every upload after the first gated one. objects *recordingObjectStorage started chan struct{} release chan struct{} @@ -2042,6 +2051,9 @@ func (s *gatingTrackerStore) UploadLocalSnapshot(ctx context.Context, actorUID s return false, ctx.Err() } uploaded, err := s.inner.UploadLocalSnapshot(ctx, actorUID, ref) + if s.objects == nil { + return uploaded, err + } // Fail any second upload attempt so PromoteSnapshot proves it skips // re-uploading once the background upload made the snapshot durable. s.objects.mu.Lock() @@ -2182,6 +2194,114 @@ func TestCheckpointKeepsSnapshotBeingUploaded(t *testing.T) { } } +// resumedWithPendingUpload runs a plugin actor, pauses it into pause-snap-1 +// with a deferred upload, and resumes it from that snapshot's local copy +// before the upload is due. It returns the actor, its gated tracker store, and +// a func that steps the clock past the upload's due time and reports whether +// the upload starts within a moment. +func resumedWithPendingUpload(t *testing.T) (*pluginActor, *gatingTrackerStore, func() bool) { + t.Helper() + a := runPluginActor(t) + clk := clocktesting.NewFakeClock(time.Now()) + store := &gatingTrackerStore{ + inner: &trackerStore{s: a.s}, + started: make(chan struct{}, 1), + release: make(chan struct{}), + } + a.s.snapshots = snapshottracker.New(t.Context(), store, clk, + snapshottracker.WithUploadDelay(time.Second), + snapshottracker.WithDetachDelay(time.Hour)) + a.s.snapshots.Start() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if err := a.s.snapshots.Wait(ctx); err != nil { + t.Errorf("Wait: %v", err) + } + }) + // An upload that starts holds on until released; release it before Wait. + t.Cleanup(func() { close(store.release) }) + + cpResp, err := a.checkpoint(t, a.snapshot("pause-snap-1", 1), ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + if err != nil { + t.Fatalf("Checkpoint (pause-snap-1): %v", err) + } + due := cpResp.GetState().GetUploadDueTime().AsTime() + a.restoreLocal(t, "pause-snap-1", 1) + // Step on in one-second steps past the due time, in case the queue's timer + // was armed late (see stepToUpload). + stepPastDue := func() bool { + clk.Step(due.Sub(clk.Now())) + for range 5 { + select { + case <-store.started: + return true + case <-time.After(20 * time.Millisecond): + clk.Step(time.Second) + } + } + return false + } + return a, store, stepPastDue +} + +// TestTerminateHoldsOffBackgroundUpload terminates an actor resumed from its +// pause snapshot, whose background upload comes due while the worker tears the +// actor down: the upload does not start, since Terminate would only cancel it. +// When the worker fails to tear the actor down, Terminate fails before it +// forgets the actor, and the upload, still due, starts once Terminate returns. +func TestTerminateHoldsOffBackgroundUpload(t *testing.T) { + for _, tc := range []struct { + name string + // teardownErr is what the worker's TerminateWorkload fails with. + teardownErr error + }{ + {name: "Terminate succeeds"}, + {name: "Terminate fails", teardownErr: status.Error(codes.Internal, "teardown failed")}, + } { + t.Run(tc.name, func(t *testing.T) { + a, store, stepPastDue := resumedWithPendingUpload(t) + var started atomic.Bool + a.worker.onTerminate = func() error { + started.Store(stepPastDue()) + return tc.teardownErr + } + + _, err := a.s.Terminate(t.Context(), &ateletpb.TerminateRequest{ + Atespace: "ate-demo", + ActorName: "counter", + ActorUid: pluginActorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + WorkerPodUid: pluginWorkerPodUID, + Spec: a.spec, + }) + switch { + case tc.teardownErr == nil && err != nil: + t.Fatalf("Terminate: %v", err) + case tc.teardownErr != nil && err == nil: + t.Fatal("Terminate succeeded, want it to fail with the worker's teardown") + } + if started.Load() { + t.Error("background upload of pause-snap-1 started during Terminate, want none") + } + if tc.teardownErr == nil { + select { + case <-store.started: + t.Error("background upload of pause-snap-1 started during Terminate, want none") + default: + } + return + } + select { + case <-store.started: + case <-time.After(10 * time.Second): + t.Fatal("background upload of pause-snap-1 did not start once Terminate failed") + } + }) + } +} + // TestCheckpointRejectsStaleGeneration checks that Checkpoint fails with // Aborted when given a generation below the actor's newest snapshot // or equal to it under another snapshot_uuid, before calling worker, pruning diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 117e852e33..24209b9c3e 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -1683,6 +1683,10 @@ func (s *Atelet) Terminate(ctx context.Context, req *ateletpb.TerminateRequest) return nil, err } defer unlockActor() + // No background work on the actor starts while it is being terminated. + // Forget below cancels an upload or detach already in progress. + endTerminate := s.snapshots.BeginTerminate(actorUID) + defer endTerminate() if req.GetWorkerPodUid() != "" { var assetPaths map[string]string @@ -1732,9 +1736,8 @@ func (s *Atelet) Terminate(ctx context.Context, req *ateletpb.TerminateRequest) } // The actor is gone, so no pause snapshot of it can ever be restored again. - // Forget cancels the actor's background upload or detach in progress - // without waiting for it. The cancelled operation leaves the actor's files - // alone, unless it was already deleting them, as this does too. + // Forget cancels the actor's background upload or detach in progress and + // waits for it to end before this prunes the actor's files. // TODO(#664): this only removes local snapshots in one node. We should clean // up the copies on any other NodeVmsWithLocalSnapshots. This is fine *as of // the day this was written* because today NodeVmsWithLocalSnapshots has at From 71e12bcf2598e27067cfe8570e24cca97d06682a Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Tue, 20 Oct 2026 16:51:11 +0000 Subject: [PATCH 03/63] e2e: fix the ingressauth flake: retry a refused handshake that ends without the router's alert (#2961) `TestIngressAuthentication` fails about one e2e job in three since static-mtls became the e2e default (5 of the 15 pr-workflow runs since #2390 merged): the refused client's request ends with `http2: client conn could not be established` instead of the router's TLS alert, and the test calls the refusal unproven. The router closes the connection right after the alert, and the port-forward (`portforward.go:549 ... write: broken pipe`) or Go's HTTP/2 client reports that close first. Recent instances, all on the `https` listener: - https://github.com/agent-substrate/substrate/actions/runs/39402878520/job/118561649470 (untrusted CA) - https://github.com/agent-substrate/substrate/actions/runs/39408627974/job/118580615105 (unlisted SPIFFE ID) - https://github.com/agent-substrate/substrate/actions/runs/39410976322/job/118588372580 (no client certificate) - https://github.com/agent-substrate/substrate/actions/runs/39411585756/job/118590529170 (no client certificate) - https://github.com/agent-substrate/substrate/actions/runs/39413905393/job/118598129904 (unlisted SPIFFE ID) **Change.** A refusal that fails without a TLS alert is retried on a fresh port-forward, three times at most. An alert is still required, and a client the router lets through still fails the test. **Tests.** `go vet` and the suite compile; the retry only shows under the flake, which CI's e2e jobs exercise. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR (none needed) --- .../suites/ingressauth/ingressauth_test.go | 42 ++++++++++++------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/internal/e2e/suites/ingressauth/ingressauth_test.go b/internal/e2e/suites/ingressauth/ingressauth_test.go index 16509ccc4c..15224938de 100644 --- a/internal/e2e/suites/ingressauth/ingressauth_test.go +++ b/internal/e2e/suites/ingressauth/ingressauth_test.go @@ -185,6 +185,12 @@ func isRemoteTLSAlert(err error) bool { return errors.As(err, &opErr) && opErr.Op == "remote error" } +// refusalAttempts bounds the tries one refused client gets, each on a fresh +// port-forward. The router closes right after its alert, and the port-forward +// or Go's HTTP/2 client sometimes reports that close first ("http2: client +// conn could not be established"), so the alert never reaches the test. +const refusalAttempts = 3 + // TestIngressAuthentication checks each kind of client against both TLS // listeners. // @@ -192,7 +198,9 @@ func isRemoteTLSAlert(err error) bool { // shown to carry the allowlisted client through: a port-forward tears down its // whole tunnel once one forwarded connection fails, so a shared one would turn // every check after the first refusal into a dial error. The refusal itself -// has to be a TLS alert from the router, which a broken tunnel cannot fake. +// has to be a TLS alert from the router, which a broken tunnel cannot fake; a +// failure without one is retried on a fresh port-forward, refusalAttempts +// times at most. func TestIngressAuthentication(t *testing.T) { requireStaticMTLS(t) ctx := context.Background() @@ -212,22 +220,28 @@ func TestIngressAuthentication(t *testing.T) { {name: "allowlisted SPIFFE ID signed by an untrusted CA", cert: &foreign}, } { t.Run(tc.name, func(t *testing.T) { - address := forwardRouterPort(t, ctx, listener.servicePort) + for attempt := 1; ; attempt++ { + address := forwardRouterPort(t, ctx, listener.servicePort) - status, err := listener.request(ctx, address, routerTLSConfig(t, ctx, &allowed)) - if err != nil { - t.Fatalf("allowlisted client %s got no response: %v", e2e.RouterClientSPIFFEID(e2e.RouterClientServiceAccount), err) - } - t.Logf("allowlisted client let through (status %d)", status) + status, err := listener.request(ctx, address, routerTLSConfig(t, ctx, &allowed)) + if err != nil { + t.Fatalf("allowlisted client %s got no response: %v", e2e.RouterClientSPIFFEID(e2e.RouterClientServiceAccount), err) + } + t.Logf("allowlisted client let through (status %d)", status) - status, err = listener.request(ctx, address, routerTLSConfig(t, ctx, tc.cert)) - if err == nil { - t.Fatalf("router let the client through (status %d); want the TLS handshake refused", status) - } - if !isRemoteTLSAlert(err) { - t.Fatalf("request failed without a TLS alert from the router, so the refusal is unproven: %v", err) + status, err = listener.request(ctx, address, routerTLSConfig(t, ctx, tc.cert)) + if err == nil { + t.Fatalf("router let the client through (status %d); want the TLS handshake refused", status) + } + if isRemoteTLSAlert(err) { + t.Logf("refused: %v", err) + return + } + if attempt == refusalAttempts { + t.Fatalf("request failed without a TLS alert from the router %d times, so the refusal is unproven: %v", attempt, err) + } + t.Logf("attempt %d failed without a TLS alert from the router; retrying on a fresh port-forward: %v", attempt, err) } - t.Logf("refused: %v", err) }) } }) From 9e9216573bb71c3adb3e15efc053123dbe247212 Mon Sep 17 00:00:00 2001 From: Luomu Date: Tue, 20 Oct 2026 16:59:48 +0000 Subject: [PATCH 04/63] ateapi: record a sync suspend's local snapshot copy and resume from it safely (#2883) Fixes #2938. Synchronous suspend could retain an uploaded checkpoint while controlapi dropped its local-copy record. Keep a matching copy and its node binding while releasing the worker. - Detach pinned actors before runtime fallback so compatible workers elsewhere can preserve memory. - Allow Resume and Revert to forget an unreachable node after the latest local snapshot is durable. PENDING snapshots stay pinned, version conflicts return `Aborted`, and assigned or crashed executions still require termination. - Preserve snapshot identity through template changes, detach, and restore retries. - Record cross-node restores as `snapshot.kind=latest` and consolidate attachment rules in the API field docs. ## Validation Passed on Linux/amd64 with Go 1.27.2 as an unprivileged user: - Focused controlapi and functional regressions with `-race -count=3 -shuffle=on`: 603 passing test entries, no skips. - `go test -race -count=1 ./...`: 11,078 passing test entries across 181 packages. - `go build ./...` - `bash hack/verify-all.sh` The full race suite skipped 44 entries: 38 root-gated, three requiring IPv6 loopback, two requiring a GCS emulator, and one requiring an S3 integration environment. Functional tests use PostgreSQL, envtest, and a fake atelet. New regressions fail without the fixes. Real gVisor/micro-VM E2E and privileged root-gated tests were not run. AI assistance: Codex helped implement the review fixes, review the diff, and run validation. --- .../controlapi/functionaltest/actor_test.go | 683 ++++++++++++++++++ .../controlapi/functionaltest/main_test.go | 16 +- .../internal/controlapi/snapshot_helpers.go | 11 +- .../internal/controlapi/workflow_resume.go | 41 +- .../controlapi/workflow_resume_detach.go | 45 +- .../controlapi/workflow_resume_detach_test.go | 64 +- .../controlapi/workflow_resume_test.go | 232 +++++- .../internal/controlapi/workflow_revert.go | 22 +- .../controlapi/workflow_revert_test.go | 115 +++ .../internal/controlapi/workflow_suspend.go | 21 +- .../controlapi/workflow_suspend_test.go | 91 +++ docs/api-guide.md | 33 +- pkg/proto/ateapipb/ateapi.pb.go | 12 +- pkg/proto/ateapipb/ateapi.proto | 12 +- 14 files changed, 1268 insertions(+), 130 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index da875809a9..597f21c8d3 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -48,7 +48,9 @@ import ( "google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/testing/protocmp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/ptr" ) // TestCreateActor_Success tests the happy path for creating an actor. @@ -4027,6 +4029,213 @@ func TestSuspendActor(t *testing.T) { assertSnapshotPresent(t, tc, cloneSnapshotURI) } +func TestResumeActor_SyncSuspendKeepsLocalCopy(t *testing.T) { + for _, tt := range []struct { + name string + repoint bool + relocate bool + wantNode string + wantTemplate string + wantFidelity ateletpb.SnapshotFidelity + }{ + { + name: "same template on the original node", + wantNode: "node1", + wantTemplate: "tmpl1", + wantFidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + }, + { + name: "new template on the original node", + repoint: true, + wantNode: "node1", + wantTemplate: "tmpl2", + wantFidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + }, + { + name: "new template on another node", + repoint: true, + relocate: true, + wantNode: "node2", + wantTemplate: "tmpl2", + wantFidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + }, + } { + t.Run(tt.name, func(t *testing.T) { + ns := namespaceForTest("ns-sync-local-resume") + tc := setupTest(t, ns) + defer tc.cleanup() + ctx := context.Background() + resumeKinds := func() map[string]uint64 { + var rm metricdata.ResourceMetrics + if err := tc.metricReader.Collect(ctx, &rm); err != nil { + t.Fatal(err) + } + counts := make(map[string]uint64) + for _, scope := range rm.ScopeMetrics { + for _, metric := range scope.Metrics { + if metric.Name != "ate.actor.lifecycle.operation.duration" { + continue + } + hist, ok := metric.Data.(metricdata.Histogram[float64]) + if !ok { + t.Fatalf("%s data = %T, want float64 histogram", metric.Name, metric.Data) + } + for _, point := range hist.DataPoints { + op, _ := point.Attributes.Value(ateattr.ActorOperationNameKey) + kind, present := point.Attributes.Value(ateattr.SnapshotKindKey) + if op.AsString() == ateattr.OperationResume && present { + counts[kind.AsString()] += point.Count + } + } + } + } + return counts + } + tmpl := createTemplate(t, tc, ns) + if tt.repoint { + replacement := proto.CloneOf(tmpl) + replacement.Metadata = &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "tmpl2"} + replacement.Status = nil + if _, err := tc.client.CreateActorTemplate(ctx, &ateapipb.CreateActorTemplateRequest{ActorTemplate: replacement}); err != nil { + t.Fatalf("CreateActorTemplate(tmpl2): %v", err) + } + } + worker := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.KeepCheckpointLocal = true + tc.fakeAtelet.Lock.Unlock() + + actorRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-1"} + if _, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: actorRef.GetName()}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor: %v", err) + } + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { + t.Fatalf("initial ResumeActor: %v", err) + } + if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: actorRef, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, + }); err != nil { + t.Fatalf("sync SuspendActor: %v", err) + } + suspended, err := tc.client.GetActor(ctx, &ateapipb.GetActorRequest{Actor: actorRef}) + if err != nil { + t.Fatalf("GetActor after suspend: %v", err) + } + if got := suspended.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + t.Fatalf("state after suspend = %v, want SUSPENDED", got) + } + if got := suspended.GetStatus().GetAssignedNode(); got != "node1" { + t.Fatalf("AssignedNode after sync suspend = %q, want node1", got) + } + if got := suspended.GetStatus().GetWorkerAssignment(); got != nil { + t.Fatalf("WorkerAssignment after sync suspend = %v, want none", got) + } + snap := localSnapshot(suspended.GetStatus()) + if snap == nil || snap.GetLocality() != "node1" { + t.Fatalf("sync snapshot = %v, want its local copy on node1 recorded", snap) + } + if got := snap.GetDurableSnapshot().GetStatus(); got != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + t.Fatalf("sync snapshot durable status = %v, want COMPLETED", got) + } + if got := snap.GetActorTemplateUid(); got != tmpl.GetMetadata().GetUid() { + t.Fatalf("snapshot template UID = %q, want original template %q", got, tmpl.GetMetadata().GetUid()) + } + if snap.GetUuid() == "" || snap.GetGeneration() == 0 || snap.GetDurableSnapshot().GetObject().GetSnapshotUri() == "" { + t.Fatalf("snapshot has incomplete identity: %v", snap) + } + assertSnapshotPresent(t, tc, snap.GetDurableSnapshot().GetObject().GetSnapshotUri()) + waitForWorkerAvailable(t, tc, worker) + + if tt.repoint { + toUpdate := proto.CloneOf(suspended) + toUpdate.ActorTemplate = &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl2"} + updated, err := tc.client.UpdateActor(ctx, &ateapipb.UpdateActorRequest{Actor: toUpdate}) + if err != nil { + t.Fatalf("UpdateActor with durable local snapshot: %v", err) + } + if got := localSnapshot(updated.GetStatus()); !proto.Equal(got, snap) { + t.Fatalf("snapshot after template update = %v, want unchanged %v", got, snap) + } + } + if tt.relocate { + // Fill node1 so resume must detach before moving to node2. + squatterRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "squatter"} + if _, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: squatterRef.GetName()}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor(squatter): %v", err) + } + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: squatterRef}); err != nil { + t.Fatalf("ResumeActor(squatter): %v", err) + } + createWorkerPod(t, tc, ns, "worker-2", "node2", "pool1") + setupAteletOnNode(t, tc, "atelet-node2", "node2") + } + + before := resumeKinds() + resumed, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}) + if err != nil { + t.Fatalf("ResumeActor with durable local snapshot: %v", err) + } + if got := resumed.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RUNNING { + t.Errorf("state after resume = %v, want RUNNING", got) + } + if got := resumed.GetActor().GetStatus().GetWorkerAssignment().GetNodeName(); got != tt.wantNode { + t.Errorf("resumed on node %q, want %q", got, tt.wantNode) + } + restore := tc.fakeAtelet.lastRestoreRequest() + if restore == nil || restore.GetActorName() != actorRef.GetName() { + t.Fatalf("Restore request = %v, want actor-1", restore) + } + if got := restore.GetWorkerPodUid(); got != resumed.GetActor().GetStatus().GetWorkerAssignment().GetWorkerPodUid() { + t.Errorf("Restore worker UID = %q, want the assigned worker", got) + } + if got := restore.GetActorTemplateName(); got != tt.wantTemplate { + t.Errorf("Restore template = %q, want %q", got, tt.wantTemplate) + } + if got := restore.GetSnapshot().GetFidelity(); got != tt.wantFidelity { + t.Errorf("Restore fidelity = %v, want %v", got, tt.wantFidelity) + } + if got := restore.GetSnapshot(); got.GetSnapshotUuid() != snap.GetUuid() || + got.GetGeneration() != snap.GetGeneration() || + got.GetObject().GetSnapshotUri() != snap.GetDurableSnapshot().GetObject().GetSnapshotUri() { + t.Errorf("Restore snapshot = %v, want original checkpoint identity %v", got, snap) + } + wantLocality := "node1" + wantKind := ateattr.SnapshotKindLocal + if tt.relocate { + wantLocality = "" + wantKind = ateattr.SnapshotKindLatest + } + if got := localSnapshot(resumed.GetActor().GetStatus()).GetLocality(); got != wantLocality { + t.Errorf("locality after resume = %q, want %q", got, wantLocality) + } + tc.fakeAtelet.Lock.Lock() + detached := tc.fakeAtelet.DetachCalled + tc.fakeAtelet.Lock.Unlock() + if detached != tt.relocate { + t.Errorf("detach called = %v, want %v", detached, tt.relocate) + } + + counts := resumeKinds() + for kind, count := range before { + counts[kind] -= count + if counts[kind] == 0 { + delete(counts, kind) + } + } + if diff := cmp.Diff(map[string]uint64{wantKind: 1}, counts); diff != "" { + t.Errorf("restore snapshot kinds (-want +got):\n%s", diff) + } + }) + } +} + // TestResumeActor_RepointTemplateBeforeResume checks what an Actor cloned from // a tag asks atelet to restore on its first resume. // @@ -4225,6 +4434,374 @@ func TestResumeActor_PausedAfterRepointUsesLocalProvenance(t *testing.T) { } } +func TestResumeActor_PausedLocalSnapshotRuntimeFallback(t *testing.T) { + rtA := &ateapipb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "sandbox_bundle_hash", Value: "hash-a"}, + }, + }, + } + rtB := proto.CloneOf(rtA) + rtB.Version.Attributes[1].Value = "hash-b" + + for _, tt := range []struct { + name string + compatibleNode bool + failDetach bool + wantWorker string + wantNode string + wantRuntime *ateapipb.SandboxRuntime + wantFidelity ateletpb.SnapshotFidelity + }{ + { + name: "compatible worker on another node preserves memory", + compatibleNode: true, + wantWorker: "worker-2", + wantNode: "node2", + wantRuntime: rtA, + wantFidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + }, + { + name: "no compatible worker falls back after detach", + wantWorker: "worker-1", + wantNode: "node1", + wantRuntime: rtB, + wantFidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, + }, + { + name: "unreachable node uses the uploaded checkpoint", + compatibleNode: true, + failDetach: true, + wantWorker: "worker-2", + wantNode: "node2", + wantRuntime: rtA, + wantFidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + }, + } { + t.Run(tt.name, func(t *testing.T) { + ns := namespaceForTest("ns-paused-runtime-fallback") + tc := setupTest(t, ns) + t.Cleanup(tc.cleanup) + ctx := context.Background() + createTemplate(t, tc, ns) + + workerPod := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + reportWorkerRuntime(t, tc, workerPod, rtA) + + actorRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-1"} + if _, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: actorRef.GetName()}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor: %v", err) + } + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { + t.Fatalf("initial ResumeActor: %v", err) + } + pauseResp, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: actorRef, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, + }) + if err != nil { + t.Fatalf("SuspendActor: %v", err) + } + paused := localSnapshot(pauseResp.GetActor().GetStatus()) + if paused == nil || !proto.Equal(paused.GetSandboxRuntime(), rtA) { + t.Fatalf("pause snapshot = %v, want runtime A", paused) + } + waitForWorkerAvailable(t, tc, workerPod) + + upload := &ateapipb.SnapshotReport{ + Uuid: paused.GetUuid(), + Generation: paused.GetGeneration(), + Fidelity: paused.GetFidelity(), + Object: paused.GetDurableSnapshot().GetObject(), + } + if err := tc.service.ApplyActorSnapshotState(ctx, resources.ActorRefFromActor(pauseResp.GetActor()), pauseResp.GetActor().GetMetadata().GetUid(), "node1", &ateapipb.ActorSnapshotState{ + LatestLocal: upload, + LatestDurable: upload, + }); err != nil { + t.Fatalf("ApplyActorSnapshotState(upload): %v", err) + } + reportWorkerRuntime(t, tc, workerPod, rtB) + if tt.compatibleNode { + other := createWorkerPod(t, tc, ns, "worker-2", "node2", "pool1") + reportWorkerRuntime(t, tc, other, rtA) + setupAteletOnNode(t, tc, "atelet-node2", "node2") + } + if tt.failDetach { + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.FailDetach = status.Error(codes.Unavailable, "node unavailable") + tc.fakeAtelet.Lock.Unlock() + } + + resumed, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}) + if err != nil { + t.Fatalf("ResumeActor after runtime change: %v", err) + } + assignment := resumed.GetActor().GetStatus().GetWorkerAssignment() + if assignment.GetWorkerPod() != tt.wantWorker || assignment.GetNodeName() != tt.wantNode { + t.Errorf("assignment = %v, want %s on %s", assignment, tt.wantWorker, tt.wantNode) + } + if !proto.Equal(assignment.GetSandboxRuntime(), tt.wantRuntime) { + t.Errorf("assigned runtime = %v, want %v", assignment.GetSandboxRuntime(), tt.wantRuntime) + } + tc.fakeAtelet.Lock.Lock() + detached := tc.fakeAtelet.DetachCalled + tc.fakeAtelet.Lock.Unlock() + if !detached { + t.Error("resume did not detach the incompatible local checkpoint") + } + if snap := localSnapshot(resumed.GetActor().GetStatus()); snap != nil { + t.Errorf("local snapshot after detach = %v, want none", snap) + } + + restore := tc.fakeAtelet.lastRestoreRequest().GetSnapshot() + if restore.GetSnapshotUuid() != paused.GetUuid() || restore.GetGeneration() != paused.GetGeneration() || + restore.GetObject().GetSnapshotUri() != paused.GetDurableSnapshot().GetObject().GetSnapshotUri() { + t.Errorf("restore snapshot = %v, want the paused checkpoint's identity", restore) + } + if restore.GetFidelity() != tt.wantFidelity { + t.Errorf("restore fidelity = %v, want %v", restore.GetFidelity(), tt.wantFidelity) + } + }) + } +} + +// An older upload cannot authorize fallback for a newer checkpoint. +func TestResumeActor_PausedPendingSnapshotRefusesRuntimeFallback(t *testing.T) { + ns := namespaceForTest("ns-paused-pending-refusal") + tc := setupTest(t, ns) + defer tc.cleanup() + + ctx := context.Background() + createTemplate(t, tc, ns) + + rtA := &ateapipb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "sandbox_bundle_hash", Value: "hash-a"}, + }, + }, + } + rtB := &ateapipb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "sandbox_bundle_hash", Value: "hash-b"}, + }, + }, + } + + workerPod := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + reportWorkerRuntime(t, tc, workerPod, rtA) + + const name = "actor-1" + actorRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: name} + if _, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor failed: %v", err) + } + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { + t.Fatalf("ResumeActor failed: %v", err) + } + // Keep an older durable generation beside the pending checkpoint. + if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: actorRef, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, + }); err != nil { + t.Fatalf("sync SuspendActor failed: %v", err) + } + waitForWorkerAvailable(t, tc, workerPod) + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { + t.Fatalf("ResumeActor from the durable snapshot failed: %v", err) + } + pauseResp, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: actorRef, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, + }) + if err != nil { + t.Fatalf("pause SuspendActor failed: %v", err) + } + paused := localSnapshot(pauseResp.GetActor().GetStatus()) + if paused == nil { + t.Fatalf("pause left no local snapshot: %v", pauseResp.GetActor().GetStatus()) + } + if got := paused.GetDurableSnapshot().GetStatus(); got != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING { + t.Fatalf("pause checkpoint durable status = %v, want PENDING (not yet uploaded)", got) + } + durable := durableSnapshot(pauseResp.GetActor().GetStatus()) + if durable == nil || durable.GetGeneration() >= paused.GetGeneration() { + t.Fatalf("want an older COMPLETED durable snapshot beside the pause checkpoint, got %v", pauseResp.GetActor().GetStatus().GetSnapshots()) + } + waitForWorkerAvailable(t, tc, workerPod) + + // The pending checkpoint still needs upload before runtime fallback. + reportWorkerRuntime(t, tc, workerPod, rtB) + tc.fakeAtelet.Reset() + tc.fakeAtelet.FailDetach = status.Error(codes.Unavailable, "checkpoint upload unavailable") + + _, err = tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}) + assertGrpcError(t, err, codes.Unavailable, "while calling atelet DetachActor: rpc error: code = Unavailable desc = checkpoint upload unavailable") + tc.fakeAtelet.Lock.Lock() + detachCalled := tc.fakeAtelet.DetachCalled + tc.fakeAtelet.Lock.Unlock() + if !detachCalled { + t.Error("ResumeActor did not attempt to detach the pending checkpoint") + } + + if restore := tc.fakeAtelet.lastRestoreRequest(); restore != nil { + t.Errorf("ResumeActor sent a Restore request: %v, want none (neither the checkpoint nor the older generation)", restore) + } + after, err := tc.persistence.GetActor(ctx, resources.ActorRefFromActor(pauseResp.GetActor())) + if err != nil { + t.Fatalf("GetActor: %v", err) + } + if got := after.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + t.Errorf("state after the refused resume = %v, want SUSPENDED", got) + } + if got := after.GetStatus().GetWorkerAssignment(); got != nil { + t.Errorf("WorkerAssignment after the refused resume = %v, want none", got) + } + stillLocal := localSnapshot(after.GetStatus()) + if stillLocal == nil || stillLocal.GetUuid() != paused.GetUuid() || + stillLocal.GetDurableSnapshot().GetStatus() != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING { + t.Errorf("pause checkpoint after the refused resume = %v, want %v still PENDING", stillLocal, paused) + } + if got := durableSnapshotURI(after.GetStatus()); got != durable.GetDurableSnapshot().GetObject().GetSnapshotUri() { + t.Errorf("older durable snapshot = %q, want it untouched at %q", got, durable.GetDurableSnapshot().GetObject().GetSnapshotUri()) + } +} + +func TestResumeActor_RetryRelaxedPauseAssignmentRestoresSameSnapshot(t *testing.T) { + ns := namespaceForTest("ns-paused-relaxed-retry") + tc := setupTest(t, ns) + defer tc.cleanup() + + ctx := context.Background() + createTemplate(t, tc, ns) + + rtA := &ateapipb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "sandbox_bundle_hash", Value: "hash-a"}, + }, + }, + } + rtB := &ateapipb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "sandbox_bundle_hash", Value: "hash-b"}, + }, + }, + } + + workerPod := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + reportWorkerRuntime(t, tc, workerPod, rtA) + + const name = "actor-1" + actorRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: name} + if _, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor failed: %v", err) + } + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { + t.Fatalf("ResumeActor failed: %v", err) + } + pauseResp, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: actorRef, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, + }) + if err != nil { + t.Fatalf("SuspendActor failed: %v", err) + } + paused := localSnapshot(pauseResp.GetActor().GetStatus()) + if paused == nil { + t.Fatalf("pause left no local snapshot: %v", pauseResp.GetActor().GetStatus()) + } + waitForWorkerAvailable(t, tc, workerPod) + + // Finish the upload before changing the worker's runtime. + upload := &ateapipb.SnapshotReport{ + Uuid: paused.GetUuid(), + Generation: paused.GetGeneration(), + Fidelity: paused.GetFidelity(), + Object: paused.GetDurableSnapshot().GetObject(), + } + if err := tc.service.ApplyActorSnapshotState(ctx, resources.ActorRefFromActor(pauseResp.GetActor()), pauseResp.GetActor().GetMetadata().GetUid(), "node1", &ateapipb.ActorSnapshotState{ + LatestLocal: upload, + LatestDurable: upload, + }); err != nil { + t.Fatalf("ApplyActorSnapshotState(upload) failed: %v", err) + } + reportWorkerRuntime(t, tc, workerPod, rtB) + + // Interrupt restore after detach and worker assignment. + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.FailRestore = status.Error(codes.Unavailable, "restore reply lost") + tc.fakeAtelet.Lock.Unlock() + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); status.Code(err) != codes.Unavailable { + t.Fatalf("first ResumeActor status code = %v, want Unavailable (err: %v)", status.Code(err), err) + } + interrupted, err := tc.persistence.GetActor(ctx, resources.ActorRefFromActor(pauseResp.GetActor())) + if err != nil { + t.Fatalf("GetActor after the interrupted resume: %v", err) + } + if got := interrupted.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RESUMING { + t.Fatalf("state after the interrupted resume = %v, want RESUMING", got) + } + if got := interrupted.GetStatus().GetWorkerAssignment().GetWorkerPod(); got != "worker-1" { + t.Fatalf("assigned worker after the interrupted resume = %q, want worker-1", got) + } + if crash := interrupted.GetStatus().GetCrash(); crash != nil { + t.Fatalf("crash recorded after the interrupted resume: %v", crash) + } + + // Re-entry keeps the assignment and restores the same snapshot. + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.FailRestore = nil + tc.fakeAtelet.Lock.Unlock() + resumed, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}) + if err != nil { + t.Fatalf("retried ResumeActor failed: %v", err) + } + if got := resumed.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RUNNING { + t.Errorf("state after the retry = %v, want RUNNING", got) + } + if got := resumed.GetActor().GetStatus().GetWorkerAssignment().GetWorkerPod(); got != "worker-1" { + t.Errorf("retried resume moved to worker %q, want the kept assignment on worker-1", got) + } + restoreReq := tc.fakeAtelet.lastRestoreRequest() + if got := restoreReq.GetSnapshot().GetSnapshotUuid(); got != paused.GetUuid() { + t.Errorf("RestoreRequest.SnapshotUuid = %q, want the pause checkpoint's %q", got, paused.GetUuid()) + } + if got := restoreReq.GetSnapshot().GetObject().GetSnapshotUri(); got != paused.GetDurableSnapshot().GetObject().GetSnapshotUri() { + t.Errorf("RestoreRequest.SnapshotUri = %q, want the pause checkpoint's upload %q", got, paused.GetDurableSnapshot().GetObject().GetSnapshotUri()) + } + if got := restoreReq.GetSnapshot().GetFidelity(); got != ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS { + t.Errorf("RestoreRequest.Fidelity = %v, want ROOTFS (assigned worker cannot run the checkpoint's runtime)", got) + } +} + // TestSuspendActorAsynchronous tests the full workflow of suspending a running // actor asynchronously. // Workflow: @@ -5794,6 +6371,112 @@ func TestRevertActor(t *testing.T) { } } +func TestRevertActor_UnreachableNode(t *testing.T) { + const sourceNode = "node-revert-source" + const sourceAtelet = "atelet-revert-source" + for _, tt := range []struct { + name string + missing bool + pending bool + }{ + {name: "missing atelet", missing: true}, + {name: "unavailable atelet"}, + {name: "pending checkpoint remains pinned", pending: true}, + } { + t.Run(tt.name, func(t *testing.T) { + ns := namespaceForTest("ns-revert-unreachable") + tc := setupTest(t, ns) + t.Cleanup(tc.cleanup) + ctx := context.Background() + createTemplate(t, tc, ns) + worker := createWorkerPod(t, tc, ns, "worker-1", sourceNode, "pool1") + setupAteletOnNode(t, tc, sourceAtelet, sourceNode) + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.KeepCheckpointLocal = true + tc.fakeAtelet.Lock.Unlock() + ref := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-1"} + if _, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: ref.GetName()}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatal(err) + } + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: ref}); err != nil { + t.Fatal(err) + } + mode := ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS + if tt.pending { + mode = ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS + } + suspended, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: ref, Options: &ateapipb.SuspendOptions{Mode: mode}, + }) + if err != nil { + t.Fatal(err) + } + snap := proto.CloneOf(localSnapshot(suspended.GetActor().GetStatus())) + if snap == nil || suspended.GetActor().GetStatus().GetAssignedNode() != sourceNode { + t.Fatalf("suspended actor has no retained checkpoint: %v", suspended.GetActor().GetStatus()) + } + waitForWorkerAvailable(t, tc, worker) + deleteWorkerPod(t, tc, ns, "worker-1") + if tt.missing { + if err := tc.k8sClient.CoreV1().Pods(ateletNamespace).Delete(ctx, sourceAtelet, metav1.DeleteOptions{GracePeriodSeconds: ptr.To[int64](0)}); err != nil { + t.Fatal(err) + } + if err := wait.PollUntilContextTimeout(ctx, 10*time.Millisecond, 5*time.Second, true, func(context.Context) (bool, error) { + pods, err := tc.ateletIndexer.ByIndex(byNode, sourceNode) + return len(pods) == 0, err + }); err != nil { + t.Fatal(err) + } + } else { + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.FailTerminate = status.Error(codes.Unavailable, "node unavailable") + tc.fakeAtelet.Lock.Unlock() + } + reverted, err := tc.client.RevertActor(ctx, &ateapipb.RevertActorRequest{Actor: ref}) + if tt.pending { + if status.Code(err) != codes.Unavailable { + t.Fatalf("RevertActor = %v, want Unavailable", err) + } + stored, err := tc.client.GetActor(ctx, &ateapipb.GetActorRequest{Actor: ref}) + if err != nil { + t.Fatal(err) + } + if stored.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_REVERTING || + stored.GetStatus().GetAssignedNode() != sourceNode || !proto.Equal(localSnapshot(stored.GetStatus()), snap) { + t.Fatalf("pending checkpoint changed: %v", stored.GetStatus()) + } + return + } + if err != nil { + t.Fatalf("RevertActor on unreachable node: %v", err) + } + got := reverted.GetActor().GetStatus() + if got.GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED || got.GetAssignedNode() != "" || + got.GetWorkerAssignment() != nil || localSnapshot(got) != nil { + t.Fatalf("reverted status = %v, want unbound SUSPENDED", got) + } + createWorkerPod(t, tc, ns, "worker-2", "node2", "pool1") + setupAteletOnNode(t, tc, "atelet-node2", "node2") + resumed, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: ref}) + if err != nil { + t.Fatalf("ResumeActor after revert: %v", err) + } + if resumed.GetActor().GetStatus().GetAssignedNode() != "node2" { + t.Fatalf("resumed on wrong node: %v", resumed.GetActor().GetStatus()) + } + restore := tc.fakeAtelet.lastRestoreRequest().GetSnapshot() + if restore.GetSnapshotUuid() != snap.GetUuid() || restore.GetGeneration() != snap.GetGeneration() || + restore.GetObject().GetSnapshotUri() != snap.GetDurableSnapshot().GetObject().GetSnapshotUri() { + t.Fatalf("restored snapshot = %v, want retained checkpoint %v", restore, snap) + } + assertSnapshotPresent(t, tc, snap.GetDurableSnapshot().GetObject().GetSnapshotUri()) + }) + } +} + // TestRevertActor_FromPaused reverts a paused actor back to its external // snapshot, discarding the local pause snapshot info on the actor record. func TestRevertActor_FromPaused(t *testing.T) { diff --git a/cmd/ateapi/internal/controlapi/functionaltest/main_test.go b/cmd/ateapi/internal/controlapi/functionaltest/main_test.go index 939b736507..a40b5674d0 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/main_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/main_test.go @@ -122,9 +122,10 @@ type FakeAteletServer struct { RunRequest *ateletpb.RunRequest FailRun error - CheckpointCalled bool - CheckpointRequest *ateletpb.CheckpointRequest - FailCheckpoint error + CheckpointCalled bool + CheckpointRequest *ateletpb.CheckpointRequest + FailCheckpoint error + KeepCheckpointLocal bool // checkpointed is each actor's last checkpointed snapshot, by UID. checkpointed map[string]*ateletpb.Snapshot @@ -184,6 +185,7 @@ func (f *FakeAteletServer) Reset() { f.CheckpointRequest = nil f.FailCheckpoint = nil f.checkpointed = nil + f.KeepCheckpointLocal = false f.RestoreCalled = false f.RestoreRequest = nil @@ -282,6 +284,14 @@ func (f *FakeAteletServer) Checkpoint(ctx context.Context, req *ateletpb.Checkpo return nil, err } } + if f.KeepCheckpointLocal { + report := &ateletpb.SnapshotReport{Snapshot: proto.CloneOf(req.GetSnapshot())} + state := &ateletpb.ActorSnapshotState{LatestLocal: report} + if req.GetUploadConfig().GetMode() == ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE { + state.LatestDurable = proto.CloneOf(report) + } + return &ateletpb.CheckpointResponse{State: state}, nil + } return &ateletpb.CheckpointResponse{}, nil } diff --git a/cmd/ateapi/internal/controlapi/snapshot_helpers.go b/cmd/ateapi/internal/controlapi/snapshot_helpers.go index 6aa9977b98..03b63e43fd 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_helpers.go +++ b/cmd/ateapi/internal/controlapi/snapshot_helpers.go @@ -60,21 +60,14 @@ func findLatestDurableSnapshot(status *ateapipb.ActorStatus, storageStatus ateap }) } -// findLatestLocalSnapshot returns the highest-generation Snapshot on status -// that holds a completed local checkpoint (Locality != ""), or nil if none -// exists. +// findLatestLocalSnapshot returns the highest-generation snapshot with a local copy. func findLatestLocalSnapshot(status *ateapipb.ActorStatus) *ateapipb.Snapshot { return latestSnapshot(status, func(snap *ateapipb.Snapshot) bool { return snap.GetLocality() != "" }) } -// latestSnapshotIsLocal reports whether the snapshot at the actor's latest -// generation is node-local: the shape a pause leaves behind, which atelet's -// upload of the snapshot doesn't change. A paused actor is SUSPENDED with this -// shape, and resumes only on the node named by Locality. A RUNNING actor -// resumed from pause keeps the local snapshot too, so callers that need "at -// rest" check the state as well. +// latestSnapshotIsLocal reports whether the latest generation has a local copy. func latestSnapshotIsLocal(status *ateapipb.ActorStatus) bool { return snapshotAtLatestGeneration(status).GetLocality() != "" } diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index 1a96d5c32c..09d081b42d 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -261,18 +261,9 @@ func (w *ActorWorkflow) ensureVolumesCreated(ctx context.Context, actorRef resou return storedActor, nil } -// ensureWorkerAssigned leaves the actor RESUMING with a validated, live, -// owned, and eligible worker. -// -// A RESUMING actor was assigned by a previous attempt: its persisted -// assignment is revalidated (worker still exists, not draining, still owned -// by this actor UID, still eligible for the actor's constraints) and reused; -// a stale assignment crashes the actor. A SUSPENDED actor goes through -// scheduling (pinned to its assigned node when it holds a node-local -// snapshot), claiming the worker and persisting RESUMING with the -// assignment. A version conflict there is retried under a bounded backoff -// only after re-reading the actor and revalidating it can still be resumed — -// the conflicting writer may have crashed, drained, or deleted it. +// ensureWorkerAssigned assigns a worker and leaves the actor RESUMING. +// Re-entry validates the recorded assignment before reuse. +// Version conflicts re-read the actor before retrying. func (w *ActorWorkflow) ensureWorkerAssigned(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (_ *ateapipb.Actor, _ *ateapipb.Worker, err error) { ctx, done := stepSpan(ctx, "AssignWorker") defer func() { err = done(err) }() @@ -649,8 +640,6 @@ func toAteletSandboxRuntime(in *ateapipb.SandboxRuntime) *ateletpb.SandboxRuntim func schedulingConstraints(actor *ateapipb.Actor, tmpl *ateapipb.ActorTemplate) scheduling.Constraints { var requiredNodes []string - // A suspended Actor with a local snapshot stays pinned to its node until - // atelet detaches it. When the node is full, ResumeActor detaches it. if node := actor.GetStatus().GetAssignedNode(); node != "" { requiredNodes = []string{node} } @@ -709,18 +698,20 @@ func requiredSnapshotRuntime(actor *ateapipb.Actor, tmpl *ateapipb.ActorTemplate return nil } -// canRelaxRuntime reports whether a resume may drop its RequiredRuntime -// constraint and fall back to restoring without memory state when no -// runtime-compatible worker has capacity. A local pause checkpoint is not -// relaxed: the resume stays pinned to its node, the local restore has no -// fidelity downgrade, and nothing invalidates the checkpoint files if the actor -// boots elsewhere. The pin lifts once atelet uploads the checkpoint and -// detaches it from the node. +// canRelaxRuntime reports whether scheduling may drop its runtime requirement. func canRelaxRuntime(actor *ateapipb.Actor, constraints scheduling.Constraints) bool { if constraints.RequiredRuntime.Runtime() == nil { return false } - return findLatestLocalSnapshot(actor.GetStatus()) == nil + status := actor.GetStatus() + local := findLatestLocalSnapshot(status) + if local == nil { + return true + } + if status.GetState() == ateapipb.ActorState_ACTOR_STATE_SUSPENDED && status.GetAssignedNode() != "" { + return false + } + return local.GetDurableSnapshot().GetStatus() == ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED } // appliesWithFallback reports whether worker satisfies constraints, or — when @@ -808,8 +799,12 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou tele.SnapshotKind = ateattr.SnapshotKindLocal fidelity := fidelityToAtelet(localSnap.GetFidelity()) - if templateReplaced(localSnap, actorTemplate) { + switch { + case templateReplaced(localSnap, actorTemplate): fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + case runtimeIncompatible: + logRuntimeFallback(ctx, actorRef, assignment, src.RequiredRuntime) + fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS } req := &ateletpb.RestoreRequest{ WorkerPodUid: assignment.GetWorkerPodUid(), diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_detach.go b/cmd/ateapi/internal/controlapi/workflow_resume_detach.go index e80e8fe167..7026bd7841 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_detach.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_detach.go @@ -40,12 +40,8 @@ func shouldDetachFromNode(actor *ateapipb.Actor, assignErr error) bool { actor.GetStatus().GetAssignedNode() != "" } -// detachFromFullNode detaches a SUSPENDED actor from its full node when -// another node has room, so the resume restores it there from the durable -// snapshot, and returns the actor as stored. A worker of another runtime -// counts: it restores the snapshot at ROOTFS, without its process memory, and -// assignment still prefers one that keeps it. With no room anywhere it returns -// assignErr. A failed detach leaves the actor on its node. +// detachFromFullNode releases the node binding so scheduling can retry. +// Unreachable nodes can be forgotten when the latest local snapshot is durable. func (w *ActorWorkflow) detachFromFullNode(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate, assignErr error) (_ *ateapipb.Actor, err error) { ctx, done := stepSpan(ctx, "DetachFromFullNode") defer func() { err = done(err) }() @@ -69,6 +65,9 @@ func (w *ActorWorkflow) detachFromFullNode(ctx context.Context, actorRef resourc // their assignments; the loser is left durable-only. ateletConn, err := w.dialer.DialForAteletOnNode(node) if err != nil { + if canDetachWithoutAtelet(ctx, actor, err) { + return w.recordDetach(ctx, actorRef, actor, node, nil) + } return nil, apierror.Unavailable("while getting atelet conn for node %q: %w", node, err) } resp, err := ateletpb.NewAteletClient(ateletConn).DetachActor(ctx, &ateletpb.DetachActorRequest{ @@ -77,6 +76,9 @@ func (w *ActorWorkflow) detachFromFullNode(ctx context.Context, actorRef resourc ActorUid: actor.GetMetadata().GetUid(), }) if err != nil { + if canDetachWithoutAtelet(ctx, actor, err) { + return w.recordDetach(ctx, actorRef, actor, node, nil) + } return nil, detachActorError(ctx, err) } state := resp.GetState() @@ -111,20 +113,20 @@ func (w *ActorWorkflow) loadPinnedActor(ctx context.Context, actorRef resources. return actor, nil } -// recordDetach records the state atelet on node returned from DetachActor on -// the actor as read, and returns it as stored. It fails with -// FailedPrecondition if the latest copy on node was not durable. +// recordDetach forgets local copies after the latest local generation is durable. +// state may be nil when atelet is unreachable. func (w *ActorWorkflow) recordDetach(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, node string, state *ateletpb.ActorSnapshotState) (*ateapipb.Actor, error) { applied := proto.CloneOf(actor.GetStatus()) if err := applyLatestDurable(ctx, applied, snapshotReportFromAtelet(state.GetLatestDurable())); err != nil { return nil, apierror.Internal("atelet on node %q reported a state for Actor %s that conflicts with its record: %w", node, actorRef, err) } - // Under the lease, atelet deleted every copy on node, including any newer - // than the snapshot it reports durable, and reports none when it holds no - // entry for the actor. + // Forget local copies only after the latest recorded generation is durable. if err := applyDetach(applied, node, applied.GetLastAssignedGeneration()); err != nil { return nil, err } + if applied.GetState() == ateapipb.ActorState_ACTOR_STATE_REVERTING && applied.GetWorkerAssignment() == nil { + applied.AssignedNode = "" + } stored, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { // The precondition makes toUpdate the version applied was built from. toUpdate.Status.Snapshots = applied.GetSnapshots() @@ -140,6 +142,25 @@ func (w *ActorWorkflow) recordDetach(ctx context.Context, actorRef resources.Act return stored, nil } +func canDetachWithoutAtelet(ctx context.Context, actor *ateapipb.Actor, err error) bool { + if ctx.Err() != nil || actor.GetStatus().GetWorkerAssignment() != nil { + return false + } + // A failed restore can leave a live workload after its assignment is cleared. + if actor.GetStatus().GetCrash() != nil { + return false + } + code := status.Code(err) + if code == codes.Unknown { + code = apierror.Code(err) + } + if !errors.Is(err, ErrNoAteletOnNode) && code != codes.Unavailable && code != codes.DeadlineExceeded { + return false + } + local := findLatestLocalSnapshot(actor.GetStatus()) + return local != nil && local.GetDurableSnapshot().GetStatus() == ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED +} + // detachActorError maps a DetachActor error to the one the resume fails with. func detachActorError(ctx context.Context, err error) error { // The caller went away or the lease was lost: report that, not atelet's error. diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go index e11902b4e0..20ac9e9a95 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go @@ -119,6 +119,8 @@ func TestDetachFromFullNode(t *testing.T) { snapRuntime *ateapipb.SandboxRuntime // completed records the snapshot's upload COMPLETED. completed bool + cancel bool + noIPs bool // before updates the actor after it was read and before the step // runs, and during while atelet detaches it. before, during func(*ateapipb.Actor) @@ -196,6 +198,52 @@ func TestDetachFromFullNode(t *testing.T) { detachState: detachedState, wantDetached: true, wantCall: true, + }, { + name: "no atelet on the node, upload completed: detaches", + node: "node-3", + otherNode: true, + completed: true, + wantDetached: true, + }, { + name: "atelet unreachable, upload completed: detaches", + node: "node-1", + otherNode: true, + completed: true, + detachErr: apierror.Unavailable("connection refused"), + wantDetached: true, + wantCall: true, + }, { + name: "atelet deadline exceeded, upload completed: detaches", + node: "node-1", + otherNode: true, + completed: true, + detachErr: context.DeadlineExceeded, + wantDetached: true, + wantCall: true, + }, { + name: "actor updated while unreachable: Aborted, record unchanged", + node: "node-1", + otherNode: true, + completed: true, + during: func(*ateapipb.Actor) {}, + detachErr: apierror.Unavailable("connection refused"), + wantCode: codes.Aborted, + wantCall: true, + }, { + name: "workflow canceled during detach: actor stays on its node", + node: "node-1", + otherNode: true, + completed: true, + cancel: true, + detachErr: apierror.Unavailable("connection refused"), + wantCode: codes.Canceled, + }, { + name: "atelet missing IP: actor stays on its node", + node: "node-1", + otherNode: true, + completed: true, + noIPs: true, + wantCode: codes.Unavailable, }, { name: "no atelet on the node: Unavailable, actor stays on its node", node: "node-3", @@ -240,7 +288,8 @@ func TestDetachFromFullNode(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) st := newTestPersistence(t) fillWorker(t, st, seedRuntimeWorker(t, ctx, st, "pod-"+tt.node, tt.node, testSandboxRuntime())) if tt.otherNode { @@ -257,6 +306,12 @@ func TestDetachFromFullNode(t *testing.T) { } actor := seedActorWithSnapshot(t, ctx, st, suspended, tt.node, snap) w, atelet := newDetachTestWorkflow(t, ctx, st) + if tt.noIPs { + w.dialer = NewAteletDialer(newTestAteletIndexer(t, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: installdefaults.SystemNamespace, Name: "atelet-1", UID: "atelet-uid"}, + Spec: corev1.PodSpec{NodeName: tt.node}, + }), installdefaults.SystemNamespace, "", "") + } if tt.detachState != nil { atelet.detachState = tt.detachState(snap) } @@ -281,6 +336,9 @@ func TestDetachFromFullNode(t *testing.T) { if tt.during != nil { atelet.onDetach = func() { update(tt.during) } } + if tt.cancel { + atelet.onDetach = cancel + } // The record unchanged by a failed step is the one as last written. want, err := st.GetActor(ctx, actorRef) if err != nil { @@ -292,7 +350,7 @@ func TestDetachFromFullNode(t *testing.T) { t.Fatalf("detachFromFullNode() = %v, want code %v", err, tt.wantCode) } req := atelet.detachRequest() - if (req != nil) != tt.wantCall { + if !tt.cancel && (req != nil) != tt.wantCall { t.Fatalf("DetachActor request = %v, want a call: %v", req, tt.wantCall) } if req != nil { @@ -302,7 +360,7 @@ func TestDetachFromFullNode(t *testing.T) { } } - after, err := st.GetActor(ctx, actorRef) + after, err := st.GetActor(context.Background(), actorRef) if err != nil { t.Fatalf("GetActor: %v", err) } diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index d932621215..fd651e99e1 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -1103,7 +1103,9 @@ type capturingAtelet struct { detachState *ateletpb.ActorSnapshotState detachErr error // onDetach, when set, runs before DetachActor returns. - onDetach func() + onDetach func() + terminateErr error + onTerminate func() } func (f *capturingAtelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (*ateletpb.RestoreResponse, error) { @@ -1163,6 +1165,22 @@ func (f *capturingAtelet) DetachActor(ctx context.Context, req *ateletpb.DetachA return &ateletpb.DetachActorResponse{State: proto.CloneOf(f.detachState)}, nil } +func (f *capturingAtelet) Terminate(ctx context.Context, req *ateletpb.TerminateRequest) (*ateletpb.TerminateResponse, error) { + f.mu.Lock() + hook := f.onTerminate + f.mu.Unlock() + if hook != nil { + hook() + } + f.mu.Lock() + err := f.terminateErr + f.mu.Unlock() + if err != nil { + return nil, err + } + return f.UnimplementedAteletServer.Terminate(ctx, req) +} + func (f *capturingAtelet) detachRequest() *ateletpb.DetachActorRequest { f.mu.Lock() defer f.mu.Unlock() @@ -1293,6 +1311,11 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { localSnap := func(name string, fidelity ateapipb.SnapshotFidelity) *ateapipb.Snapshot { return newPausedSnapshot(t, 1, fidelity, "", name, "node-1") } + stampedLocalSnap := func(name string, fidelity ateapipb.SnapshotFidelity, rt *ateapipb.SandboxRuntime) *ateapipb.Snapshot { + snap := localSnap(name, fidelity) + snap.SandboxRuntime = rt + return snap + } // pauseUploadURI is where atelet uploads the pause snapshot. pauseUploadURI := someActorSnapshotURI(t, testStorageLocation, "team-a", localSnapshotName) uploadedLocalSnap := func(name string, fidelity ateapipb.SnapshotFidelity) *ateapipb.Snapshot { @@ -1318,11 +1341,10 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { localSnapshot *ateapipb.Snapshot // externalSnapshot seeds a completed durable snapshot. externalSnapshot *ateapipb.Snapshot - // tmplUID seeds the template UID the snapshot's guest state was built - // on, stamped onto externalSnapshot: "current" stands for the created - // template's store-assigned UID (unknown until runtime), "" leaves the - // field unset, anything else mismatches (a repointed actor). + // tmplUID stamps externalSnapshot. "current" means the created template's UID. tmplUID string + // localTmplUID stamps localSnapshot independently. + localTmplUID string // sourceTag seeds Actor.SourceTag when testing explicit tag clones. sourceTag *ateapipb.ObjectRef // assignedRuntime seeds WorkerAssignment.SandboxRuntime. @@ -1526,7 +1548,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { name: "14 local snapshot built on the current template stays Full", actor: actorSeed{ localSnapshot: localSnap(localSnapshotName, fullScope), - tmplUID: "current", + localTmplUID: "current", }, want: restoreWant{ snapshotName: localSnapshotName, @@ -1538,7 +1560,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { name: "14a repointed actor's Full local snapshot drops to Data", actor: actorSeed{ localSnapshot: localSnap(localSnapshotName, fullScope), - tmplUID: "mismatch", + localTmplUID: "mismatch", }, want: restoreWant{ snapshotName: localSnapshotName, @@ -1550,7 +1572,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { name: "14b repointed actor's Rootfs local snapshot drops to Data", actor: actorSeed{ localSnapshot: localSnap(localSnapshotName, rootfsScope), - tmplUID: "mismatch", + localTmplUID: "mismatch", }, want: restoreWant{ snapshotName: localSnapshotName, @@ -1609,6 +1631,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }(), externalSnapshot: extSnap(actorURI, fullScope), tmplUID: "mismatch", + localTmplUID: "current", }, want: restoreWant{ snapshotName: localSnapshotName, @@ -1697,6 +1720,32 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, }, }, + { + name: "24a stamped local snapshot on incompatible worker drops to Rootfs", + actor: actorSeed{ + localSnapshot: stampedLocalSnap(localSnapshotName, fullScope, amd64RT), + localTmplUID: "current", + assignedRuntime: arm64RT, + }, + want: restoreWant{ + snapshotName: localSnapshotName, + snapshotURI: pauseUploadURI, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, + }, + }, + { + name: "24b stamped local snapshot on matching worker restores in Full", + actor: actorSeed{ + localSnapshot: stampedLocalSnap(localSnapshotName, fullScope, amd64RT), + localTmplUID: "current", + assignedRuntime: amd64RT, + }, + want: restoreWant{ + snapshotName: localSnapshotName, + snapshotURI: pauseUploadURI, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + }, + }, { // A recorded upload changes nothing: a local restore names the // pause snapshot's upload whether or not it is COMPLETED. @@ -1772,9 +1821,11 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { a.SourceTag = tt.actor.sourceTag a.Status.WorkerAssignment = wireTestAssignment() a.Status.WorkerAssignment.SandboxRuntime = proto.CloneOf(tt.actor.assignedRuntime) - uid := tt.actor.tmplUID - if uid == "current" { - uid = createdTmpl.GetMetadata().GetUid() + resolve := func(uid string) string { + if uid == "current" { + return createdTmpl.GetMetadata().GetUid() + } + return uid } var gen int32 if tt.actor.externalSnapshot != nil { @@ -1783,7 +1834,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { ext := proto.CloneOf(tt.actor.externalSnapshot) ext.Generation = gen if ext.ActorTemplateUid == "" { - ext.ActorTemplateUid = uid + ext.ActorTemplateUid = resolve(tt.actor.tmplUID) } a.Status.Snapshots = append(a.Status.Snapshots, ext) } @@ -1794,7 +1845,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { loc := proto.CloneOf(tt.actor.localSnapshot) loc.Generation = gen if loc.ActorTemplateUid == "" { - loc.ActorTemplateUid = uid + loc.ActorTemplateUid = resolve(tt.actor.localTmplUID) } else if loc.ActorTemplateUid == "current" { loc.ActorTemplateUid = createdTmpl.GetMetadata().GetUid() } @@ -2750,6 +2801,49 @@ func TestValidateAssignedWorker_RuntimeFallback(t *testing.T) { t.Errorf("validated worker = %q, want %q", got, "arm-pod") } }) + + t.Run("accepts fallback worker on re-entry for an uploaded pause snapshot", func(t *testing.T) { + ctx := context.Background() + st := newTestPersistence(t) + arm64RT := testRuntime("arm64", "") + seedRuntimeWorker(t, ctx, st, "arm-pod", "node-arm", arm64RT) + seedAssignment(t, st, testWorkerUID("arm-pod"), &ateapipb.ActorAssignment{ + Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "id1"}, + ActorUid: "own-actor-uid", + }) + + // Re-entry must accept a previously allowed ROOTFS assignment. + local := newPausedSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-1", "node-arm") + local.SandboxRuntime = testSandboxRuntime() + local.DurableSnapshot.Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + resumingActor := &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "id1", Uid: "own-actor-uid"}, + Status: &ateapipb.ActorStatus{ + State: ateapipb.ActorState_ACTOR_STATE_RESUMING, + AssignedNode: "node-arm", + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{local}, + WorkerAssignment: &ateapipb.WorkerAssignment{ + Worker: &ateapipb.ObjectRef{Name: testWorkerUID("arm-pod")}, + WorkerNamespace: "worker-ns", + WorkerPool: "pool", + WorkerPod: "arm-pod", + WorkerPodUid: testWorkerUID("arm-pod"), + NodeName: "node-arm", + SandboxRuntime: arm64RT, + }, + }, + } + + w := &ActorWorkflow{store: st, scheduler: scheduling.New(nil)} + worker, err := w.validateAssignedWorker(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, resumingActor, &ateapipb.ActorTemplate{}) + if err != nil { + t.Fatalf("validateAssignedWorker: %v", err) + } + if got := worker.GetWorkerPod(); got != "arm-pod" { + t.Errorf("validated worker = %q, want %q", got, "arm-pod") + } + }) } // TestRequiredSnapshotRuntime pins which snapshot a resume takes its runtime @@ -2770,8 +2864,8 @@ func TestRequiredSnapshotRuntime(t *testing.T) { snap.SandboxRuntime = rt return snap } - local := func(gen int32, fidelity ateapipb.SnapshotFidelity, rt *ateapipb.SandboxRuntime) *ateapipb.Snapshot { - snap := newPausedSnapshot(t, gen, fidelity, tmplUID, "local", "node-1") + local := func(gen int32, fidelity ateapipb.SnapshotFidelity, capturedUnder string, rt *ateapipb.SandboxRuntime) *ateapipb.Snapshot { + snap := newPausedSnapshot(t, gen, fidelity, capturedUnder, "local", "node-1") snap.SandboxRuntime = rt return snap } @@ -2814,18 +2908,17 @@ func TestRequiredSnapshotRuntime(t *testing.T) { }, { name: "local checkpoint requires its runtime", - snapshots: []*ateapipb.Snapshot{local(1, memory, localRT)}, + snapshots: []*ateapipb.Snapshot{local(1, memory, tmplUID, localRT)}, want: localRT, }, { - name: "local checkpoint captured under a replaced template imposes no constraint", - snapshots: []*ateapipb.Snapshot{ - func() *ateapipb.Snapshot { - s := newPausedSnapshot(t, 1, memory, "old-tmpl-uid", "local", "node-1") - s.SandboxRuntime = localRT - return s - }(), - }, + name: "local checkpoint with no template uid keeps its runtime", + snapshots: []*ateapipb.Snapshot{local(1, memory, "", localRT)}, + want: localRT, + }, + { + name: "local checkpoint captured under a replaced template is not restored", + snapshots: []*ateapipb.Snapshot{local(1, memory, "old-tmpl-uid", localRT)}, }, { // ensureAteletRestored restores a local checkpoint before any @@ -2833,7 +2926,7 @@ func TestRequiredSnapshotRuntime(t *testing.T) { // which is newer. name: "local checkpoint takes precedence over a newer durable snapshot", snapshots: []*ateapipb.Snapshot{ - local(1, memory, localRT), + local(1, memory, tmplUID, localRT), durable(2, memory, tmplUID, completed, durableRT), }, want: localRT, @@ -2841,17 +2934,26 @@ func TestRequiredSnapshotRuntime(t *testing.T) { { name: "ROOTFS local checkpoint imposes no constraint beside a stamped durable snapshot", snapshots: []*ateapipb.Snapshot{ - local(2, rootfs, localRT), + local(2, rootfs, tmplUID, localRT), durable(1, memory, tmplUID, completed, durableRT), }, }, { name: "VOLUMES local checkpoint imposes no constraint beside a stamped durable snapshot", snapshots: []*ateapipb.Snapshot{ - local(2, volumes, localRT), + local(2, volumes, tmplUID, localRT), durable(1, memory, tmplUID, completed, durableRT), }, }, + { + // Restore uses the newer local snapshot's template. + name: "local checkpoint ignores an older durable snapshot's replaced template", + snapshots: []*ateapipb.Snapshot{ + durable(1, memory, "old-tmpl-uid", completed, durableRT), + local(2, memory, tmplUID, localRT), + }, + want: localRT, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -2863,48 +2965,116 @@ func TestRequiredSnapshotRuntime(t *testing.T) { } } -// TestCanRelaxRuntime pins when a resume may drop its runtime constraint: only -// when it has one, and never while a local pause checkpoint pins the actor. func TestCanRelaxRuntime(t *testing.T) { rt := testSandboxRuntime() memory := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + completed := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + pending := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + suspended := ateapipb.ActorState_ACTOR_STATE_SUSPENDED + resuming := ateapipb.ActorState_ACTOR_STATE_RESUMING + durable := func(gen int32) *ateapipb.Snapshot { - return newDurableSnapshot(gen, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", "snap", "gs://bucket/snap", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + return newDurableSnapshot(gen, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", "snap", "gs://bucket/snap", completed) } local := func(gen int32) *ateapipb.Snapshot { return newPausedSnapshot(t, gen, memory, "", "local", "node-1") } + localDurable := func(gen int32, storageStatus ateapipb.SnapshotStorageStatus) *ateapipb.Snapshot { + snap := local(gen) + snap.GetDurableSnapshot().Status = storageStatus + return snap + } + localNoDurable := func(gen int32) *ateapipb.Snapshot { + snap := local(gen) + snap.DurableSnapshot = nil + return snap + } tests := []struct { name string + state ateapipb.ActorState + node string snapshots []*ateapipb.Snapshot required *ateapipb.SandboxRuntime want bool }{ { name: "nothing to relax without a runtime constraint", + state: suspended, snapshots: []*ateapipb.Snapshot{durable(1)}, }, { name: "durable snapshot may fall back", + state: suspended, snapshots: []*ateapipb.Snapshot{durable(1)}, required: rt, want: true, }, { name: "local checkpoint stays pinned", + state: suspended, + node: "node-1", snapshots: []*ateapipb.Snapshot{local(1)}, required: rt, }, { name: "local checkpoint stays pinned beside a durable snapshot", + state: suspended, + node: "node-1", snapshots: []*ateapipb.Snapshot{durable(1), local(2)}, required: rt, }, + { + name: "uploaded checkpoint detaches before relaxing a pinned actor", + state: suspended, + node: "node-1", + snapshots: []*ateapipb.Snapshot{localDurable(2, completed)}, + required: rt, + }, + { + name: "unbound uploaded checkpoint may fall back", + state: suspended, + snapshots: []*ateapipb.Snapshot{localDurable(2, completed)}, + required: rt, + want: true, + }, + { + name: "resuming uploaded checkpoint keeps its relaxed assignment", + state: resuming, + node: "node-1", + snapshots: []*ateapipb.Snapshot{localDurable(2, completed)}, + required: rt, + want: true, + }, + { + name: "resuming pending checkpoint cannot relax", + state: resuming, + node: "node-1", + snapshots: []*ateapipb.Snapshot{localDurable(2, pending)}, + required: rt, + }, + { + name: "resuming checkpoint without a durable copy cannot relax", + state: resuming, + node: "node-1", + snapshots: []*ateapipb.Snapshot{localNoDurable(2)}, + required: rt, + }, + { + name: "older durable copy cannot authorize a resuming checkpoint", + state: resuming, + node: "node-1", + snapshots: []*ateapipb.Snapshot{durable(1), localNoDurable(2)}, + required: rt, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - actor := &ateapipb.Actor{Status: &ateapipb.ActorStatus{Snapshots: tc.snapshots}} + actor := &ateapipb.Actor{Status: &ateapipb.ActorStatus{ + State: tc.state, + AssignedNode: tc.node, + Snapshots: tc.snapshots, + }} if got := canRelaxRuntime(actor, scheduling.Constraints{RequiredRuntime: hardware.NewRequirement(tc.required)}); got != tc.want { t.Errorf("canRelaxRuntime = %v, want %v", got, tc.want) } diff --git a/cmd/ateapi/internal/controlapi/workflow_revert.go b/cmd/ateapi/internal/controlapi/workflow_revert.go index dcae8cc7c0..2d6f447797 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert.go @@ -186,16 +186,21 @@ func (w *ActorWorkflow) ensureWorkerDiscarded(ctx context.Context, actorRef reso defer func() { err = done(err) }() if actor.GetStatus().GetAssignedNode() != "" { + forgetNode := false if err := w.ensureAteletTerminated(ctx, actorRef, actor, actorTemplate, ateattr.OperationRevert); err != nil { - // A failed terminate leaves the actor REVERTING with its - // assignment, so the next revert terminates it again. If the - // worker's pod goes away, worker deletion crashes the actor, and - // the next revert needs no live worker. - return err + if !canDetachWithoutAtelet(ctx, actor, err) { + return err + } + forgetNode = true } if err := w.ensureVolumesDetached(ctx, actor, actorTemplate, "DetachVolumesForRevert", ateattr.OperationRevert); err != nil { return err } + if forgetNode { + if _, err := w.recordDetach(ctx, actorRef, actor, actor.GetStatus().GetAssignedNode(), nil); err != nil { + return err + } + } if actor.GetStatus().GetWorkerAssignment() != nil { if _, _, err := releaseWorker(ctx, w.store, actor); err != nil { return fmt.Errorf("while releasing worker: %w", err) @@ -206,11 +211,8 @@ func (w *ActorWorkflow) ensureWorkerDiscarded(ctx context.Context, actorRef reso return w.releaseAssignmentWithoutBacklink(ctx, actor) } -// ensureInProgressSnapshotDiscarded deletes the objects of every PENDING -// durable snapshot on the actor: the one a suspend was partway through writing -// when the actor was reverted, and any earlier pause's whose upload was never -// recorded. The revert keeps none of them. atelet's upload of them has already -// been canceled when ensureWorkerDiscarded terminated the actor. +// ensureInProgressSnapshotDiscarded removes pending uploads of the discarded execution. +// The actor keeps its latest completed durable snapshot. func (w *ActorWorkflow) ensureInProgressSnapshotDiscarded(ctx context.Context, actor *ateapipb.Actor) (err error) { ctx, done := stepSpan(ctx, "DiscardInProgressSnapshot") defer func() { err = done(err) }() diff --git a/cmd/ateapi/internal/controlapi/workflow_revert_test.go b/cmd/ateapi/internal/controlapi/workflow_revert_test.go index 64de763c38..6a346166da 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/cmd/ateapi/internal/store" "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/objectstore/objectstoretest" @@ -26,6 +27,7 @@ import ( "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" + "google.golang.org/protobuf/proto" ) // TestEnsureMarkedReverting_StateMatrix pins which states a revert is accepted @@ -220,6 +222,119 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { } } +func TestRevertActor_UnreachableNode(t *testing.T) { + for _, tt := range []struct { + name string + node string + err error + pending bool + assigned bool + crashed bool + conflict bool + wantCode codes.Code + }{ + {name: "missing atelet", node: "node-3"}, + {name: "unavailable atelet", node: "node-1", err: apierror.Unavailable("offline")}, + {name: "atelet deadline", node: "node-1", err: context.DeadlineExceeded}, + {name: "pending beside older completed snapshot", node: "node-1", err: apierror.Unavailable("offline"), pending: true, wantCode: codes.Unavailable}, + {name: "worker still assigned", node: "node-1", err: apierror.Unavailable("offline"), assigned: true, wantCode: codes.Unavailable}, + {name: "crashed execution still needs termination", node: "node-1", err: apierror.Unavailable("offline"), crashed: true, wantCode: codes.Unavailable}, + {name: "other RPC failure", node: "node-1", err: apierror.Internal("injected failure"), wantCode: codes.Internal}, + {name: "concurrent update", node: "node-1", err: apierror.Unavailable("offline"), conflict: true, wantCode: codes.Aborted}, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + st, cleanup := storetest.SetupTestStore(t) + t.Cleanup(cleanup) + w := newTestActorWorkflow(t, st, "ns", "tmpl1") + wire, atelet := newWireCaptureWorkflow(t, st) + w.dialer = wire.dialer + ref := resources.ActorRef{Atespace: "team-a", Name: "id1"} + seedWorkflowActor(t, ctx, st, ref, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDED) + actor, err := st.GetActor(ctx, ref) + if err != nil { + t.Fatal(err) + } + actor = mustUpdateActorStatus(t, ctx, st, actor, func(s *ateapipb.ActorStatus) { + s.AssignedNode = tt.node + s.LastAssignedGeneration = 2 + snap := newPausedSnapshot(t, 2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "uploaded", tt.node) + snap.DurableSnapshot.Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + if tt.pending { + snap.DurableSnapshot.Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + } + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, snap.GetFidelity(), "", "older", "gs://snapshots/team-a/actors/older/snapshot", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + snap, + } + if tt.assigned { + s.State = ateapipb.ActorState_ACTOR_STATE_RUNNING + s.WorkerAssignment = wireTestAssignment() + } + if tt.crashed { + s.State = ateapipb.ActorState_ACTOR_STATE_CRASHED + s.Crash = newActorCrash("ResumeActor", "interrupted execution") + } + }) + want := proto.CloneOf(actor.GetStatus().GetSnapshots()[1]) + atelet.terminateErr = tt.err + if tt.conflict { + atelet.onTerminate = func() { + cur, err := st.GetActor(ctx, ref) + if err != nil { + t.Errorf("GetActor: %v", err) + return + } + if _, err := st.UpdateActor(ctx, ref, store.PreconditionFrom(cur), func(a *ateapipb.Actor) error { + a.WorkerSelector = &ateapipb.Selector{MatchLabels: map[string]string{"changed": "true"}} + return nil + }); err != nil { + t.Errorf("UpdateActor: %v", err) + } + } + } + reverted, err := w.RevertActor(ctx, ref) + if got := apierror.Code(err); got != tt.wantCode { + t.Fatalf("RevertActor = %v, want %v", err, tt.wantCode) + } + if tt.wantCode != codes.OK { + stored, err := st.GetActor(ctx, ref) + if err != nil { + t.Fatal(err) + } + got := stored.GetStatus() + if got.GetState() != ateapipb.ActorState_ACTOR_STATE_REVERTING || got.GetAssignedNode() != tt.node { + t.Fatalf("status = %v, want REVERTING on %s", got, tt.node) + } + if !proto.Equal(got.GetSnapshots()[1], want) { + t.Fatalf("snapshot changed after failed revert: %v", got.GetSnapshots()) + } + if !tt.conflict { + return + } + atelet.mu.Lock() + atelet.onTerminate = nil + atelet.mu.Unlock() + reverted, err = w.RevertActor(ctx, ref) + if err != nil { + t.Fatalf("retry after conflict: %v", err) + } + } + got := reverted.GetStatus() + if got.GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED || got.GetAssignedNode() != "" || got.GetWorkerAssignment() != nil { + t.Fatalf("reverted status = %v, want unbound SUSPENDED", got) + } + if len(got.GetSnapshots()) != 1 { + t.Fatalf("snapshots = %v, want only the latest durable snapshot", got.GetSnapshots()) + } + want.Locality = "" + if !proto.Equal(got.GetSnapshots()[0], want) { + t.Errorf("retained snapshot = %v, want %v", got.GetSnapshots()[0], want) + } + }) + } +} + // TestRevertActor_PausedSnapshotAlreadyUploaded verifies a revert of a paused // actor whose snapshot atelet uploaded returns it to that upload, its latest // durable snapshot, and collects the durable snapshot the upload replaced. The diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend.go b/cmd/ateapi/internal/controlapi/workflow_suspend.go index 2dfa0066cb..52d42c15eb 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend.go @@ -402,11 +402,12 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r switch mode { case suspendSync: snap.GetDurableSnapshot().Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED - // Remove all local snapshots since they are superseded by the new durable snapshot. - // TODO: This will change when an async suspend uploads its snapshot - // and both local and durable snapshots are kept. - pruneSnapshots(toUpdate.Status, nil) - toUpdate.Status.AssignedNode = "" + if ateletKeptLocal(ateletState, snap) { + snap.Locality = toUpdate.GetStatus().GetAssignedNode() + } else { + toUpdate.Status.AssignedNode = "" + } + pruneSnapshots(toUpdate.Status, snap) case suspendAsync: // Record the node locality of the local checkpoint and clear older local checkpoints. // AssignedNode is kept: a paused actor resumes on that node. @@ -434,6 +435,16 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r return storedActor, nil } +// ateletKeptLocal reports whether atelet retained this checkpoint. +func ateletKeptLocal(state *ateletpb.ActorSnapshotState, snap *ateapipb.Snapshot) bool { + local := state.GetLatestLocal().GetSnapshot() + uri := snap.GetDurableSnapshot().GetObject().GetSnapshotUri() + return local != nil && uri != "" && + local.GetSnapshotUuid() == snap.GetUuid() && + local.GetGeneration() == snap.GetGeneration() && + local.GetObject().GetSnapshotUri() == uri +} + // releaseDroppedSnapshots releases each external snapshot that before records // as COMPLETED and after doesn't. A failure is only logged, and leaves the // snapshot's objects in storage until the actor is deleted, which removes its diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index afb704349b..b46186de57 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -1548,3 +1548,94 @@ func TestSuspendActor_AppliesCheckpointState(t *testing.T) { }) } } + +func TestSuspendActor_Finalized_RecordsLocalCopyWhenKept(t *testing.T) { + const node = "node-1" + memory := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + completed := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + localReport := func(snap *ateletpb.Snapshot) *ateletpb.ActorSnapshotState { + return &ateletpb.ActorSnapshotState{LatestLocal: &ateletpb.SnapshotReport{Snapshot: snap}} + } + variant := func(req *ateletpb.CheckpointRequest, mutate func(*ateletpb.Snapshot)) *ateletpb.Snapshot { + snap := proto.CloneOf(req.GetSnapshot()) + mutate(snap) + return snap + } + tests := []struct { + name string + state func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState + wantLocality string + }{{ + name: "kept", + state: func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState { + return localReport(req.GetSnapshot()) + }, + wantLocality: node, + }, { + name: "not kept", + state: func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState { + return &ateletpb.ActorSnapshotState{LatestDurable: &ateletpb.SnapshotReport{Snapshot: req.GetSnapshot()}} + }, + }, { + name: "uuid mismatch", + state: func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState { + return localReport(variant(req, func(snap *ateletpb.Snapshot) { snap.SnapshotUuid = "another-snapshot" })) + }, + }, { + name: "generation mismatch", + state: func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState { + return localReport(variant(req, func(snap *ateletpb.Snapshot) { snap.Generation = req.GetSnapshot().GetGeneration() - 1 })) + }, + }, { + name: "uri mismatch", + state: func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState { + return localReport(variant(req, func(snap *ateletpb.Snapshot) { + snap.Object = &ateletpb.ObjectStorage{SnapshotUri: "gs://bucket/root/another-snapshot"} + })) + }, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: testStorageLocation, PreferredFidelity: memory}) + actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} + created := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, + ActorTemplate: suspendTestTemplateRef(), + Status: &ateapipb.ActorStatus{ + State: ateapipb.ActorState_ACTOR_STATE_RUNNING, + AssignedNode: node, + WorkerAssignment: wireTestAssignment(), + }, + }) + seedWireWorker(t, ctx, persistence, assignmentOf(created)) + w, atelet := newWireCaptureWorkflow(t, persistence) + atelet.onCheckpoint = func(context.Context) error { + st := tt.state(atelet.checkpointRequest()) + atelet.mu.Lock() + atelet.checkpointState = st + atelet.mu.Unlock() + return nil + } + + if _, err := w.SuspendActor(ctx, actorRef, suspendSync); err != nil { + t.Fatalf("suspend: %v", err) + } + stored := mustReadActor(t, ctx, persistence, actorRef) + if got := stored.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + t.Errorf("state = %v, want SUSPENDED", got) + } + if got := stored.GetStatus().GetAssignedNode(); got != tt.wantLocality { + t.Errorf("AssignedNode = %q, want %q", got, tt.wantLocality) + } + if got := stored.GetStatus().GetWorkerAssignment(); got != nil { + t.Errorf("WorkerAssignment = %v, want cleared", got) + } + want := []snapshotOutcome{{generation: 1, fidelity: memory, locality: tt.wantLocality, durable: completed}} + if diff := cmp.Diff(want, describeSnapshots(stored.GetStatus()), cmp.AllowUnexported(snapshotOutcome{})); diff != "" { + t.Errorf("snapshots mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/docs/api-guide.md b/docs/api-guide.md index 0c79a76109..e883609471 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -371,20 +371,12 @@ a `Snapshot` with the following fields: `SNAPSHOT_OWNER_TAG`). - `fidelity`: The state layers the snapshot holds (`SNAPSHOT_FIDELITY_VOLUMES`, `SNAPSHOT_FIDELITY_ROOTFS`, or `SNAPSHOT_FIDELITY_MEMORY`). -- `actorTemplateUid`: The UID of the `ActorTemplate` whose sandbox the guest - state was captured from. This is not always the template the actor points at - now: an actor may be repointed while `SUSPENDED`, while the snapshot on disk - still came from the old one. A resume that finds the two disagree restores the - durable data only and boots the guest fresh, because memory captured under one - sandbox image cannot be resumed under another. -- `locality`: The node that holds the snapshot's local files. It is set when a - pause completes and cleared once those files are deleted from the node (when - atelet detaches the actor in the background, the actor is next paused or - suspended, or it is reverted). A resume restores from the files without - deleting them, so an actor resumed from pause still reports where they are. A - paused actor is an `ACTOR_STATE_SUSPENDED` actor whose latest snapshot has - `locality`; it is pinned to that node (`status.assignedNode`) until atelet - detaches it or it is reverted. +- `actorTemplateUid`: The UID of the `ActorTemplate` used to capture this snapshot. + An actor can change templates while `SUSPENDED`. A resume checks the snapshot + it restores. A snapshot captured under a replaced template restores volume + data and boots fresh. +- `locality`: The node holding the snapshot's local files. For attachment rules, + see `ActorStatus.assigned_node` in the [API schema](../pkg/proto/ateapipb/ateapi.proto). - `durableSnapshot`: The copy in object storage, with its URI in `object.snapshotUri` and a `status` that is `PENDING` while the copy is being written and `COMPLETED` once it can be restored from. A pause records the copy @@ -406,11 +398,12 @@ a `Snapshot` with the following fields: worker and downgrades the restore to `SNAPSHOT_FIDELITY_ROOTFS`, preserving the snapshot's rootfs and volume data while booting the guest fresh under the new worker's `defaultRuntime`. - - A local pause checkpoint (with `locality`) is not relaxed; the resume - returns `RESOURCE_EXHAUSTED` and the actor stays paused on its node until a - compatible worker there has room, or until atelet detaches it. A detach - uploads the checkpoint, deletes the local files, and lifts the node pin, - after which the next resume may fall back like a durable snapshot. + - A local checkpoint first tries compatible workers on its attached node. + If that fails, `ResumeActor` checks capacity without the node and runtime + restrictions. When that capacity exists, it detaches the checkpoint and + retries placement. Detach uploads it if needed, deletes the local files, + and releases the binding. The retry prefers compatible workers before + using the durable fallback above. A detach error leaves the actor attached. A snapshot without `sandboxRuntime` or with `SNAPSHOT_FIDELITY_ROOTFS` or `SNAPSHOT_FIDELITY_VOLUMES` imposes no runtime placement constraint. Because a @@ -613,7 +606,7 @@ Replaces the mutable fields of an existing actor with the ones in the request (` * `actor`: `Actor` — the complete replacement actor. `metadata.atespace` and `metadata.name` identify the resource; `metadata.uid` and `metadata.version` are **required** preconditions. `metadata` and `status` are server-owned and whatever the request carries in them is ignored. `source_tag` is immutable. * `actor_template` may change only while the actor is `SUSPENDED`. The new template must declare the same volumes and volume mounts. Once the actor owns an external snapshot, the new template must also store snapshots under the same location: its `snapshotConfig.storageLocation` with the `PerTemplateSnapshotLocation` preview gate, otherwise ate-api-server's `--snapshot-location` (see [Snapshot Storage Layout](#snapshot-storage-layout)). * `worker_selector` may be updated or cleared (`nil`) in any state; the updated selector takes effect on the next `ResumeActor`. - * A paused actor stays pinned to its node until atelet detaches it (see `locality` in [Snapshot Storage Layout](#snapshot-storage-layout)). Its next `ResumeActor` therefore needs a worker on that node that matches both the new template's `workerSelector` and the actor's updated `worker_selector`. If no worker there matches, `ResumeActor` returns `RESOURCE_EXHAUSTED` until the detach lifts the pin. + * A paused actor is initially pinned to its node (see `locality` in [Snapshot Storage Layout](#snapshot-storage-layout)). `ResumeActor` first looks for a worker there matching the template and actor selectors. If none is available but a worker satisfies the remaining constraints without node or runtime restrictions, resume detaches the checkpoint and retries placement. If no eligible worker has room, it returns `RESOURCE_EXHAUSTED`. * **Response:** the updated `Actor`. * **Errors:** `INVALID_ARGUMENT` if `uid` or `version` is unset, or if the request changes an immutable field — including by leaving one unset; `FAILED_PRECONDITION` if a change to `actor_template` breaks one of the rules above; `ABORTED` if either guard no longer matches the stored resource. diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index c39bf6f695..413d0e17ff 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -732,10 +732,8 @@ type Snapshot struct { // +k8s:optional // +k8s:format=k8s-uuid ActorTemplateUid string `protobuf:"bytes,5,opt,name=actor_template_uid,json=actorTemplateUid,proto3" json:"actor_template_uid,omitempty"` - // locality identifies the node holding this snapshot's files (currently the - // Kubernetes node name). Set when a local pause checkpoint completes, and - // cleared once the files are deleted from that node; a resume does not - // delete them. + // locality identifies the Kubernetes node holding this snapshot's files. + // Detach, revert, or a later checkpoint clears it. Resume keeps the files. // // +k8s:optional // +k8s:format=k8s-long-name @@ -2057,9 +2055,9 @@ type ActorStatus struct { // // +k8s:optional Crash *ActorCrash `protobuf:"bytes,4,opt,name=crash,proto3" json:"crash,omitempty"` - // assigned_node is the Kubernetes node the Actor is currently attached to - // (hosting its active worker or its node-local snapshots). Unset when the - // Actor is detached from any node. + // assigned_node is the node hosting the Actor's worker or local snapshot. + // A suspended Actor remains attached while its local snapshot is kept. + // Detach or revert clears the binding. // // +k8s:optional // +k8s:format=k8s-long-name diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index fcc3a8976b..9445beeec0 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -354,10 +354,8 @@ message Snapshot { // +k8s:format=k8s-uuid string actor_template_uid = 5; - // locality identifies the node holding this snapshot's files (currently the - // Kubernetes node name). Set when a local pause checkpoint completes, and - // cleared once the files are deleted from that node; a resume does not - // delete them. + // locality identifies the Kubernetes node holding this snapshot's files. + // Detach, revert, or a later checkpoint clears it. Resume keeps the files. // // +k8s:optional // +k8s:format=k8s-long-name @@ -902,9 +900,9 @@ message ActorStatus { // +k8s:optional ActorCrash crash = 4; - // assigned_node is the Kubernetes node the Actor is currently attached to - // (hosting its active worker or its node-local snapshots). Unset when the - // Actor is detached from any node. + // assigned_node is the node hosting the Actor's worker or local snapshot. + // A suspended Actor remains attached while its local snapshot is kept. + // Detach or revert clears the binding. // // +k8s:optional // +k8s:format=k8s-long-name From f302cf0b82474195d566f62b39e16d09abfe5139 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Tue, 20 Oct 2026 17:12:59 +0000 Subject: [PATCH 05/63] hardening: fix submounts of read-only image volumes being writable (#2944) Image volumes were only read-only at the top mount. Mounts nested under them, or mounted under the source later, stayed writable. Current image volumes have no submounts, so this is hardening. Fixes #2933 - micro-VM: the share binds for image and system-info volumes are now read-only and private all the way down - imagecache: the single-layer image volume bind gets the same treatment. gVisor binds image volumes non-recursively, so gVisor actors never saw submounts, and this is hygiene The new root-gated test cases each fail without their fix. I ran them locally as root. AI use: AI-assisted, I reviewed and tested it. - [x] Tests pass - [x] Docs updated --- .../internal/kata/overlay_linux.go | 17 ++--- .../internal/kata/overlay_linux_test.go | 68 +++++++++++++------ internal/imagecache/bundle_linux.go | 9 ++- internal/imagecache/bundle_linux_test.go | 31 +++++++-- 4 files changed, 84 insertions(+), 41 deletions(-) diff --git a/cmd/ateworker-microvm/internal/kata/overlay_linux.go b/cmd/ateworker-microvm/internal/kata/overlay_linux.go index ddf2ac6e85..9d0ae91cca 100644 --- a/cmd/ateworker-microvm/internal/kata/overlay_linux.go +++ b/cmd/ateworker-microvm/internal/kata/overlay_linux.go @@ -206,19 +206,12 @@ func Unmount(dst string) { } } -// RemountReadOnly remounts a bind mount at dst as read-only, preserving existing -// per-mount flags (nosuid, nodev, noexec, etc.) like libmount's remount,bind,ro. +// RemountReadOnly makes the mount at dst and every mount beneath it read-only, +// keeping their other flags. They are also made private, otherwise a mount made +// under the source later would propagate in writable. func RemountReadOnly(dst string) error { - var st unix.Statfs_t - if err := unix.Statfs(dst, &st); err != nil { - return fmt.Errorf("statfs %q: %w", dst, err) - } - flags := unix.MS_BIND | unix.MS_REMOUNT | unix.MS_RDONLY | - int(st.Flags&(unix.ST_NOSUID|unix.ST_NODEV|unix.ST_NOEXEC|unix.ST_NOATIME|unix.ST_NODIRATIME)) - if err := unix.Mount("", dst, "", uintptr(flags), ""); err != nil { - return err - } - return nil + attr := &unix.MountAttr{Attr_set: unix.MOUNT_ATTR_RDONLY, Propagation: unix.MS_PRIVATE} + return unix.MountSetattr(unix.AT_FDCWD, dst, unix.AT_RECURSIVE, attr) } // StageImageVolume bind-mounts one composed image volume read-only at diff --git a/cmd/ateworker-microvm/internal/kata/overlay_linux_test.go b/cmd/ateworker-microvm/internal/kata/overlay_linux_test.go index fb9ca8269f..14f2130bd4 100644 --- a/cmd/ateworker-microvm/internal/kata/overlay_linux_test.go +++ b/cmd/ateworker-microvm/internal/kata/overlay_linux_test.go @@ -100,50 +100,74 @@ func TestVirtiofsdArgs(t *testing.T) { } } -func TestRemountReadOnly_NonExistent(t *testing.T) { - dst := filepath.Join(t.TempDir(), "nonexistent") - err := RemountReadOnly(dst) - if err == nil { - t.Fatalf("RemountReadOnly(%q) = nil, want error", dst) - } - if !strings.Contains(err.Error(), "statfs") { - t.Errorf("RemountReadOnly(%q) error = %v, want error from statfs", dst, err) - } -} - func TestUnmount_NonExistent(t *testing.T) { dst := filepath.Join(t.TempDir(), "nonexistent") // Unmount must handle non-existent destinations cleanly without panicking. Unmount(dst) } +// Image volumes are rbound into the share, so submounts must go read-only too +// without losing their other flags, and later mounts under the source must not +// propagate in. func TestRemountReadOnlyAndUnmount(t *testing.T) { roottest.Require(t, "bind mounts") dir := t.TempDir() src := filepath.Join(dir, "src") dst := filepath.Join(dir, "dst") - if err := os.Mkdir(src, 0o755); err != nil { - t.Fatal(err) + sub := filepath.Join(src, "sub") + later := filepath.Join(src, "later") + for _, d := range []string{src, dst, sub, later} { + if err := os.Mkdir(d, 0o755); err != nil { + t.Fatal(err) + } } - if err := os.Mkdir(dst, 0o755); err != nil { - t.Fatal(err) + // a shared source, so mounts under it propagate to binds of it + if err := unix.Mount(src, src, "", unix.MS_BIND, ""); err != nil { + t.Fatalf("self-binding %q: %v", src, err) + } + t.Cleanup(func() { _ = unix.Unmount(src, unix.MNT_DETACH) }) + if err := unix.Mount("", src, "", unix.MS_SHARED, ""); err != nil { + t.Fatalf("making %q shared: %v", src, err) + } + if err := unix.Mount("tmpfs", sub, "tmpfs", 0, ""); err != nil { + t.Fatalf("mounting nested tmpfs: %v", err) } - if err := unix.Mount(src, dst, "", unix.MS_BIND, ""); err != nil { - t.Fatalf("Mount(%q, %q, BIND) = %v", src, dst, err) + if err := unix.Mount(src, dst, "", unix.MS_BIND|unix.MS_REC, ""); err != nil { + t.Fatalf("Mount(%q, %q, BIND|REC) = %v", src, dst, err) } defer Unmount(dst) + // as StageImageVolume gets from BindIntoShare + if err := setNosuidNodev(dst); err != nil { + t.Fatalf("setNosuidNodev: %v", err) + } - testFile := filepath.Join(dst, "test.txt") - if err := os.WriteFile(testFile, []byte("write test"), 0o644); err != nil { - t.Fatalf("WriteFile before remount = %v", err) + testFiles := []string{filepath.Join(dst, "test.txt"), filepath.Join(dst, "sub", "test.txt")} + for _, f := range testFiles { + if err := os.WriteFile(f, []byte("write test"), 0o644); err != nil { + t.Fatalf("WriteFile before remount = %v", err) + } } if err := RemountReadOnly(dst); err != nil { t.Fatalf("RemountReadOnly(%q) = %v", dst, err) } + if err := unix.Mount("tmpfs", later, "tmpfs", 0, ""); err != nil { + t.Fatalf("mounting tmpfs after remount: %v", err) + } - if err := os.WriteFile(testFile, []byte("should fail"), 0o644); !errors.Is(err, syscall.EROFS) { - t.Fatalf("WriteFile after remount = %v, want EROFS", err) + for _, f := range append(testFiles, filepath.Join(dst, "later", "test.txt")) { + if err := os.WriteFile(f, []byte("should fail"), 0o644); !errors.Is(err, syscall.EROFS) { + t.Errorf("WriteFile(%q) after remount = %v, want EROFS", f, err) + } + } + for _, path := range []string{dst, filepath.Join(dst, "sub")} { + var st unix.Statfs_t + if err := unix.Statfs(path, &st); err != nil { + t.Fatalf("statfs %q: %v", path, err) + } + if st.Flags&unix.ST_NOSUID == 0 || st.Flags&unix.ST_NODEV == 0 { + t.Errorf("%q: flags %#x, want nosuid and nodev kept", path, st.Flags) + } } Unmount(dst) diff --git a/internal/imagecache/bundle_linux.go b/internal/imagecache/bundle_linux.go index 118cf0c9e0..0cb364352d 100644 --- a/internal/imagecache/bundle_linux.go +++ b/internal/imagecache/bundle_linux.go @@ -133,9 +133,12 @@ func mountImageVolume(mountpoint string, layers []string) error { if err := unix.Mount(fsDir, mountpoint, "", unix.MS_BIND|unix.MS_REC, ""); err != nil { return fmt.Errorf("while binding %q: %w", fsDir, err) } - // MS_BIND and MS_RDONLY cannot be combined; a second call applies read-only. - if err := unix.Mount("", mountpoint, "", unix.MS_REMOUNT|unix.MS_BIND|unix.MS_RDONLY, ""); err != nil { - return fmt.Errorf("while remounting %q read-only: %w", mountpoint, err) + // A bind can't be created read-only, so set it after, on every mount the + // rbind brought along. Also private, otherwise a mount made under fsDir + // later would propagate in writable. + attr := &unix.MountAttr{Attr_set: unix.MOUNT_ATTR_RDONLY, Propagation: unix.MS_PRIVATE} + if err := unix.MountSetattr(unix.AT_FDCWD, mountpoint, unix.AT_RECURSIVE, attr); err != nil { + return fmt.Errorf("while making %q read-only: %w", mountpoint, err) } return nil } diff --git a/internal/imagecache/bundle_linux_test.go b/internal/imagecache/bundle_linux_test.go index 82901031c8..cae2b44354 100644 --- a/internal/imagecache/bundle_linux_test.go +++ b/internal/imagecache/bundle_linux_test.go @@ -323,6 +323,24 @@ func TestSetupBundleRootfs_ImageVolumes(t *testing.T) { }, nil) layers = append(layers, dir) } + // A shared layer with a submount now and one mounted after setup. Only + // the bind arm could carry either along. + fsDir := filepath.Join(layers[tc.layers-1], layerFSDirName) + for _, d := range []string{"sub", "later"} { + if err := os.Mkdir(filepath.Join(fsDir, d), 0o755); err != nil { + t.Fatal(err) + } + } + if err := unix.Mount(fsDir, fsDir, "", unix.MS_BIND, ""); err != nil { + t.Fatalf("self-binding %q: %v", fsDir, err) + } + t.Cleanup(func() { _ = unix.Unmount(fsDir, unix.MNT_DETACH) }) + if err := unix.Mount("", fsDir, "", unix.MS_SHARED, ""); err != nil { + t.Fatalf("making %q shared: %v", fsDir, err) + } + if err := unix.Mount("tmpfs", filepath.Join(fsDir, "sub"), "tmpfs", 0, ""); err != nil { + t.Fatalf("mounting nested tmpfs: %v", err) + } bundle := t.TempDir() if err := WriteSpec(bundle, &OverlaySpec{ @@ -335,6 +353,9 @@ func TestSetupBundleRootfs_ImageVolumes(t *testing.T) { t.Fatalf("SetupBundleRootfs: %v", err) } t.Cleanup(func() { _ = UnmountAllUnder(bundle) }) + if err := unix.Mount("tmpfs", filepath.Join(fsDir, "later"), "tmpfs", 0, ""); err != nil { + t.Fatalf("mounting tmpfs after setup: %v", err) + } mnt := filepath.Join(bundle, "volumes", "agent") for i := range tc.layers { @@ -348,10 +369,12 @@ func TestSetupBundleRootfs_ImageVolumes(t *testing.T) { t.Errorf("shadowed.txt = %q (%v), want %q", got, err, want) } // No upper on either arm, so there is nowhere for a write to go. - if err := os.WriteFile(filepath.Join(mnt, "nope.txt"), []byte("x"), 0o644); err == nil { - t.Error("write succeeded through a read-only image volume") - } else if !errors.Is(err, unix.EROFS) { - t.Errorf("write failed with %v, want EROFS", err) + for _, name := range []string{"nope.txt", "sub/nope.txt", "later/nope.txt"} { + if err := os.WriteFile(filepath.Join(mnt, name), []byte("x"), 0o644); err == nil { + t.Errorf("write of %s succeeded through a read-only image volume", name) + } else if !errors.Is(err, unix.EROFS) { + t.Errorf("write of %s failed with %v, want EROFS", name, err) + } } // The shared pool must never see the attempt. if _, err := os.Stat(filepath.Join(layers[tc.layers-1], layerFSDirName, "nope.txt")); err == nil { From 4b4fbb2c7ca191c77eb9145f47a3edefb2e99f47 Mon Sep 17 00:00:00 2001 From: igooch Date: Tue, 20 Oct 2026 17:23:30 +0000 Subject: [PATCH 06/63] imagecache: Verify layer content against its diffID before it enters the pool (#2279) Fixes #2082. ## The bug The layer pool is keyed by diffID and shared by every image on a node, but nothing checked a layer's content against its diffID: - The diffID comes from the image config, which the image's author controls. An image could claim a common base layer's diffID and plant its own content under it. - The old check compared a remote layer's `DiffID()` with the config's diffID, but go-containerregistry reads that `DiffID()` from the same config, so the check could never fail. - go-containerregistry checks the compressed digest only at EOF, and untar stops at the tar end marker, before EOF. ## The fix - Unpack hashes the uncompressed stream, drains it to EOF, and rejects a mismatch before the layer is renamed into the pool. - The layer flight is keyed by blob as well as diffID, so a lying image fails only its own pull, not honest pulls of the same diffID. This is safe because a pull's pins, not its flight, keep eviction off its layers, so the flight is only deduplication. - The cost is that concurrent pulls of the same diffID from different blobs (for example, the same layer recompressed) no longer share a flight. Each one downloads and unpacks the layer, and one rename wins. - A mismatch isn't retried per candidate credential. - The recorded size counts only what untar read. - The on-disk layout is unchanged, so the layout version stays at 1. Layers unpacked before this change were never verified and stay in the pool until eviction retires them. ## Not in this PR A node that already has a verified layer reuses it without downloading the new image's blob, so an image whose blob doesn't match its claimed diffID can still succeed there. Checking each image's own blob on reuse is the follow-up. --- internal/imagecache/README.md | 26 +- internal/imagecache/imagecache.go | 59 ++-- internal/imagecache/imagecache_test.go | 8 +- internal/imagecache/pull_gated_test.go | 44 ++- internal/imagecache/retire_test.go | 6 +- internal/imagecache/verify.go | 64 +++++ internal/imagecache/verify_test.go | 383 +++++++++++++++++++++++++ 7 files changed, 529 insertions(+), 61 deletions(-) create mode 100644 internal/imagecache/verify.go create mode 100644 internal/imagecache/verify_test.go diff --git a/internal/imagecache/README.md b/internal/imagecache/README.md index 7b9b5861e0..96f2838db2 100644 --- a/internal/imagecache/README.md +++ b/internal/imagecache/README.md @@ -91,10 +91,11 @@ layer diffIDs in order — layers shared by N images exist once. present, return with no network I/O. Missing layers (only) are re-pulled. 3. **Pull** by resolved digest: layers download in parallel (bounded at 4), each streamed download → decompress → untar directly into the pool. - Concurrent pulls of the same image or layer are collapsed with - singleflight, so simultaneous actor starts never duplicate work — and - each completed layer lands individually, so an interrupted pull makes - incremental progress across retries. + Concurrent pulls of the same image share one singleflight, and so do + concurrent unpacks of the same layer blob. A layer's flight is keyed by + its blob digest and its diffID, so one image's bad blob cannot fail + another image's pull of the same diffID. Each completed layer lands on + its own, so an interrupted pull makes progress across retries. 4. **Unpack** (`unpackLayer`) is the repo's hardened untar: `os.Root` confinement (path traversal and symlink/hardlink escapes are refused), "later entry wins" within a layer, read-only-dir handling that works @@ -105,7 +106,18 @@ layer diffIDs in order — layers shared by N images exist once. into the tree — overlayfs whiteouts are char devices atelet cannot create — they are recorded in `whiteouts.json` for the consumer to materialize. -5. **Record**: the image config + diffID list is written under the +5. **Verify** before the layer enters the pool. Images share layers by + diffID, but each image's author chooses the diffIDs in its config. So + the uncompressed stream is hashed as it unpacks and must match the + claimed diffID. The stream is also read to EOF, because + go-containerregistry checks the blob against the manifest digest only + at EOF. On a mismatch the `.tmp-*` tree is discarded and the pull fails. + + A layer already in the pool is reused without re-hashing, because it + was checked when it was written. The reusing image's own blob is never + downloaded, so it is never checked. Layers unpacked by an atelet that + lacked this check stay in the pool until eviction retires them. +6. **Record**: the image config + diffID list is written under the requested digest (and the per-platform child digest for multi-arch refs). `prepareOCIDirectory` in atelet then writes `rootfs-overlay.json` @@ -258,8 +270,8 @@ directory that is a live overlay lowerdir in another mount namespace succeeds silently, leaves the overlay's behavior undefined, and doesn't even free the space until the mount goes away. -Deleting the cache root by hand (while no actors are starting) remains -safe — the store re-pulls whatever is missing. +Deleting the cache root by hand (while no actors are running on the node) +remains safe — the store re-pulls whatever is missing. This is Phase 2 of [#463](https://github.com/agent-substrate/substrate/issues/463); the watermark loop, flags, and cache metrics complete it. Phase 3 adds diff --git a/internal/imagecache/imagecache.go b/internal/imagecache/imagecache.go index 9693cf1c8a..c93e19c398 100644 --- a/internal/imagecache/imagecache.go +++ b/internal/imagecache/imagecache.go @@ -51,6 +51,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "log/slog" "net" "os" @@ -628,14 +629,9 @@ func (s *Store) pullWith(ctx context.Context, parsedRef name.Reference, digest v g.SetLimit(layerPullConcurrency) for i, layer := range layers { g.Go(func() error { - diffID, err := layer.DiffID() - if err != nil { - return fmt.Errorf("while reading layer diffID: %w", err) - } - if diffID.String() != diffIDs[i] { - // The record must reference exactly what lands on disk. - return fmt.Errorf("layer %d diffID %s does not match config rootfs diffID %s", i, diffID, diffIDs[i]) - } + // This is the config's claim. Unpack hashes the content to check + // it, since a remote layer's DiffID() would just repeat the config. + diffID := cfgFile.RootFS.DiffIDs[i] dir, err := s.ensureLayer(gctx, diffID, layer) if err != nil { return fmt.Errorf("while unpacking layer %s: %w", diffID, err) @@ -716,14 +712,19 @@ func (s *Store) pinned(hex string) bool { } // ensureLayer makes the unpacked tree for diffID present in the pool, -// collapsing concurrent requests for the same layer across images. +// collapsing concurrent requests for the same layer blob across images. // // The caller must hold a pin on the layer (see pinLayers) from before this // call until it no longer relies on the returned dir. The pin, not the // flight, is what keeps eviction from retiring the layer meanwhile. func (s *Store) ensureLayer(ctx context.Context, diffID v1.Hash, layer v1.Layer) (string, error) { dir := s.layerDir(diffID) - _, err, _ := s.layerSF.Do(diffID.String(), func() (any, error) { + // From the manifest: no network I/O. + blobDigest, err := layer.Digest() + if err != nil { + return "", fmt.Errorf("while reading layer blob digest: %w", err) + } + _, err, _ = s.layerSF.Do(layerFlightKey(diffID, blobDigest), func() (any, error) { if _, err := os.Stat(filepath.Join(dir, layerFSDirName)); err == nil { // Refresh the dir mtime so eviction's LRU order sees the reuse. // The pull's pin, not this stamp, is what keeps the layer from @@ -742,6 +743,14 @@ func (s *Store) ensureLayer(ctx context.Context, diffID v1.Hash, layer v1.Layer) return dir, nil } +// layerFlightKey keys ensureLayer's flight by blob digest as well as +// diffID. A caller that joins a flight gets the leader's result, so an +// image that lies about a common layer's diffID leads its own flight and +// fails only its own pull. +func layerFlightKey(diffID, blobDigest v1.Hash) string { + return diffID.String() + "@" + blobDigest.String() +} + // unpackLayerToPool streams the layer (download → decompress → untar) into a // temp dir and renames it into place, so a layer dir either exists complete // or not at all. @@ -764,43 +773,42 @@ func (s *Store) unpackLayerToPool(ctx context.Context, diffID v1.Hash, layer v1. return fmt.Errorf("while creating layer fs dir: %w", err) } - rc, err := layer.Uncompressed() - if err != nil { - return fmt.Errorf("while opening layer stream: %w", err) - } - defer rc.Close() - root, err := os.OpenRoot(fsDir) if err != nil { return fmt.Errorf("while opening layer fs dir as os.Root: %w", err) } defer root.Close() - // The uncompressed tar stream is the recorded size: an optimistic + var layerMetadata *whiteoutSet + // The tar bytes untar read are the recorded size: an optimistic // estimate (tar framing vs. block rounding), cheap to capture here. - cr := &countingReader{r: rc} - wh, err := unpackLayer(ctx, cr, root) + size, err := readVerifiedLayer(layer, diffID, func(r io.Reader) (err error) { + layerMetadata, err = unpackLayer(ctx, r, root) + return err + }) if err != nil { + // Returning before the rename discards the unpacked tree, so + // unverified content never enters the pool. return err } // Non-fatal: a missing size file is recovered by layerSize's backfill, // so a metadata write must not discard a successful unpack. - if err := os.WriteFile(filepath.Join(tmp, layerSizeFileName), []byte(strconv.FormatInt(cr.n, 10)+"\n"), 0o600); err != nil { + if err := os.WriteFile(filepath.Join(tmp, layerSizeFileName), []byte(strconv.FormatInt(size, 10)+"\n"), 0o600); err != nil { slog.WarnContext(ctx, "Failed to record layer size; will backfill lazily", slog.String("diffid", diffID.String()), slog.Any("err", err)) } - whBytes, err := json.Marshal(wh) + layerMetadataJSON, err := json.Marshal(layerMetadata) if err != nil { return fmt.Errorf("while encoding layer whiteouts: %w", err) } - if err := os.WriteFile(filepath.Join(tmp, layerWhiteoutsFileName), whBytes, 0o600); err != nil { + if err := os.WriteFile(filepath.Join(tmp, layerWhiteoutsFileName), layerMetadataJSON, 0o600); err != nil { return fmt.Errorf("while writing layer whiteouts: %w", err) } if err := os.Rename(tmp, s.layerDir(diffID)); err != nil { - // A concurrent unpack (another process sharing the pool) may have won; - // its layer is as good as ours. + // A concurrent unpack (another flight for this diffID, or another + // process sharing the pool) may have won; its layer is as good as ours. if _, statErr := os.Stat(filepath.Join(s.layerDir(diffID), layerFSDirName)); statErr == nil { return nil } @@ -873,7 +881,8 @@ func (s *Store) withCredentials(ctx context.Context, parsedRef name.Reference, f return nil } errs = append(errs, err) - if ctx.Err() != nil { + // Another credential cannot change the image's content. + if ctx.Err() != nil || errors.Is(err, errDiffIDMismatch) { break } } diff --git a/internal/imagecache/imagecache_test.go b/internal/imagecache/imagecache_test.go index d0a3a8f9f1..58bd1f402f 100644 --- a/internal/imagecache/imagecache_test.go +++ b/internal/imagecache/imagecache_test.go @@ -44,7 +44,13 @@ import ( // over plain HTTP). func newTestRegistry(t *testing.T) (*httptest.Server, string) { t.Helper() - srv := httptest.NewServer(registry.New(registry.Logger(log.New(io.Discard, "", 0)))) + return newWrappedRegistry(t, func(h http.Handler) http.Handler { return h }) +} + +// newWrappedRegistry starts an in-memory OCI registry behind wrap. +func newWrappedRegistry(t *testing.T, wrap func(http.Handler) http.Handler) (*httptest.Server, string) { + t.Helper() + srv := httptest.NewServer(wrap(registry.New(registry.Logger(log.New(io.Discard, "", 0))))) t.Cleanup(srv.Close) u, err := url.Parse(srv.URL) if err != nil { diff --git a/internal/imagecache/pull_gated_test.go b/internal/imagecache/pull_gated_test.go index 639b4fe03e..ddc866cd4b 100644 --- a/internal/imagecache/pull_gated_test.go +++ b/internal/imagecache/pull_gated_test.go @@ -23,11 +23,7 @@ import ( "context" "encoding/json" "errors" - "io" - "log" "net/http" - "net/http/httptest" - "net/url" "os" "path/filepath" "strings" @@ -35,7 +31,6 @@ import ( "testing" "time" - "github.com/google/go-containerregistry/pkg/registry" v1 "github.com/google/go-containerregistry/pkg/v1" ) @@ -51,30 +46,25 @@ type gatedRegistry struct { func newGatedRegistry(t *testing.T) *gatedRegistry { t.Helper() g := &gatedRegistry{gates: map[string]chan struct{}{}} - inner := registry.New(registry.Logger(log.New(io.Discard, "", 0))) - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/blobs/") { - g.mu.Lock() - var ch chan struct{} - for hex, c := range g.gates { - if strings.Contains(r.URL.Path, hex) { - ch = c - break + _, g.host = newWrappedRegistry(t, func(inner http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/blobs/") { + g.mu.Lock() + var ch chan struct{} + for hex, c := range g.gates { + if strings.Contains(r.URL.Path, hex) { + ch = c + break + } + } + g.mu.Unlock() + if ch != nil { + <-ch } } - g.mu.Unlock() - if ch != nil { - <-ch - } - } - inner.ServeHTTP(w, r) - })) - t.Cleanup(srv.Close) - u, err := url.Parse(srv.URL) - if err != nil { - t.Fatalf("parsing registry URL: %v", err) - } - g.host = u.Host + inner.ServeHTTP(w, r) + }) + }) return g } diff --git a/internal/imagecache/retire_test.go b/internal/imagecache/retire_test.go index ef62fa4904..bafe64e985 100644 --- a/internal/imagecache/retire_test.go +++ b/internal/imagecache/retire_test.go @@ -323,6 +323,10 @@ func TestPullPinsBeforeJoiningLayerFlight(t *testing.T) { if err != nil { t.Fatal(err) } + blobDigest, err := layer.Digest() + if err != nil { + t.Fatal(err) + } // Lead the layer's flight ourselves, and do its real work only once // released, so the pull below has to join and wait. @@ -330,7 +334,7 @@ func TestPullPinsBeforeJoiningLayerFlight(t *testing.T) { var once sync.Once t.Cleanup(func() { once.Do(func() { close(release) }) }) go func() { - _, _, _ = store.layerSF.Do(diffID.String(), func() (any, error) { + _, _, _ = store.layerSF.Do(layerFlightKey(diffID, blobDigest), func() (any, error) { close(held) <-release return nil, store.unpackLayerToPool(context.Background(), diffID, layer) diff --git a/internal/imagecache/verify.go b/internal/imagecache/verify.go new file mode 100644 index 0000000000..3d779de802 --- /dev/null +++ b/internal/imagecache/verify.go @@ -0,0 +1,64 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecache + +import ( + "encoding/hex" + "errors" + "fmt" + "io" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// errDiffIDMismatch marks layer content that does not hash to its claimed +// diffID. It is a property of the image, so retrying cannot fix it. +var errDiffIDMismatch = errors.New("layer content does not match its claimed diffID") + +// readVerifiedLayer passes the layer's uncompressed stream to consume, then +// reads whatever consume left unread and checks that the whole stream +// hashes to diffID. It returns only the number of bytes consume read. The +// rest never lands on disk, so counting it would let an image inflate the +// size GC accounts for. +// +// The pool shares layers across images by diffID, but each image's author +// chooses the diffIDs in its config. Checking the hash stops one image from +// planting content under another image's layer. Reading to EOF also lets +// go-containerregistry check the compressed blob against the manifest +// digest, which it does only at EOF. +func readVerifiedLayer(layer v1.Layer, diffID v1.Hash, consume func(io.Reader) error) (int64, error) { + rc, err := layer.Uncompressed() + if err != nil { + return 0, fmt.Errorf("while opening layer stream: %w", err) + } + defer rc.Close() + + hasher, err := v1.Hasher(diffID.Algorithm) + if err != nil { + return 0, fmt.Errorf("while creating hasher for %s: %w", diffID, err) + } + tee := io.TeeReader(rc, hasher) + counter := &countingReader{r: tee} + if err := consume(counter); err != nil { + return 0, err + } + if _, err := io.Copy(io.Discard, tee); err != nil { + return 0, fmt.Errorf("while draining layer stream: %w", err) + } + if got := (v1.Hash{Algorithm: diffID.Algorithm, Hex: hex.EncodeToString(hasher.Sum(nil))}); got != diffID { + return 0, fmt.Errorf("%w: content hashes to %s, config claims %s", errDiffIDMismatch, got, diffID) + } + return counter.n, nil +} diff --git a/internal/imagecache/verify_test.go b/internal/imagecache/verify_test.go new file mode 100644 index 0000000000..0aa59f5e55 --- /dev/null +++ b/internal/imagecache/verify_test.go @@ -0,0 +1,383 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecache + +import ( + "archive/tar" + "bytes" + "context" + "errors" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "testing/synctest" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +// bufferedResponse buffers a response so a test registry can alter it. +type bufferedResponse struct { + header http.Header + code int + buf bytes.Buffer +} + +func (r *bufferedResponse) Header() http.Header { return r.header } +func (r *bufferedResponse) WriteHeader(code int) { r.code = code } +func (r *bufferedResponse) Write(p []byte) (int, error) { return r.buf.Write(p) } + +// newTamperingRegistry serves blobs as pushed, except that GETs of the blob +// whose hex is in *tamper get byte 4 flipped. That is the gzip header MTIME, +// which no gzip check covers, so the blob still decompresses to the same +// content and only the compressed digest check can notice. +func newTamperingRegistry(t *testing.T, tamper *string) string { + t.Helper() + _, host := newWrappedRegistry(t, func(inner http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || *tamper == "" || !strings.HasSuffix(r.URL.Path, "/blobs/sha256:"+*tamper) { + inner.ServeHTTP(w, r) + return + } + resp := &bufferedResponse{header: http.Header{}, code: http.StatusOK} + inner.ServeHTTP(resp, r) + body := resp.buf.Bytes() + body[4] ^= 0xff + for k, v := range resp.header { + w.Header()[k] = v + } + w.WriteHeader(resp.code) + w.Write(body) + }) + }) + return host +} + +// pushLyingImage pushes an image whose only layer is content but whose +// config claims claimed's diffID. +func pushLyingImage(t *testing.T, ref string, content, claimed v1.Layer) { + t.Helper() + img, err := mutate.AppendLayers(empty.Image, content) + if err != nil { + t.Fatal(err) + } + cfgFile, err := img.ConfigFile() + if err != nil { + t.Fatal(err) + } + cfgFile = cfgFile.DeepCopy() + claimedDiffID, err := claimed.DiffID() + if err != nil { + t.Fatal(err) + } + cfgFile.RootFS.DiffIDs = []v1.Hash{claimedDiffID} + if img, err = mutate.ConfigFile(img, cfgFile); err != nil { + t.Fatal(err) + } + tag, err := name.ParseReference(ref, name.Insecure) + if err != nil { + t.Fatal(err) + } + if err := remote.Write(tag, img); err != nil { + t.Fatal(err) + } +} + +func libLayers(t *testing.T) (good, evil v1.Layer) { + t.Helper() + good = layerFromEntries(t, []tarEntry{{name: "lib", typeflag: tar.TypeReg, mode: 0o644, body: "good"}}) + evil = layerFromEntries(t, []tarEntry{{name: "lib", typeflag: tar.TypeReg, mode: 0o644, body: "EVIL"}}) + return good, evil +} + +func readLib(t *testing.T, layerDir string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join(layerDir, layerFSDirName, "lib")) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// assertPoolEmpty fails on any entry in the layer pool. Unlike +// layerDirsOnDisk it counts .tmp-* dirs too: a rejected unpack must leave +// nothing behind. +func assertPoolEmpty(t *testing.T, s *Store) { + t.Helper() + entries, err := os.ReadDir(s.layersDir()) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + t.Errorf("unexpected entry in layer pool: %s", e.Name()) + } +} + +// A blob whose bytes differ from the manifest digest is rejected, even +// though it decompresses to the claimed diffID. +func TestPullRejectsBlobNotMatchingManifestDigest(t *testing.T) { + var tamper string + host := newTamperingRegistry(t, &tamper) + good, _ := libLayers(t) + ref := host + "/test/tampered:latest" + pushImage(t, ref, v1.Config{}, good) + blobDigest, err := good.Digest() + if err != nil { + t.Fatal(err) + } + tamper = blobDigest.Hex + + s := newTestStore(t) + _, err = s.EnsureImage(context.Background(), ref) + if err == nil || !strings.Contains(err.Error(), "error verifying sha256 checksum") { + t.Fatalf("EnsureImage error = %v, want a compressed digest mismatch", err) + } + assertPoolEmpty(t, s) +} + +// An image claiming another layer's diffID cannot plant its content under +// that diffID for a later image to reuse. +func TestPullRejectsContentNotMatchingClaimedDiffID(t *testing.T) { + _, host := newTestRegistry(t) + good, evil := libLayers(t) + evilRef, goodRef := host+"/test/evil:latest", host+"/test/good:latest" + pushLyingImage(t, evilRef, evil, good) + pushImage(t, goodRef, v1.Config{}, good) + + s := newTestStore(t) + if _, err := s.EnsureImage(context.Background(), evilRef); !errors.Is(err, errDiffIDMismatch) { + t.Fatalf("lying pull error = %v, want errDiffIDMismatch", err) + } + assertPoolEmpty(t, s) + + img, err := s.EnsureImage(context.Background(), goodRef) + if err != nil { + t.Fatal(err) + } + if got := readLib(t, img.LayerDirs[0]); got != "good" { + t.Errorf("good image's lib = %q, want %q", got, "good") + } +} + +// A content mismatch is the image's, so it is not re-downloaded once per +// candidate credential. +func TestDiffIDMismatchNotRetriedPerCredential(t *testing.T) { + good, evil := libLayers(t) + evilBlobDigest, err := evil.Digest() + if err != nil { + t.Fatal(err) + } + var gets atomic.Int32 + _, host := newWrappedRegistry(t, func(inner http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/blobs/"+evilBlobDigest.String()) { + gets.Add(1) + } + inner.ServeHTTP(w, r) + }) + }) + ref := host + "/test/evil:latest" + pushLyingImage(t, ref, evil, good) + + kc := &candidateKeychain{auths: []authn.Authenticator{ + authn.FromConfig(authn.AuthConfig{Username: "a", Password: "a"}), + authn.FromConfig(authn.AuthConfig{Username: "b", Password: "b"}), + }} + s := newTestStore(t, WithKeychain(kc)) + if _, err := s.EnsureImage(context.Background(), ref); !errors.Is(err, errDiffIDMismatch) { + t.Fatalf("EnsureImage error = %v, want errDiffIDMismatch", err) + } + if n := gets.Load(); n != 1 { + t.Errorf("lying blob downloaded %d times, want 1", n) + } +} + +// gatedLayer holds a layer flight open: its uncompressed stream closes +// started, blocks until release is closed, then fails with err if set. +type gatedLayer struct { + v1.Layer + started, release chan struct{} + err error +} + +func newGatedLayer(layer v1.Layer, err error) gatedLayer { + return gatedLayer{Layer: layer, started: make(chan struct{}), release: make(chan struct{}), err: err} +} + +func (g gatedLayer) Uncompressed() (io.ReadCloser, error) { + close(g.started) + <-g.release + if g.err != nil { + return nil, g.err + } + return g.Layer.Uncompressed() +} + +// openCountingLayer counts opens of its uncompressed stream. +type openCountingLayer struct { + v1.Layer + opens *atomic.Int32 +} + +func (c openCountingLayer) Uncompressed() (io.ReadCloser, error) { + c.opens.Add(1) + return c.Layer.Uncompressed() +} + +// joinFlight runs ensureLayer for leader, then for joiner once the leader's +// flight is open, and releases the leader only after the joiner has joined +// its flight. Call inside a synctest bubble. +func joinFlight(t *testing.T, s *Store, diffID v1.Hash, leader gatedLayer, joiner v1.Layer) (leaderErr error, joinerDir string, joinerErr error) { + t.Helper() + leaderDone := make(chan error, 1) + go func() { + _, err := s.ensureLayer(t.Context(), diffID, leader) + leaderDone <- err + }() + <-leader.started + + joinerDone := make(chan struct{}) + go func() { + joinerDir, joinerErr = s.ensureLayer(t.Context(), diffID, joiner) + close(joinerDone) + }() + // Every goroutine is now blocked: the leader on release, the joiner + // waiting on the leader's flight. + synctest.Wait() + close(leader.release) + leaderErr = <-leaderDone + <-joinerDone + return leaderErr, joinerDir, joinerErr +} + +// A lying pull holds its own flight, so an honest pull of the same diffID +// completes without waiting on it or inheriting its error. +func TestLyingPullDoesNotFailConcurrentHonestPull(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + s := newTestStore(t) + good, evil := libLayers(t) + diffID, err := good.DiffID() + if err != nil { + t.Fatal(err) + } + lying := newGatedLayer(evil, nil) + lyingDone := make(chan error, 1) + go func() { + _, err := s.ensureLayer(t.Context(), diffID, lying) + lyingDone <- err + }() + <-lying.started + + // The lying flight is still held open. + dir, err := s.ensureLayer(t.Context(), diffID, good) + if err != nil { + t.Fatalf("honest ensureLayer: %v", err) + } + if got := readLib(t, dir); got != "good" { + t.Errorf("honest layer's lib = %q, want %q", got, "good") + } + + close(lying.release) + if err := <-lyingDone; !errors.Is(err, errDiffIDMismatch) { + t.Fatalf("lying ensureLayer error = %v, want errDiffIDMismatch", err) + } + if got := readLib(t, dir); got != "good" { + t.Errorf("after the lying pull, lib = %q, want %q", got, "good") + } + }) +} + +// Pulls of the same blob still share one flight, and its result. +func TestSameBlobPullsShareOneFlight(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + s := newTestStore(t) + good, _ := libLayers(t) + diffID, err := good.DiffID() + if err != nil { + t.Fatal(err) + } + outage := errors.New("registry unavailable") + var opens atomic.Int32 + _, _, err = joinFlight(t, s, diffID, newGatedLayer(good, outage), openCountingLayer{Layer: good, opens: &opens}) + if !errors.Is(err, outage) { + t.Errorf("joiner error = %v, want the leader's %v", err, outage) + } + if n := opens.Load(); n != 0 { + t.Errorf("joiner opened its own layer %d times, want 0", n) + } + }) +} + +// The hash covers the whole uncompressed stream, while the returned length +// covers only what the consumer read. +func TestReadVerifiedLayerDrainsStream(t *testing.T) { + good, evil := libLayers(t) + diffID, err := good.DiffID() + if err != nil { + t.Fatal(err) + } + readTen := func(r io.Reader) error { + _, err := io.CopyN(io.Discard, r, 10) + return err + } + n, err := readVerifiedLayer(good, diffID, readTen) + if err != nil { + t.Fatalf("readVerifiedLayer(good): %v", err) + } + if n != 10 { + t.Errorf("length = %d, want 10", n) + } + if _, err := readVerifiedLayer(evil, diffID, readTen); !errors.Is(err, errDiffIDMismatch) { + t.Errorf("readVerifiedLayer(evil) error = %v, want errDiffIDMismatch", err) + } +} + +// Bytes after the tar end marker are hashed but never land on disk, so they +// must not count toward the size GC accounts for. +func TestRecordedSizeExcludesTrailingBytes(t *testing.T) { + tarBytes := buildTar(t, []tarEntry{{name: "lib", typeflag: tar.TypeReg, mode: 0o644, body: "good"}}) + padded := append(bytes.Clone(tarBytes), make([]byte, 1<<20)...) + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(padded)), nil + }) + if err != nil { + t.Fatal(err) + } + diffID, err := layer.DiffID() + if err != nil { + t.Fatal(err) + } + s := newTestStore(t) + dir, err := s.ensureLayer(context.Background(), diffID, layer) + if err != nil { + t.Fatalf("ensureLayer: %v", err) + } + size, ok, err := recordedLayerSize(dir) + if err != nil || !ok { + t.Fatalf("recordedLayerSize = %d, %v, %v", size, ok, err) + } + if size <= 0 || size > int64(len(tarBytes)) { + t.Errorf("recorded size = %d, want in (0, %d]", size, len(tarBytes)) + } +} From a09f87ce0cab884dbf356fdf9b3609909f312a59 Mon Sep 17 00:00:00 2001 From: Taahir Ahmed Date: Tue, 20 Oct 2026 17:30:35 +0000 Subject: [PATCH 07/63] Remove all logic from ./hack/install-ate.sh (#2718) Right now it still has a bunch of logic that needs to be extended each time new functionality is added to cmd/ate-setup. Convert it to a simple wrapper that forwards args to cmd/ate-setup, and change direct references to it to instead be proper references to cmd/ate-setup --- .github/workflows/pr-workflow.yaml | 37 +- README.md | 15 +- benchmarking/README.md | 4 +- benchmarking/telemetry/README.md | 5 +- cmd/ate-setup/cli-diff.md | 171 +------- cmd/ate-setup/preflight_test.go | 60 ++- cmd/ate-setup/shim_test.go | 605 -------------------------- demos/claude-code-multiplex/README.md | 6 +- demos/counter/README.md | 8 +- demos/egress/README.md | 4 +- demos/multi-template/README.md | 6 +- demos/parking/README.md | 6 +- demos/sandbox/README.md | 4 +- docs/egress-trust-bundle.md | 2 +- hack/install-ate-kind.sh | 2 +- hack/install-ate.sh | 446 +------------------ hack/run-microvm-demo.sh | 8 +- tools/setup-gcp/cloud-sql.md | 4 +- 18 files changed, 98 insertions(+), 1295 deletions(-) delete mode 100644 cmd/ate-setup/shim_test.go diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 8822fe7eef..052794438a 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -275,14 +275,27 @@ jobs: # gateway. ATE_PREVIEW above selects the preview features. env: ATE_ATENET_DATAPLANE: ${{ matrix.dataplane }} - run: hack/install-ate-kind.sh --deploy-ate-system + NO_DEV_ENV: "true" + KO_DOCKER_REPO: "localhost:5001" + ATE_INSTALL_KIND: "true" + BUCKET_NAME: "ate-snapshot" + KIND_CLUSTER_NAME: "kind" + KUBECTL_CONTEXT: "kind-kind" + run: go run ./cmd/ate-setup deploy ate-system - name: Enable NFS # Load NFS kernel modules so in-cluster NFS server and CSI driver can run. run: | sudo modprobe nfs || true sudo modprobe nfsd || true - name: Install CSI NFS driver - run: hack/install-ate-kind.sh --setup-csi=nfs + env: + NO_DEV_ENV: "true" + KO_DOCKER_REPO: "localhost:5001" + ATE_INSTALL_KIND: "true" + BUCKET_NAME: "ate-snapshot" + KIND_CLUSTER_NAME: "kind" + KUBECTL_CONTEXT: "kind-kind" + run: go run ./cmd/ate-setup setup csi nfs - name: Deploy micro-VM counter demo # Stages the (cached) assets into the cluster's rustfs and deploys the # counter-microvm demo onto the control plane installed above. The demo @@ -291,15 +304,29 @@ jobs: ATE_ATENET_DATAPLANE: ${{ matrix.dataplane }} run: hack/run-microvm-demo-kind.sh - name: Deploy gVisor counter demo - run: hack/install-ate-kind.sh --deploy-demo-counter + env: + NO_DEV_ENV: "true" + KO_DOCKER_REPO: "localhost:5001" + ATE_INSTALL_KIND: "true" + BUCKET_NAME: "ate-snapshot" + KIND_CLUSTER_NAME: "kind" + KUBECTL_CONTEXT: "kind-kind" + run: go run ./cmd/ate-setup deploy demo counter - name: Deploy egress demos # TestActorEgress in the networking suite builds its Actor from the egress # ActorTemplate for the class under test, so both fixtures have to exist # before their respective lane runs. The demo's actors trust the CA the # gateway mints its leaves from. + env: + NO_DEV_ENV: "true" + KO_DOCKER_REPO: "localhost:5001" + ATE_INSTALL_KIND: "true" + BUCKET_NAME: "ate-snapshot" + KIND_CLUSTER_NAME: "kind" + KUBECTL_CONTEXT: "kind-kind" run: | - hack/install-ate-kind.sh --deploy-demo-egress - hack/install-ate-kind.sh --deploy-demo-egress-microvm + go run ./cmd/ate-setup deploy demo egress + go run ./cmd/ate-setup deploy demo egress-microvm - name: Trust local TLS origins in egress gateway run: hack/setup-e2e-egress-tls-kind.sh # scale the idle sandbox class's demo pools to zero so each lane has the node diff --git a/README.md b/README.md index 53823c2104..fb7c783492 100644 --- a/README.md +++ b/README.md @@ -119,10 +119,10 @@ To quickly set up the complete environment: hack/create-kind-cluster.sh # install ate, PostgreSQL, rustfs -hack/install-ate-kind.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' +hack/install-ate-kind.sh deploy ate-system --credential-provider='{"name":"k8s.io"}' # install counter demo -hack/install-ate-kind.sh --deploy-demo-counter +hack/install-ate-kind.sh deploy demo-counter # install kubectl-ate go install ./cmd/kubectl-ate @@ -177,7 +177,7 @@ curl -X POST \ 4. Deploy the Agent Substrate system to your cluster: ```bash - ./hack/install-ate.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' + go run ./cmd/ate-setup deploy ate-system --credential-provider='{"name":"k8s.io"}' ``` `--credential-provider` is required. `{"name":"k8s.io"}` also deploys the @@ -193,7 +193,7 @@ curl -X POST \ 5. You can then deploy the sample applications. See [demos/counter/README.md](demos/counter/README.md) or [demos/sandbox/README.md](demos/sandbox/README.md) for detailed walkthroughs. ```bash - ./hack/install-ate.sh --deploy-demo-counter + go run ./cmd/ate-setup deploy demo demo-counter ``` #### Custom Setup and Deployment @@ -207,13 +207,14 @@ go run ./tools/setup-gcp create bucket To run the PostgreSQL store backend on Cloud SQL — with IAM database authentication and no passwords — see [tools/setup-gcp/cloud-sql.md](tools/setup-gcp/cloud-sql.md). -Similarly, you can deploy or cleanup specific Agent Substrate components using the installation script. See `./hack/install-ate.sh --help` for all options. +Similarly, you can deploy or cleanup specific Agent Substrate components using +the installation script. See `go run ./cmd/ate-setup` for all options. ```bash # Re-deploy only ate-apiserver of the ATE system -./hack/install-ate.sh --deploy-ate-apiserver +go run ./cmd/ate-setup deploy apiserver # Delete everything (core system and all demos) -./hack/install-ate.sh --delete-all +go run ./cmd/ate-setup delete all ``` #### Tearing down resources (GCP) diff --git a/benchmarking/README.md b/benchmarking/README.md index 58a6679a76..92238a7141 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -62,8 +62,8 @@ The same operations are also reachable from the top-level installer for convenience: ```bash -./hack/install-ate.sh --deploy-benchmarks -./hack/install-ate.sh --delete-benchmarks +go run ./cmd/ate-setup deploy benchmarks +go run ./cmd/ate-setup delete benchmarks ``` `--skip-build` and `--worker-memory` are only available when invoking diff --git a/benchmarking/telemetry/README.md b/benchmarking/telemetry/README.md index c5aa2c431b..9be2106b93 100644 --- a/benchmarking/telemetry/README.md +++ b/benchmarking/telemetry/README.md @@ -37,7 +37,8 @@ Install the meter, then send the control plane to it: kubectl apply -f benchmarking/telemetry/meter.yaml METER=http://telemetry-meter.benchmarking.svc.cluster.local:4317 -./hack/install-ate.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' --otlp-endpoint "${METER}" + +go run ./cmd/ate-setup deploy ate-system --credential-provider='{"name":"k8s.io"}' --otlp-endpoint "${METER}" ``` `--otlp-endpoint` patches the `ate-otel-config` ConfigMap and restarts the @@ -223,7 +224,7 @@ upgrade. ## Remove the meter ```bash -./hack/install-ate.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' +go run ./cmd/ate-setup deploy ate-system --credential-provider='{"name":"k8s.io"}' ./benchmarking/workloads/deploy.sh --deploy kubectl delete -f benchmarking/telemetry/meter.yaml ``` diff --git a/cmd/ate-setup/cli-diff.md b/cmd/ate-setup/cli-diff.md index da775db851..5f538cd2a9 100644 --- a/cmd/ate-setup/cli-diff.md +++ b/cmd/ate-setup/cli-diff.md @@ -1,171 +1,4 @@ # `hack/install-ate.sh` after the move to `ate-setup` -`hack/install-ate.sh` no longer installs anything. It is a translation shim: it -parses the flags and environment variables the installer has always accepted -and runs `go run ./cmd/ate-setup ` for each action, in command-line -order. [`commands.md`](commands.md) has the flag-by-flag mapping; -[`differences.md`](differences.md) explains how the Go implementation differs -from the shell one it replaced. - -This document is for the person whose command line or CI job already worked. -Everything not listed here behaves as it did. What is listed here either could -not be reproduced through the shim, or was reproduced closely enough to keep -scripts working but not closely enough to keep a sharp-eyed reader from -noticing. - -## What the shim preserves - -Deliberately, and covered by `shim_test.go`: - -- Every flag, with the same spelling, the same `=value` and separated-value - forms, and the same "value flags may appear anywhere on the line" pre-scan. -- Several actions per invocation, executed in the order they were written. -- Every environment variable, including `SETUP_CSI`, `STORAGE_CLASS`, - `NO_DEV_ENV`, `ATE_INSTALL_KIND`, `KUBECTL_CONTEXT`, `PROJECT_ID`, and the - `ATE_API_POSTGRES_*` set. `ate-setup` reads them directly, so they need no - translation. -- `--help` / `-h` anywhere on the line, no arguments at all (usage, exit 1), - and an unrecognized flag (`Error: unknown option: …`, usage, exit 1) — - including after earlier actions on the same line have already run. -- `--deploy-ate-system --setup-csi=nfs` still sets the CSI driver up twice, - once inside the system deploy and once as its own action, as it always has. - -## Differences you can observe - -### One process per action - -Each action is a separate `go run ./cmd/ate-setup`. The build is cached after -the first, but a line asking for six actions pays six process starts, and each -one re-resolves the configuration — including, on GKE with `PROJECT_ID` set and -no `KUBECTL_CONTEXT`, a `gcloud container clusters get-credentials` per action -rather than one for the whole line. - -`go` is therefore required to run the installer. In exchange, `kubectl`, `jq`, -`openssl`, `sed`, and `base64` are not; see the external-binaries table -in [`differences.md`](differences.md#external-binaries) for what is still -needed. - -### Value flags are validated only when an action uses them - -The shell pre-scan validated `--atenet-dataplane`, `--podcert-workers-per-signer`, -`--rollout-timeout`, and `--benchmark-sandbox-class` on every run, whether or -not anything used them. Validation now happens inside `ate-setup`, so: - -- A line with no action at all — `./hack/install-ate.sh --rollout-timeout=zzz` - — exits 0 without complaining. It used to exit 1. -- `--benchmark-sandbox-class=zzz` is rejected only by `--deploy-benchmarks` / - `--delete-benchmarks`. Paired with any other action it is ignored, where it - used to fail the run up front. - -The four flags that apply to every action (`--atenet-dataplane`, -`--podcert-workers-per-signer`, `--rollout-timeout`, `--otlp-endpoint`) are -passed to every `ate-setup` invocation, so as long as the line has one action -they are still rejected before that action touches the cluster. - -### `--deploy-demo-autoscaled-workerpool` fails later, off Kind - -The demo registered itself only under `ATE_INSTALL_KIND=true`, so on GKE the -flag was rejected as an unknown option, before anything ran, and `--help` did -not list it. The cobra command tree is built before flags are parsed, so the -subcommand now always exists: `--help` always lists the demo, and off Kind it -fails when it runs, with `demo-autoscaled-workerpool is only supported for Kind -installations; re-run with --kind`. Earlier actions on the same line will -already have completed. `--delete-all` still skips it off Kind. - -### The demo list lives in two places - -`ATE_DEMOS` in the shim is a literal list, so that `--help` answers without a -Go build and an unknown demo is rejected here rather than reaching `ate-setup` -as an unknown subcommand. A demo added to `internal/demos/all` and not to that -list is invisible to the shim; `TestShimDemoListMatchesTheRegistry` fails when -the two drift. - -### Two step log lines changed - -`log.Step` names are otherwise unchanged, but two lost a parenthesized -qualifier that CI log scrapers may be matching on: - -| before | now | -|---|---| -| `demo-counter_deploy (with_external_volume=true)` | `demo-counter_deploy` | -| `setup_csi (nfs)` | `setup_csi` | - -### `--setup-csi` on its own does more - -The standalone action used to ensure the CRDs and then install the driver. -`ate-setup setup csi` also ensures the `ate-system` namespace, the -podcertificate CAs, and a ready podcertificate-controller with its trust -bundles first. The hostpath driver's ghostunnel sidecar projects a -podCertificate volume and cannot roll out without them, which is why -`deploy_ate_system` had always called CSI setup from that point in its -sequence; the standalone path just did not. - -### `.ate-dev-env.sh` is layered, not sourced over you - -The script sourced the file into its own shell, so an `export FOO=...` in the -file won over a `FOO` already exported in yours. `ate-setup` still runs the -file through bash — it can contain arbitrary shell — but treats the result as -the lowest-precedence layer: process environment first, then flags. If you -have been relying on the file to override your shell, unset the variable in -your shell instead. - -Sourcing is also skipped for Kind installs. `hack/install-ate-kind.sh` exports -`NO_DEV_ENV`, so that path is unchanged, but running the shim yourself with -`ATE_INSTALL_KIND=true` no longer reads the file. - -### `--rollout-timeout` reaches further - -It used to govern only the 60s workload rollout waits. It now also governs the -two waits fixed at 120s (podcertificate-controller and the CSI drivers) — -but only when it is passed, so the 60s default still cannot shorten those slow -bootstrap paths. - -It also has to be positive. The value reached `kubectl rollout status ---timeout=`, where `0` means "wait forever"; `ate-setup` has no unbounded wait, -so `--rollout-timeout=0` is rejected at startup rather than read as its -opposite. - -### Applies are server-side - -`kubectl apply -f -` was a client-side apply. `ate-setup` uses server-side -apply with field manager `ate-setup` and `force: true`. Objects previously -installed by the shell script show both managers in `managedFields` until the -next apply reconciles them. `last-applied-configuration` stops growing on the -generated CRDs. - -### The first install after this change rolls `ate-api-server` once - -Both installers stamp `ate.dev/env-hash` on the pod template so a changed -environment starts a rollout, but they compute the digest differently. The -value is opaque, so the only consequence is one extra rollout the first time -`ate-setup` installs over a shell-installed cluster. - -### `--create-*-ca-pool-secret` is idempotent - -Against a cluster that already has the pool, these used to fail with -`AlreadyExists`. They now log `already exists; keeping it` and succeed. They -still refuse to overwrite: regenerating would rotate the root out from under -every certificate already issued from it. - -### `--setup-csi=hostpath` and `=both` off Kind are hard errors - -They used to warn and continue. Only the hostpath plugin is patched for the -single-node Kind layout; `nfs` is accepted anywhere, as before. - -### The per-demo scripts are gone - -`hack/install-demo-*.sh` and `hack/experimental-additional-egress-extproc.sh` -have been deleted — each demo is a Go package under -`cmd/ate-setup/internal/demos/` now. Nothing that goes through -`hack/install-ate.sh` notices, but anything sourcing those files directly, or -calling a `demo-…_deploy` shell function, has to move to -`go run ./cmd/ate-setup deploy demo `. - -## Not replicated, and not worth replicating - -Two behaviors of the old script were bugs that the shim does not reproduce: - -- `--delete-atenet` worked but was missing from `--help`. It is listed now. -- `--deploy-demo-jupyter` was printed twice in `--help`, and the - `demo-counter-microvm` flags were printed inside `demo-counter`'s block. - Each demo now gets exactly one block. +`hack/install-ate.sh` no longer installs anything. It merely directly invokes +`go run ./cmd/ate-setup`. New logic should not be added to it. \ No newline at end of file diff --git a/cmd/ate-setup/preflight_test.go b/cmd/ate-setup/preflight_test.go index f54785ee71..729907dc9e 100644 --- a/cmd/ate-setup/preflight_test.go +++ b/cmd/ate-setup/preflight_test.go @@ -31,6 +31,15 @@ import ( "go.ate.dev/substrate/cmd/ate-setup/internal/cmd" ) +func repoRoot(t *testing.T) string { + t.Helper() + out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() + if err != nil { + t.Fatalf("git rev-parse --show-toplevel: %v", err) + } + return strings.TrimSpace(string(out)) +} + // Each invocation gets its own command tree and flags, including invocations // made by the real shell shims. The test executable runs the installer without // rebuilding it or contacting a real cluster. @@ -48,7 +57,7 @@ func TestInstallerPreflightProcess(t *testing.T) { t.Fatal("missing installer arguments") } -func runPreflightInstaller(t *testing.T, kubeconfig, dockerStatus string, shim bool, args ...string) (string, int) { +func runPreflightInstaller(t *testing.T, kubeconfig, dockerStatus string, args ...string) (string, int) { t.Helper() bin := t.TempDir() executable, err := os.Executable() @@ -76,9 +85,6 @@ fi ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) defer cancel() process := exec.CommandContext(ctx, executable, append([]string{"-test.run=^TestInstallerPreflightProcess$", "--"}, args...)...) - if shim { - process = exec.CommandContext(ctx, "bash", append([]string{"hack/install-ate-kind.sh"}, args...)...) - } process.Dir = repoRoot(t) process.Env = []string{ "PATH=" + bin + string(os.PathListSeparator) + os.Getenv("PATH"), @@ -134,22 +140,15 @@ current-context: kind-kind } func TestKindInstallerMissingContext(t *testing.T) { - for _, shim := range []bool{false, true} { - t.Run(fmt.Sprintf("shim=%t", shim), func(t *testing.T) { - args := []string{"--kind", "deploy", "demo", "counter"} - if shim { - args = []string{"--deploy-demo-counter"} - } - out, code := runPreflightInstaller(t, preflightKubeconfig(t, ""), "0", shim, args...) - if code == 0 || !strings.Contains(out, "create-kind-cluster.sh") || !strings.Contains(out, "export kubeconfig") { - t.Fatalf("exit=%d, want missing-context recovery guidance\n%s", code, out) - } - }) + args := []string{"--kind", "deploy", "demo", "counter"} + out, code := runPreflightInstaller(t, preflightKubeconfig(t, ""), "0", args...) + if code == 0 || !strings.Contains(out, "create-kind-cluster.sh") || !strings.Contains(out, "export kubeconfig") { + t.Fatalf("exit=%d, want missing-context recovery guidance\n%s", code, out) } } func TestKindInstallerDockerUnavailable(t *testing.T) { - out, code := runPreflightInstaller(t, preflightKubeconfig(t, "http://127.0.0.1:1"), "1", false, + out, code := runPreflightInstaller(t, preflightKubeconfig(t, "http://127.0.0.1:1"), "1", "--kind", "deploy", "demo", "counter") if code == 0 || !strings.Contains(out, "Docker") || !strings.Contains(out, "docker info") { t.Fatalf("exit=%d, want Docker readiness guidance\n%s", code, out) @@ -159,14 +158,10 @@ func TestKindInstallerDockerUnavailable(t *testing.T) { func TestInstallerRequiresControlPlaneBeforeDemoOrBenchmark(t *testing.T) { for _, tc := range []struct { name string - shim bool args []string }{ - {"native demo", false, []string{"deploy", "demo", "counter"}}, - {"native benchmark", false, []string{"deploy", "benchmarks"}}, - {"kind demo", true, []string{"--deploy-demo-autoscaled-workerpool"}}, - {"kind benchmark", true, []string{"--deploy-benchmarks"}}, - {"demo before system", true, []string{"--deploy-demo-counter", "--deploy-ate-system"}}, + {"native demo", []string{"deploy", "demo", "counter"}}, + {"native benchmark", []string{"deploy", "benchmarks"}}, } { t.Run(tc.name, func(t *testing.T) { var mu sync.Mutex @@ -180,7 +175,7 @@ func TestInstallerRequiresControlPlaneBeforeDemoOrBenchmark(t *testing.T) { _, _ = fmt.Fprint(w, `{"kind":"Status","apiVersion":"v1","status":"Failure","reason":"NotFound","code":404,"message":"deployments.apps ate-controller not found"}`) })) defer server.Close() - out, code := runPreflightInstaller(t, preflightKubeconfig(t, server.URL), "0", tc.shim, tc.args...) + out, code := runPreflightInstaller(t, preflightKubeconfig(t, server.URL), "0", tc.args...) if code == 0 || !strings.Contains(out, "deploy ate-system") || !strings.Contains(out, "credential-provider") { t.Errorf("exit=%d, want control-plane install guidance\n%s", code, out) } @@ -195,15 +190,10 @@ func TestInstallerRequiresControlPlaneBeforeDemoOrBenchmark(t *testing.T) { } func TestKindInstallerHelpNeedsNoCluster(t *testing.T) { - for _, shim := range []bool{false, true} { - args := []string{"--kind", "deploy", "demo", "counter", "--help"} - if shim { - args = []string{"--help"} - } - out, code := runPreflightInstaller(t, preflightKubeconfig(t, ""), "1", shim, args...) - if code != 0 || !strings.Contains(out, "Usage:") { - t.Fatalf("shim=%t exit=%d, want help without prerequisites\n%s", shim, code, out) - } + args := []string{"--kind", "deploy", "demo", "counter", "--help"} + out, code := runPreflightInstaller(t, preflightKubeconfig(t, ""), "1", args...) + if code != 0 || !strings.Contains(out, "Usage:") { + t.Fatalf("exit=%d, want help without prerequisites\n%s", code, out) } } @@ -225,7 +215,7 @@ func TestInstallerControlPlaneCheckPreservesErrorsAndAllowsInstalledSystem(t *te _, _ = fmt.Fprint(w, `{"kind":"Status","apiVersion":"v1","status":"Failure","reason":"Forbidden","code":403,"message":"fixture access denied"}`) })) defer server.Close() - out, code := runPreflightInstaller(t, preflightKubeconfig(t, server.URL), "0", false, "deploy", "demo", "counter") + out, code := runPreflightInstaller(t, preflightKubeconfig(t, server.URL), "0", "deploy", "demo", "counter") want := "fixture access denied" if installed { want = "demo-counter_deploy" @@ -258,7 +248,7 @@ func TestKindInstallerUsesResolvedContextAndClusterName(t *testing.T) { if err := os.WriteFile(document, []byte("apiVersion: install.ate.dev/v1alpha1\nkind: SubstrateInstall\ncontext: selected-by-config\nkindCluster:\n name: local-dev\n"), 0o600); err != nil { t.Fatal(err) } - out, code := runPreflightInstaller(t, preflightKubeconfig(t, "http://127.0.0.1:1"), "0", false, + out, code := runPreflightInstaller(t, preflightKubeconfig(t, "http://127.0.0.1:1"), "0", "--kind", "--config", document, "deploy", "demo", "counter") for _, want := range []string{`context "selected-by-config"`, `export kubeconfig --name="local-dev"`, `KIND_CLUSTER_NAME="local-dev"`, "replaces an existing cluster"} { if code == 0 || !strings.Contains(out, want) { @@ -277,7 +267,7 @@ func TestKindInstallerMissingOrMalformedKubeconfig(t *testing.T) { } want = "while reading kubeconfig" } - out, code := runPreflightInstaller(t, path, "0", false, "--kind", "deploy", "demo", "counter") + out, code := runPreflightInstaller(t, path, "0", "--kind", "deploy", "demo", "counter") if code == 0 || !strings.Contains(out, want) { t.Errorf("malformed=%t exit=%d, want %q\n%s", malformed, code, want, out) } diff --git a/cmd/ate-setup/shim_test.go b/cmd/ate-setup/shim_test.go deleted file mode 100644 index 9c76926869..0000000000 --- a/cmd/ate-setup/shim_test.go +++ /dev/null @@ -1,605 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -// hack/install-ate.sh translates the installer's historical flags onto this -// binary's commands. These tests pin that translation: they put a stub `go` on -// PATH, run the script, and check the command lines it would have run. Nothing -// is built and no cluster is touched, so the whole file is a unit test of the -// argument mapping -- the one part of the shim that can silently break a CI job -// or a developer's muscle memory. - -import ( - "errors" - "os" - "os/exec" - "path/filepath" - "regexp" - "strings" - "testing" - - "go.ate.dev/substrate/cmd/ate-setup/internal/cmd" - "go.ate.dev/substrate/cmd/ate-setup/internal/config" - "go.ate.dev/substrate/cmd/ate-setup/internal/demos" - // Registers every bundled demo, so demos.All() is the real list. - _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/all" -) - -// stubGo is a `go` that records the arguments of every `go run ./cmd/ate-setup` -// the script makes, one invocation per line, and succeeds. -const stubGo = `#!/usr/bin/env bash -printf '%s\n' "$*" >> "${SHIM_TEST_LOG}" -` - -// runShim runs hack/install-ate.sh with args and returns the recorded -// invocations and the script's exit status. -func runShim(t *testing.T, env []string, args ...string) (invocations []string, exitCode int) { - t.Helper() - - root := repoRoot(t) - bin := t.TempDir() - if err := os.WriteFile(filepath.Join(bin, "go"), []byte(stubGo), 0o755); err != nil { - t.Fatalf("writing the stub go: %v", err) - } - log := filepath.Join(t.TempDir(), "invocations") - - script := exec.Command("bash", filepath.Join(root, "hack", "install-ate.sh")) - script.Args = append(script.Args, args...) - script.Dir = root - // A fixed environment: the script reads SETUP_CSI and STORAGE_CLASS, so a - // developer who exports either would otherwise change the expected output. - // git needs PATH, and the stub go has to come first on it. - script.Env = append([]string{ - "PATH=" + bin + ":" + os.Getenv("PATH"), - "HOME=" + os.Getenv("HOME"), - "SHIM_TEST_LOG=" + log, - }, env...) - - out, err := script.CombinedOutput() - var exitErr *exec.ExitError - switch { - case err == nil: - case errors.As(err, &exitErr): - exitCode = exitErr.ExitCode() - default: - t.Fatalf("running install-ate.sh %v: %v\n%s", args, err, out) - } - - recorded, err := os.ReadFile(log) - if err != nil && !os.IsNotExist(err) { - t.Fatalf("reading the invocation log: %v", err) - } - for _, line := range strings.Split(string(recorded), "\n") { - if line != "" { - invocations = append(invocations, line) - assertResolves(t, line) - } - } - return invocations, exitCode -} - -// goRunPrefix is what the script puts in front of every command line, since it -// invokes the installer as `go run ./cmd/ate-setup`. -const goRunPrefix = "run ./cmd/ate-setup " - -// assertResolves checks that a command line the script produced names a command -// ate-setup actually has, with flags it accepts and arguments it allows. -// -// Without this the tests below only pin the script against itself: the stub go -// exits 0 whatever it is handed, so renaming `deploy apiserver` would keep them -// green while every job that goes through the shim broke. -func assertResolves(t *testing.T, invocation string) { - t.Helper() - - argv, ok := strings.CutPrefix(invocation, goRunPrefix) - if !ok { - t.Errorf("the script ran `go %s`, want it to run the installer as `go %s...`", invocation, goRunPrefix) - return - } - args := strings.Fields(argv) - - // Find walks as deep into the tree as the arguments name a command, and - // hands back what is left for that command to parse. - target, rest, err := cmd.Root().Find(args) - if err != nil { - t.Errorf("`ate-setup %s`: %v", argv, err) - return - } - if err := target.ParseFlags(rest); err != nil { - t.Errorf("`ate-setup %s`: %v", argv, err) - return - } - if err := target.ValidateArgs(target.Flags().Args()); err != nil { - t.Errorf("`ate-setup %s`: %v", argv, err) - return - } - // A command that only groups subcommands prints help and does nothing, so - // resolving to one means the script named a subcommand that is gone. - if !target.Runnable() { - t.Errorf("`ate-setup %s` resolves to %q, which is not runnable", argv, target.CommandPath()) - } -} - -func repoRoot(t *testing.T) string { - t.Helper() - out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() - if err != nil { - t.Fatalf("git rev-parse --show-toplevel: %v", err) - } - return strings.TrimSpace(string(out)) -} - -// The script keeps its own copy of the demo list so that --help answers -// without a build and an unknown --deploy-demo-* is rejected outright. That -// copy has to track the registry: a demo added only to internal/demos would -// otherwise have no installer flag, which is exactly the regression the shim -// exists to prevent. -func TestShimDemoListMatchesTheRegistry(t *testing.T) { - script, err := os.ReadFile(filepath.Join(repoRoot(t), "hack", "install-ate.sh")) - if err != nil { - t.Fatalf("reading install-ate.sh: %v", err) - } - match := regexp.MustCompile(`(?s)\nATE_DEMOS=\((.*?)\n\)`).FindSubmatch(script) - if match == nil { - t.Fatal("no ATE_DEMOS=( ... ) array in install-ate.sh") - } - listed := map[string]bool{} - for _, name := range strings.Fields(string(match[1])) { - listed[name] = true - } - - registered := map[string]bool{} - for _, d := range demos.All() { - registered[d.Name()] = true - } - - for name := range registered { - if !listed[name] { - t.Errorf("%s is registered but has no --deploy-%s flag; add it to ATE_DEMOS", name, name) - } - } - for name := range listed { - if !registered[name] { - t.Errorf("ATE_DEMOS lists %s, which no demo package registers", name) - } - } -} - -func TestShimTranslatesFlags(t *testing.T) { - const prefix = "run ./cmd/ate-setup " - - for _, tc := range []struct { - name string - env []string - args []string - want []string - }{{ - name: "deploy ate-system carries the CSI driver, and --setup-csi also acts on its own", - args: []string{"--deploy-ate-system", "--setup-csi=nfs"}, - want: []string{ - "deploy ate-system --setup-csi=nfs", - "setup csi nfs", - }, - }, { - name: "a bare --setup-csi means nfs", - args: []string{"--setup-csi"}, - want: []string{"setup csi nfs"}, - }, { - name: "--setup-csi takes a separated value", - args: []string{"--setup-csi", "hostpath"}, - want: []string{"setup csi hostpath"}, - }, { - name: "no CSI driver by default", - args: []string{"--deploy-ate-system"}, - want: []string{"deploy ate-system --setup-csi=none"}, - }, { - // The value-bearing flags were pre-scanned, so they shape every action - // regardless of where they appear. - name: "global flags apply to actions that precede them", - args: []string{"--deploy-atenet", "--atenet-dataplane", "agentgateway"}, - want: []string{"--atenet-dataplane=agentgateway deploy atenet"}, - }, { - name: "cluster profile flags are forwarded in either value form", - args: []string{"--deploy-ate-system", "--cluster-size", "size10", "--cordon-control-plane"}, - want: []string{"--cluster-size=size10 --cordon-control-plane deploy ate-system --setup-csi=none"}, - }, { - name: "--ingress-allowed-spiffe-ids is forwarded in either value form", - args: []string{"--deploy-ate-system", "--ingress-allowed-spiffe-ids", "spiffe://cluster.local/ns/a/sa/a", "--deploy-atenet", "--ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/b/sa/b"}, - want: []string{ - "--ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/a/sa/a --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/b/sa/b deploy ate-system --setup-csi=none", - "--ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/a/sa/a --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/b/sa/b deploy atenet", - }, - }, { - name: "--fake-data-plane is forwarded to every action", - args: []string{"--deploy-ate-system", "--fake-data-plane", "--deploy-atelet"}, - want: []string{ - "--fake-data-plane deploy ate-system --setup-csi=none", - "--fake-data-plane deploy atelet", - }, - }, { - name: "--cluster-size takes an attached value", - args: []string{"--cluster-size=size10", "--deploy-ate-apiserver"}, - want: []string{"--cluster-size=size10 deploy apiserver"}, - }, { - name: "--ateapi-log-level is forwarded in either value form", - args: []string{"--ateapi-log-level", "debug", "--deploy-ate-apiserver", "--ateapi-log-level=warn"}, - want: []string{"--ateapi-log-level=debug --ateapi-log-level=warn deploy apiserver"}, - }, { - name: "--credential-provider is forwarded with a separate value", - args: []string{"--deploy-atenet", "--credential-provider", `{"name":"custom","address":"cred.ate-system.svc:50051"}`}, - want: []string{`--credential-provider={"name":"custom","address":"cred.ate-system.svc:50051"} deploy atenet`}, - }, { - name: "--credential-provider takes an attached value", - args: []string{`--credential-provider={"name":"k8s.io"}`, "--deploy-ate-system"}, - want: []string{`--credential-provider={"name":"k8s.io"} deploy ate-system --setup-csi=none`}, - }, { - name: "actions run in command line order", - args: []string{"--deploy-ate-apiserver", "--deploy-atelet", "--delete-atenet"}, - want: []string{ - "deploy apiserver", - "deploy atelet", - "delete atenet", - }, - }, { - name: "every create flag maps to a create subcommand", - args: []string{ - "--create-jwt-authority-pool-secret", - "--create-actor-id-ca-pool-secret", - "--create-actor-id-ca-certs-secret", - "--create-egress-mitm-ca-pool-secret", - "--create-podcertificate-controller-cas", - "--create-api-server-env-vars", - "--create-api-authentication-config", - "--create-api-config", - }, - want: []string{ - "create jwt-authority-pool", - "create actor-id-ca-pool", - "create actor-id-ca-certs", - "create egress-mitm-ca-pool", - "create podcertificate-controller-cas", - "create api-server-env-vars", - "create api-authentication-config", - "create api-config", - }, - }, { - name: "delete flags", - args: []string{"--delete-ate-system", "--delete-all"}, - want: []string{"delete ate-system", "delete all"}, - }, { - name: "benchmark flags are renamed and apply to both benchmark actions", - args: []string{ - "--benchmark-worker-count=4", "--deploy-benchmarks", - "--benchmark-sandbox-class", "microvm", "--delete-benchmarks", - }, - want: []string{ - "deploy benchmarks --worker-count=4 --sandbox-class=microvm", - "delete benchmarks --worker-count=4 --sandbox-class=microvm", - }, - }, { - name: "demo flags drop the demo- prefix", - args: []string{"--deploy-demo-egress-microvm", "--delete-demo-counter-microvm"}, - want: []string{ - "deploy demo egress-microvm", - "delete demo counter-microvm", - }, - }, { - name: "the counter external volume defaults to the standard storage class", - args: []string{"--deploy-demo-counter-with-external-volume"}, - want: []string{"deploy demo counter --with-external-volume --storage-class=standard"}, - }, { - name: "the counter external volume follows --setup-csi", - args: []string{"--setup-csi=hostpath", "--deploy-demo-counter-with-external-volume"}, - want: []string{ - "setup csi hostpath", - "deploy demo counter --with-external-volume --storage-class=csi-hostpath-sc", - }, - }, { - name: "STORAGE_CLASS beats the driver --setup-csi installed", - env: []string{"STORAGE_CLASS=my-class"}, - args: []string{"--setup-csi=nfs", "--deploy-demo-counter-with-external-volume"}, - want: []string{ - "setup csi nfs", - "deploy demo counter --with-external-volume --storage-class=my-class", - }, - }} { - t.Run(tc.name, func(t *testing.T) { - got, exitCode := runShim(t, tc.env, tc.args...) - if exitCode != 0 { - t.Fatalf("exit code = %d, want 0", exitCode) - } - want := make([]string, len(tc.want)) - for i, w := range tc.want { - want[i] = prefix + w - } - if strings.Join(got, "\n") != strings.Join(want, "\n") { - t.Errorf("invocations:\n got %q\n want %q", got, want) - } - }) - } -} - -// --benchmark-actor-memory has no ate-setup flag; the script exports it, which -// the stub go cannot observe through its arguments. -func TestShimExportsBenchmarkActorMemory(t *testing.T) { - root := repoRoot(t) - bin := t.TempDir() - const reportEnv = `#!/usr/bin/env bash -printf '%s\n' "BENCHMARK_ACTOR_MEMORY=${BENCHMARK_ACTOR_MEMORY:-}" -` - if err := os.WriteFile(filepath.Join(bin, "go"), []byte(reportEnv), 0o755); err != nil { - t.Fatalf("writing the stub go: %v", err) - } - - script := exec.Command("bash", filepath.Join(root, "hack", "install-ate.sh"), - "--benchmark-actor-memory", "512Mi", "--deploy-benchmarks") - script.Dir = root - script.Env = []string{"PATH=" + bin + ":" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME")} - - out, err := script.CombinedOutput() - if err != nil { - t.Fatalf("running install-ate.sh: %v\n%s", err, out) - } - if want := "BENCHMARK_ACTOR_MEMORY=512Mi\n"; string(out) != want { - t.Errorf("stub go saw %q, want %q", out, want) - } -} - -func TestShimRejectsUnknownFlags(t *testing.T) { - for _, args := range [][]string{ - {"--no-such-flag"}, - {"--deploy-demo-no-such-demo"}, - {"--delete-demo-no-such-demo"}, - } { - t.Run(strings.Join(args, " "), func(t *testing.T) { - got, exitCode := runShim(t, nil, args...) - if exitCode != 1 { - t.Errorf("exit code = %d, want 1", exitCode) - } - if len(got) != 0 { - t.Errorf("ran %q, want nothing", got) - } - }) - } -} - -// An action before an unknown flag still runs: the script dispatches as it -// walks the command line, as it always has. -func TestShimRunsActionsBeforeAnUnknownFlag(t *testing.T) { - got, exitCode := runShim(t, nil, "--deploy-atelet", "--no-such-flag") - if exitCode != 1 { - t.Errorf("exit code = %d, want 1", exitCode) - } - if want := []string{"run ./cmd/ate-setup deploy atelet"}; strings.Join(got, "\n") != strings.Join(want, "\n") { - t.Errorf("invocations = %q, want %q", got, want) - } -} - -func TestShimUsage(t *testing.T) { - for _, tc := range []struct { - name string - args []string - exitCode int - }{ - {name: "no arguments is an error", args: nil, exitCode: 1}, - {name: "--help anywhere exits cleanly", args: []string{"--deploy-ate-system", "--help"}, exitCode: 0}, - {name: "-h anywhere exits cleanly", args: []string{"-h"}, exitCode: 0}, - } { - t.Run(tc.name, func(t *testing.T) { - got, exitCode := runShim(t, nil, tc.args...) - if exitCode != tc.exitCode { - t.Errorf("exit code = %d, want %d", exitCode, tc.exitCode) - } - if len(got) != 0 { - t.Errorf("ran %q, want nothing: usage must not touch the cluster", got) - } - }) - } -} - -// Every flag the script dispatches on has to be documented, since --help is -// the only place a user can discover them now that the demos are Go packages. -func TestShimUsageDocumentsEveryFlag(t *testing.T) { - script, err := os.ReadFile(filepath.Join(repoRoot(t), "hack", "install-ate.sh")) - if err != nil { - t.Fatalf("reading install-ate.sh: %v", err) - } - usage := shimUsage(t) - - // The dispatch arms below `run_demo`'s definition, minus the demo - // wildcards, which usage covers one demo at a time. - for _, flag := range regexp.MustCompile(`(?m)^ (--[a-z0-9-]+)[)|]`).FindAllStringSubmatch(string(script), -1) { - if !strings.Contains(usage, flag[1]) { - t.Errorf("--help does not mention %s", flag[1]) - } - } - for _, d := range demos.All() { - for _, flag := range []string{"--deploy-" + d.Name(), "--delete-" + d.Name()} { - if !strings.Contains(usage, flag) { - t.Errorf("--help does not mention %s", flag) - } - } - } -} - -func shimUsage(t *testing.T) string { - t.Helper() - root := repoRoot(t) - script := exec.Command("bash", filepath.Join(root, "hack", "install-ate.sh"), "--help") - script.Dir = root - script.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME")} - out, err := script.Output() - if err != nil { - t.Fatalf("install-ate.sh --help: %v", err) - } - return string(out) -} - -// Settings register from each command package's init, so the merged set only -// exists in a binary that links the whole command tree. Package config cannot -// check it: the commands that register are the ones config does not import. -// This package does import them, which is what makes the check possible here -// and nowhere below it. -func TestSettingRegistryIsValidAcrossEveryCommand(t *testing.T) { - if err := config.Validate(); err != nil { - t.Fatal(err) - } -} - -// Command-scoped settings exist so a flag appears only where it applies. A -// setting that named a command nobody built would silently never bind. -func TestEveryScopedSettingNamesARealCommand(t *testing.T) { - for _, s := range config.All() { - for _, path := range s.Commands { - t.Run(s.Key+" on "+path, func(t *testing.T) { - target, _, err := cmd.Root().Find(strings.Fields(path)) - if err != nil { - t.Fatalf("Find(%q): %v", path, err) - } - if target.CommandPath() != "ate-setup "+path { - t.Fatalf("%q resolved to %q, which is not that command", path, target.CommandPath()) - } - if target.Flags().Lookup(s.Flag) == nil { - t.Errorf("--%s is not bound to %q", s.Flag, path) - } - }) - } - } -} - -// docPath resolves a repository path from this package's directory. -func docPath(t *testing.T, rel string) string { - t.Helper() - root, err := config.RepoRoot() - if err != nil { - t.Fatalf("RepoRoot: %v", err) - } - return filepath.Join(root, rel) -} - -// The worked example is the first thing an operator copies, so a renamed or -// removed key has to break the build rather than their install. -func TestExampleConfigurationParses(t *testing.T) { - path := docPath(t, "docs/examples/substrate-install.yaml") - f, err := config.ParseFile(path) - if err != nil { - t.Fatalf("ParseFile(%s) = %v", path, err) - } - if len(f.Keys()) == 0 { - t.Error("the example sets no settings, so it demonstrates nothing") - } -} - -const operatorDoc = "docs/operator-install.md" - -// settingDocs maps a key prefix to the document that lists the settings under -// it. Demo, benchmark, and e2e settings are documented with what uses them, -// so the operator page lists only what an install uses. -var settingDocs = []struct{ prefix, doc string }{ - {"demo.counter.", "demos/counter/README.md"}, - {"demo.anthropicAPIKey", "demos/claude-code-multiplex/README.md"}, - {"benchmark.", "benchmarking/README.md"}, - {"e2e.", "internal/e2e/README.md"}, -} - -// settingDoc returns the document that lists the setting with this key. -func settingDoc(key string) string { - for _, d := range settingDocs { - if strings.HasPrefix(key, d.prefix) { - return d.doc - } - } - return operatorDoc -} - -// readDoc returns the contents of a repository file, by relative path. -func readDoc(t *testing.T, rel string) string { - t.Helper() - raw, err := os.ReadFile(docPath(t, rel)) - if err != nil { - t.Fatalf("ReadFile: %v", err) - } - return string(raw) -} - -// The settings tables are written by hand, so a setting added without -// documenting it is the drift this catches. A demo, benchmark, or e2e setting -// must also stay out of the operator page. -func TestDocsListEverySetting(t *testing.T) { - docs := map[string]string{operatorDoc: readDoc(t, operatorDoc)} - for _, d := range settingDocs { - docs[d.doc] = readDoc(t, d.doc) - } - - for _, s := range config.All() { - t.Run(s.Key, func(t *testing.T) { - rel := settingDoc(s.Key) - want := []string{"`" + s.Key + "`", "`" + s.Env + "`"} - // A secret has no flag, so the docs have none to name. - if s.Flag != "" { - want = append(want, "`--"+s.Flag+"`") - } - for _, want := range want { - if !strings.Contains(docs[rel], want) { - t.Errorf("%s does not mention %s", rel, want) - } - } - if rel != operatorDoc && strings.Contains(docs[operatorDoc], "`"+s.Key+"`") { - t.Errorf("%s lists %s, which belongs in %s", operatorDoc, s.Key, rel) - } - }) - } -} - -// The settings documentation must not name a setting that no longer exists: -// following it would configure nothing and the install would silently use a -// default. -func TestDocsNameNoRemovedSetting(t *testing.T) { - // The two controls that select how configuration is read and reported. - // They are not settings themselves, so no registry entry declares them. - notSettings := map[string]bool{ - "ATE_CONFIG": true, - "ATE_NO_REPORT_DEFAULTS": true, - } - - declared := map[string]bool{} - for _, s := range config.All() { - declared[s.Env] = true - } - - root, err := config.RepoRoot() - if err != nil { - t.Fatalf("RepoRoot: %v", err) - } - files := []string{operatorDoc, "docs/examples/substrate-install.yaml"} - for _, d := range settingDocs { - files = append(files, d.doc) - } - for _, rel := range files { - raw, err := os.ReadFile(filepath.Join(root, rel)) - if err != nil { - t.Fatalf("ReadFile(%s): %v", rel, err) - } - seen := map[string]bool{} - for _, m := range regexp.MustCompile(`\b(ATE_[A-Z0-9_]{3,})\b`).FindAllStringSubmatch(string(raw), -1) { - name := m[1] - if declared[name] || notSettings[name] || seen[name] { - continue - } - seen[name] = true - t.Errorf("%s names %s, which no setting declares", rel, name) - } - } -} diff --git a/demos/claude-code-multiplex/README.md b/demos/claude-code-multiplex/README.md index 23cbd0ddb8..1e2c4c930d 100644 --- a/demos/claude-code-multiplex/README.md +++ b/demos/claude-code-multiplex/README.md @@ -26,7 +26,7 @@ This guide assumes you know Kubernetes and the general shape of agent runtimes ( ## Prerequisites -- A Kubernetes cluster with **Agent Substrate** installed (`./hack/install-ate.sh` from this repo's root). +- A Kubernetes cluster with **Agent Substrate** installed. - `kubectl` configured against that cluster (the dashboard uses the operator's kubeconfig via [`client-go`](https://github.com/kubernetes/client-go) for pod-log reads). - Network reach to the substrate **api** gRPC service (`api.ate-system:443`). If you run the dashboard as a pod in the cluster, label its namespace `ate.dev/api-access=true`, because ate-system's NetworkPolicies admit API clients only from labeled namespaces (see [`ate-system-networkpolicies.yaml`](../../manifests/ate-install/ate-system-networkpolicies.yaml)). When running the dashboard from outside the cluster, port-forward it in a separate terminal (same convention as [`demos/sandbox/README.md`](../sandbox/README.md#3-port-forward-services)) and keep it running for the lifetime of the demo: ```bash @@ -57,7 +57,7 @@ From the repo root, with your Anthropic key and substrate bucket name in the env ```bash ANTHROPIC_API_KEY=sk-ant-... \ BUCKET_NAME=your-substrate-bucket \ - ./hack/install-ate.sh --deploy-demo-claude-code-multiplex +go run ./cmd/ate-setup deploy demo claude-code-multiplex ``` This creates the `claude-multiplex-demo` namespace and its 2-pod `WorkerPool`, then the `claude-multiplex-demo` atespace and three actor templates in it named `agent-luna`, `agent-mars`, `agent-orion`, and waits for each template's golden snapshot. Under the hood, the deploy function builds the workload image with `docker buildx`, pushes it to `${KO_DOCKER_REPO}/claude-multiplex-demo-workload`, resolves the pushed sha256 digest, and substitutes the digest-pinned reference plus `ANTHROPIC_API_KEY` and `BUCKET_NAME` into the agent templates at apply time. @@ -127,7 +127,7 @@ This demo currently applies workarounds at runtime for two Substrate issues. See ## Teardown ```bash -./hack/install-ate.sh --delete-demo-claude-code-multiplex +go run ./cmd/ate-setup delete demo claude-code-multiplex ``` This removes the demo's actors, templates, atespace, and the `claude-multiplex-demo` namespace with everything the deploy step created in it. You can also stop the port-forward and the dashboard processes in their respective terminals. diff --git a/demos/counter/README.md b/demos/counter/README.md index 7a4b99317a..4098d6aa71 100644 --- a/demos/counter/README.md +++ b/demos/counter/README.md @@ -8,7 +8,7 @@ The demo uses two kinds of resources: the `WorkerPool` is a Kubernetes CRD, whil ## Prerequisites -- A k8s cluster with Agent Substrate installed (`./hack/install-ate.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}'`). +- A k8s cluster with Agent Substrate installed. - `ko` installed for building images. - A GCS bucket for storing snapshots (configured via `BUCKET_NAME` env var). @@ -52,7 +52,7 @@ On a cluster that already has the micro-VM deps (`hack/install-microvm-deps.sh`) deploy directly instead: ```bash -./hack/install-ate.sh --deploy-demo-counter-microvm +go run ./cmd/ate-setup deploy demo counter-microvm ``` @@ -276,7 +276,7 @@ To remove the counter demo resources from your cluster, run: Delete from MicroVM ```bash -./hack/install-ate.sh --delete-demo-counter +go run ./cmd/ate-setup delete demo counter-microvm ``` @@ -284,7 +284,7 @@ To remove the counter demo resources from your cluster, run: Delete from GVisor ```bash -./hack/install-ate.sh --delete-demo-counter-microvm +go run ./cmd/ate-setup delete demo counter ``` diff --git a/demos/egress/README.md b/demos/egress/README.md index 9535e699d3..78daf91713 100644 --- a/demos/egress/README.md +++ b/demos/egress/README.md @@ -105,7 +105,7 @@ rejects that option with agentgateway rather than silently omitting it. ## Deploy the demo fixture ```bash -./hack/install-ate.sh --deploy-demo-egress +go run ./cmd/ate-setup deploy demo egress ``` The install applies the worker pool, creates the `ate-demo-egress` atespace and @@ -253,5 +253,5 @@ from the cluster, works for a manual run. ```bash ./demos/egress/test-egress.sh --cleanup -./hack/install-ate.sh --delete-demo-egress +go run ./cmd/ate-setup delete demo egress ``` diff --git a/demos/multi-template/README.md b/demos/multi-template/README.md index 2f8295fab9..bbf8926608 100644 --- a/demos/multi-template/README.md +++ b/demos/multi-template/README.md @@ -9,7 +9,7 @@ or namespace. ## Prerequisites -- A k8s cluster with Agent Substrate installed (`./hack/install-ate.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}'`). +- A k8s cluster with Agent Substrate installed. - `ko` installed for building images. - A GCS bucket for storing snapshots (configured via `BUCKET_NAME` env var). @@ -23,7 +23,7 @@ or namespace. > variable during deployment. ```bash -./hack/install-ate.sh --deploy-demo-multi-template +go run ./cmd/ate-setup deploy demo multi-template ``` This command will: @@ -99,5 +99,5 @@ Remove the demo — this deletes the actors (suspending running ones first), the templates, both atespaces, and then the pool and its namespace: ```bash -./hack/install-ate.sh --delete-demo-multi-template +go run ./cmd/ate-setup delete demo multi-template ``` diff --git a/demos/parking/README.md b/demos/parking/README.md index 5aeaaa3529..22a95c12ef 100644 --- a/demos/parking/README.md +++ b/demos/parking/README.md @@ -13,7 +13,7 @@ See [docs/request-parking.md](../../docs/request-parking.md) for the design. ## Prerequisites -- A k8s cluster with Agent Substrate installed (`./hack/install-ate.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}'`). +- A k8s cluster with Agent Substrate installed. - `ko` installed for building images. - A GCS bucket for storing snapshots (configured via `BUCKET_NAME` env var). @@ -27,7 +27,7 @@ See [docs/request-parking.md](../../docs/request-parking.md) for the design. > deployment. ```bash -./hack/install-ate.sh --deploy-demo-parking +go run ./cmd/ate-setup deploy demo parking ``` This command will: @@ -166,5 +166,5 @@ Remove the demo — this deletes the demo's actors (suspending running ones first) and then the template, atespace, pool, and namespace: ```bash -./hack/install-ate.sh --delete-demo-parking +go run ./cmd/ate-setup delete demo parking ``` diff --git a/demos/sandbox/README.md b/demos/sandbox/README.md index 49a1ea8e52..4be9024390 100644 --- a/demos/sandbox/README.md +++ b/demos/sandbox/README.md @@ -29,7 +29,7 @@ It allows you to run arbitrary commands in an sandboxed, isolated container (run Use the core installation script to build the image and deploy the demo to your cluster: ```bash -./hack/install-ate.sh --deploy-demo-sandbox +go run ./cmd/ate-setup deploy demo sandbox ``` This command will: @@ -109,5 +109,5 @@ kubectl ate delete actor my-sandbox-1 -a ate-demo-sandbox To remove the sandbox demo resources (actors, template, atespace, workerpool, and namespace) from your cluster, run: ```bash -./hack/install-ate.sh --delete-demo-sandbox +go run ./cmd/ate-setup delete demo sandbox ``` diff --git a/docs/egress-trust-bundle.md b/docs/egress-trust-bundle.md index 591c92c48a..d82241860b 100644 --- a/docs/egress-trust-bundle.md +++ b/docs/egress-trust-bundle.md @@ -123,7 +123,7 @@ Note: Do not bake the bundle into the image at build time. It will tie the image does exactly this. Deploy it: ```bash -./hack/install-ate.sh --deploy-demo-egress +go run ./cmd/ate-setup deploy demo egress ``` Then drive an actor's egress at an HTTPS URL an `https` rule allows and diff --git a/hack/install-ate-kind.sh b/hack/install-ate-kind.sh index cc6a6042be..4c4e1ffc6e 100755 --- a/hack/install-ate-kind.sh +++ b/hack/install-ate-kind.sh @@ -41,4 +41,4 @@ export KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-kind-${KIND_CLUSTER_NAME}}" # unset other env from ate-dev-env.sh in case the developer already sourced them unset GCE_REGION CLUSTER_LOCATION NETWORK SUBNETWORK MEMORYSTORE_INSTANCE PROJECT_ID -hack/install-ate.sh "$@" +go run ./cmd/ate-setup "$@" diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 984942bfff..edc0fb905f 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -32,448 +32,4 @@ set -o errexit -o nounset -o pipefail -ROOT="$(git rev-parse --show-toplevel)" -cd "${ROOT}" - -# Demos, in the order --help lists them. Keeping the list here rather than -# asking ate-setup for it means --help answers without a build, and that a demo -# flag this script does not know is rejected outright rather than handed to -# ate-setup as an unknown subcommand. `go run ./cmd/ate-setup deploy demo -# --help` is the authoritative list; add new demos there and mirror them here. -ATE_DEMOS=( - demo-counter - demo-counter-microvm - demo-egress - demo-egress-microvm - demo-jupyter - demo-sandbox - demo-claude-code-multiplex - demo-multi-template - demo-parking - demo-autoscaled-workerpool -) - -# Prerequisites and caveats worth repeating in --help, keyed by demo. -demo_usage() { - case "$1" in - demo-counter) - echo " --deploy-demo-counter-with-external-volume Deploy demo-counter with external volume validation" - echo " (STORAGE_CLASS names the class; it otherwise follows --setup-csi)" - ;; - demo-counter-microvm|demo-egress-microvm) - echo " Needs hack/install-microvm-deps.sh --install to have run (cluster-wide microvm SandboxConfig)." - ;; - demo-claude-code-multiplex) - echo " Required env: ANTHROPIC_API_KEY, BUCKET_NAME, KO_DOCKER_REPO" - echo " See demos/claude-code-multiplex/README.md for the walkthrough." - ;; - demo-autoscaled-workerpool) - echo " Kind only: it ships its own prometheus-adapter and a Kind-specific HPA." - ;; - esac -} - -usage() { - echo "Usage: $0 [options]" - echo "" - echo "Configuration:" - echo "" - echo " ATE_CONFIG=PATH Read settings from a configuration document. Every setting below that" - echo " names an environment variable can be set there instead, including the" - echo " ones this script turns into flags. Settings in the document outrank" - echo " the environment; a flag passed here outranks both." - echo " See docs/operator-install.md." - echo "" - echo " ATE_NO_REPORT_DEFAULTS=1 List only the settings something set in the configuration" - echo " report, rather than every setting and where it came from." - echo "" - echo "Overall infrastructure (all infrastructure components):" - echo "" - echo " --deploy-ate-system Deploy core system (CRDs, atelet, apiserver)" - echo " --preview GATES Enable preview gates (comma-separated, ALL for all, '' for none)" - echo " --setup-csi[=DRIVER] Setup CSI driver: nfs, hostpath, both, none (default: none;" - echo " a bare --setup-csi means nfs; hostpath is Kind only)" - echo " --delete-ate-system Delete core system" - echo " --delete-all Delete core system and all registered demos" - echo " --atenet-dataplane=envoy|agentgateway Select the atenet ingress and egress dataplane (default: envoy)" - echo " --podcert-workers-per-signer N Concurrent workers per podcertificate-controller signer (default: 1)" - echo " --cluster-size size0|size10 Cluster size profile (default: size0). \"size10\" assumes a dedicated postgres node" - echo " --cordon-control-plane Keep the control plane off the worker nodes: assumes a small shared pool labeled and" - echo " tainted ate.dev/workloadType=ate-control-plane:NoSchedule, and a one-node pool" - echo " labeled and tainted ate.dev/workloadType=ate-postgres:NoSchedule for postgres alone" - echo " --fake-data-plane Install fake-atelet in place of atelet and fake-workersync in place of" - echo " ate-controller, for control-plane benchmarks. Actors do not exist; never" - echo " use it on a cluster that serves real actors" - echo " --ingress-auth-mode=deprecated-insecure|static-mtls" - echo " How the ingress router authenticates clients (default: deprecated-insecure)." - echo " Both check a client certificate on the TLS ports against the podidentity CA and" - echo " --ingress-allowed-spiffe-ids." - echo " deprecated-insecure also accepts TLS clients without one and serves plaintext" - echo " ports; static-mtls requires the certificate and --atenet-dataplane=envoy." - echo " --ingress-allowed-spiffe-ids=IDS SPIFFE IDs (comma-separated) of the clients the ingress router lets reach actors" - echo " over its TLS ports. Required with --atenet-dataplane=envoy" - echo " --rollout-timeout DURATION Per-workload readiness wait timeout, kubectl-style Go duration (default: 60s)" - echo " --otlp-endpoint URL Send all control plane telemetry to URL, not to the cluster default (see benchmarking/telemetry/README.md)" - echo " --ateapi-log-level LEVEL ate-api-server log level: debug, info, warn, or error" - echo " (default: keep the installed level, else info)" - echo "" - echo "Experiments:" - echo "" - echo " --experimental-additional-egress-extproc-service NS/SVC:PORT" - echo " Run an additional ext_proc authorization filter, served by that Service." - echo " Requires --atenet-dataplane=envoy." - echo "" - echo "Egress credential injection (required by --deploy-ate-system and --deploy-atenet):" - echo "" - echo " --credential-provider JSON Credential provider the egress gateway injects credentials from, or" - echo " ATE_CREDENTIAL_PROVIDER. One of:" - echo " '{\"name\":\"k8s.io\"}' deploy the bundled Kubernetes Secrets provider," - echo " with a NetworkPolicy that admits only the egress gateway" - echo " '{\"enabled\":false}' turn injection off" - echo " '{\"name\":\"\",\"address\":\":\"}'" - echo " use a provider you deploy yourself" - echo " Any provider requires --atenet-dataplane=envoy." - echo "" - echo "Infrastructure components:" - echo "" - echo " --deploy-atelet Deploy atelet only" - echo " --deploy-ate-apiserver Deploy ate-api-server and ate-idp-server" - echo " --deploy-atenet Deploy atenet only" - echo " --delete-atenet Delete atenet only" - echo "" - echo "To create individual resources used by ate-system (Note: These are" - echo "called automatically by --deploy-ate-system):" - echo "" - echo " --create-jwt-authority-pool-secret Create JWT authority pool secret" - echo " --create-actor-id-ca-pool-secret Create actor ID CA pool secret" - echo " --create-actor-id-ca-certs-secret Create actor ID CA certs secret" - echo " --create-egress-mitm-ca-pool-secret Create egress MITM CA pool secret" - echo " --create-podcertificate-controller-cas Create podcertificate controller CAs" - echo " --create-api-server-env-vars Create ate-api-server env vars" - echo " --create-api-authentication-config Create the default ate-api-server authentication config" - echo " --create-api-config Create the ate-api-server config file (log level from --ateapi-log-level)" - echo "" - echo "PostgreSQL configuration (a connection DSN or Cloud SQL instance selects" - echo "an external database and skips the bundled instance):" - echo "" - echo " ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING Application DSN for external PostgreSQL (stored in a Secret;" - echo " pair with ATE_API_POSTGRES_SERVER_CA_FILE for sslmode=verify-ca)" - echo " ATE_API_POSTGRES_OWNER_CONNECTION_STRING Optional owner DSN; defaults to the read/write DSN" - echo " ATE_API_POSTGRES_READ_WRITE_ROLE Role assumed by the read/write pool (default: substrate_readwrite)" - echo " ATE_API_POSTGRES_OWNER_ROLE Role assumed by the owner pool (default: substrate_owner)" - echo " ATE_API_POSTGRES_CLOUDSQL_INSTANCE Cloud SQL instance connection name (project:region:instance)." - echo " Deploys the Cloud SQL Auth Proxy sidecar: connector-managed TLS" - echo " and automatic IAM database auth, no passwords (see tools/setup-gcp/cloud-sql.md)." - echo " Unset = keep the cluster's current Cloud SQL config; set to \"\" to remove it" - echo " ATE_API_POSTGRES_CLOUDSQL_GSA GSA email backing Workload Identity + the IAM database user" - echo " ATE_API_POSTGRES_CLOUDSQL_IP_TYPE private (default) | public | psc" - echo " ATE_API_POSTGRES_CLOUDSQL_IAM_AUTH true (default) | false (password-over-proxy escape hatch)" - echo " ATE_API_POSTGRES_POOL_MAX_CONNS pgxpool max connections per ateapi replica (default: max(4, NumCPU))" - echo " ATE_API_POSTGRES_SERVER_CA_FILE PEM file to mount for verify-ca DSNs (non-Cloud-SQL databases)" - echo " ATE_API_POSTGRES_SCHEMA Select the Substrate schema (default: substrate)" - echo "" - echo "Authentication configuration:" - echo "" - echo " EXPECTED_JWT_ISSUER Issuer URL ate-api-server requires in service account tokens, verbatim." - echo " Default: derived from PROJECT_ID/CLUSTER_LOCATION/CLUSTER_NAME" - echo " (https://container.googleapis.com/v1/projects/.../clusters/...)," - echo " else the cluster's OIDC discovery document" - echo " ACTOR_JWT_ALGORITHM Signing algorithm of a newly created actor JWT pool: ES256 (default) | RS256." - echo " Use RS256 (a 4096-bit RSA key) for relying parties that don't support ES256. Has no effect once the pool exists." - echo " ACTOR_JWT_ISSUER iss claim of actor JWTs, as https:// with no path. Relying parties fetch /.well-known/openid-configuration." - echo " Default: https://idp..svc" - echo "" - echo "Authorization configuration:" - echo "" - echo " ATE_API_EXPERIMENTAL_ENABLE_AUTHZ false turns off ate-api-server's runtime authorization. Default: true" - echo " ATE_API_AUTHZ_BOOTSTRAP_OWNERS Comma-separated principals that are always global owners." - echo " Default: kubectl-ate's, system:serviceaccount::ate-client" - echo " See docs/authorization.md." - echo "" - echo "Benchmarks (see benchmarking/README.md for details and customization):" - echo "" - echo " --deploy-benchmarks Deploy workloads + locust load test stack" - echo " --delete-benchmarks Delete the locust stack and workloads" - echo " --benchmark-worker-count N Number of WorkerPool replicas (default: 1)" - echo " --benchmark-sandbox-class CLASS Sandbox runtime for the benchmark WorkerPool: gvisor | microvm (default: gvisor)." - echo " microvm requires hack/install-microvm-deps.sh --install to have run." - echo " --benchmark-actor-memory SIZE Memory limit for the benchmark ActorTemplates (default: 256Mi," - echo " the smallest size microvm admits)" - echo "" - local demo_name - for demo_name in "${ATE_DEMOS[@]}"; do - echo "Demo: ${demo_name}" - echo "" - echo " --deploy-${demo_name} Deploy ${demo_name}" - echo " --delete-${demo_name} Delete ${demo_name}" - demo_usage "${demo_name}" - echo "" - done -} - -# ate_setup runs one ate-setup command. GLOBAL_FLAGS holds the pre-scanned -# flags that shape every action, which is how they behaved here: a single -# command line could ask for several actions and one --atenet-dataplane. -ate_setup() { - go run ./cmd/ate-setup ${GLOBAL_FLAGS[@]+"${GLOBAL_FLAGS[@]}"} "$@" -} - -# counter_storage_class names the StorageClass the counter demo's external -# volume is provisioned from. STORAGE_CLASS names it outright; without it, fall -# back to whatever --setup-csi just installed, and to ate-setup's own default -# when it installed nothing. -counter_storage_class() { - if [[ -n "${STORAGE_CLASS:-}" ]]; then - echo "${STORAGE_CLASS}" - return - fi - case "${SETUP_CSI}" in - hostpath) echo "csi-hostpath-sc" ;; - nfs|both|true) echo "csi-nfs-sc" ;; - *) echo "standard" ;; - esac -} - -# run_demo maps --deploy-demo-NAME / --delete-demo-NAME onto -# `ate-setup {deploy,delete} demo NAME`. A demo this installer does not -# register is rejected the way any other unknown flag is, rather than reaching -# ate-setup as an unknown subcommand. -run_demo() { - local action="$1" flag="$2" - local demo="${flag#--"${action}"-}" - local known - for known in "${ATE_DEMOS[@]}"; do - if [[ "${demo}" == "${known}" ]]; then - ate_setup "${action}" demo "${demo#demo-}" - return - fi - done - echo "Error: unknown option: ${flag}" >&2 - echo "" - usage - exit 1 -} - -if [ "$#" -eq 0 ]; then - usage - exit 1 -fi - -# If -h or --help appears anywhere in the command line, print the usage and exit. -for arg in "$@"; do - case "$arg" in - -h|--help) - usage - exit 0 - ;; - esac -done - -# Pre-scan value-bearing flags so they can appear before or after the action -# flag they configure (e.g. --benchmark-worker-count before/after -# --deploy-benchmarks). The dispatch loop below also accepts these flags but -# treats them as no-ops since the value is already captured here. -GLOBAL_FLAGS=() -# Valid values for SETUP_CSI: nfs, hostpath, both, none. Defaults to none: a -# CSI driver is an opt-in extra, and NFS needs kernel modules a plain -# workstation will not have loaded. -SETUP_CSI="${SETUP_CSI:-none}" -BENCHMARK_FLAGS=() -prescan_args=("$@") -for ((i = 0; i < ${#prescan_args[@]}; i++)); do - case "${prescan_args[i]}" in - --atenet-dataplane=*) GLOBAL_FLAGS+=("${prescan_args[i]}") ;; - --atenet-dataplane) - if (( i + 1 >= ${#prescan_args[@]} )); then - echo "Error: --atenet-dataplane requires envoy or agentgateway" >&2 - exit 1 - fi - GLOBAL_FLAGS+=("--atenet-dataplane=${prescan_args[$((i + 1))]}") - ;; - --preview=*) GLOBAL_FLAGS+=("${prescan_args[i]}") ;; - --preview) - if (( i + 1 >= ${#prescan_args[@]} )); then - echo "Error: --preview requires a value" >&2 - exit 1 - fi - GLOBAL_FLAGS+=("--preview=${prescan_args[$((i + 1))]}") - ;; - --experimental-additional-egress-extproc-service=*) - GLOBAL_FLAGS+=("${prescan_args[i]}") - ;; - --experimental-additional-egress-extproc-service) - if (( i + 1 >= ${#prescan_args[@]} )); then - echo "Error: --experimental-additional-egress-extproc-service requires /:" >&2 - exit 1 - fi - GLOBAL_FLAGS+=("--experimental-additional-egress-extproc-service=${prescan_args[$((i + 1))]}") - ;; - --credential-provider=*) GLOBAL_FLAGS+=("${prescan_args[i]}") ;; - --credential-provider) - if (( i + 1 >= ${#prescan_args[@]} )); then - echo "Error: --credential-provider requires a JSON value, e.g. '{\"name\":\"k8s.io\"}'" >&2 - exit 1 - fi - GLOBAL_FLAGS+=("--credential-provider=${prescan_args[$((i + 1))]}") - ;; - --podcert-workers-per-signer=*) GLOBAL_FLAGS+=("${prescan_args[i]}") ;; - --podcert-workers-per-signer) - if (( i + 1 >= ${#prescan_args[@]} )); then - echo "Error: --podcert-workers-per-signer requires a positive integer" >&2 - exit 1 - fi - GLOBAL_FLAGS+=("--podcert-workers-per-signer=${prescan_args[$((i + 1))]}") - ;; - --cluster-size=*) GLOBAL_FLAGS+=("${prescan_args[i]}") ;; - --cluster-size) - if (( i + 1 >= ${#prescan_args[@]} )); then - echo "Error: --cluster-size requires size0 or size10" >&2 - exit 1 - fi - GLOBAL_FLAGS+=("--cluster-size=${prescan_args[$((i + 1))]}") - ;; - --cordon-control-plane|--cordon-control-plane=*) GLOBAL_FLAGS+=("${prescan_args[i]}") ;; - --fake-data-plane|--fake-data-plane=*) GLOBAL_FLAGS+=("${prescan_args[i]}") ;; - --ingress-auth-mode=*) GLOBAL_FLAGS+=("${prescan_args[i]}") ;; - --ingress-auth-mode) - if (( i + 1 >= ${#prescan_args[@]} )); then - echo "Error: --ingress-auth-mode requires deprecated-insecure or static-mtls" >&2 - exit 1 - fi - GLOBAL_FLAGS+=("--ingress-auth-mode=${prescan_args[$((i + 1))]}") - ;; - --ingress-allowed-spiffe-ids=*) GLOBAL_FLAGS+=("${prescan_args[i]}") ;; - --ingress-allowed-spiffe-ids) - if (( i + 1 >= ${#prescan_args[@]} )); then - echo "Error: --ingress-allowed-spiffe-ids requires a comma-separated list of SPIFFE IDs" >&2 - exit 1 - fi - GLOBAL_FLAGS+=("--ingress-allowed-spiffe-ids=${prescan_args[$((i + 1))]}") - ;; - --rollout-timeout=*) GLOBAL_FLAGS+=("${prescan_args[i]}") ;; - --rollout-timeout) - if (( i + 1 >= ${#prescan_args[@]} )); then - echo "Error: --rollout-timeout requires a Go duration (e.g. 300s, 10m)" >&2 - exit 1 - fi - GLOBAL_FLAGS+=("--rollout-timeout=${prescan_args[$((i + 1))]}") - ;; - --otlp-endpoint=*) GLOBAL_FLAGS+=("${prescan_args[i]}") ;; - --otlp-endpoint) - if (( i + 1 >= ${#prescan_args[@]} )); then - echo "Error: --otlp-endpoint requires a URL" >&2 - exit 1 - fi - GLOBAL_FLAGS+=("--otlp-endpoint=${prescan_args[$((i + 1))]}") - ;; - --ateapi-log-level=*) GLOBAL_FLAGS+=("${prescan_args[i]}") ;; - --ateapi-log-level) - if (( i + 1 >= ${#prescan_args[@]} )); then - echo "Error: --ateapi-log-level requires debug, info, warn, or error" >&2 - exit 1 - fi - GLOBAL_FLAGS+=("--ateapi-log-level=${prescan_args[$((i + 1))]}") - ;; - --benchmark-worker-count=*) BENCHMARK_FLAGS+=("--worker-count=${prescan_args[i]#*=}") ;; - --benchmark-worker-count) - BENCHMARK_FLAGS+=("--worker-count=${prescan_args[i+1]:-1}") - ;; - --benchmark-sandbox-class=*) BENCHMARK_FLAGS+=("--sandbox-class=${prescan_args[i]#*=}") ;; - --benchmark-sandbox-class) - if (( i + 1 >= ${#prescan_args[@]} )); then - echo "Error: --benchmark-sandbox-class requires gvisor or microvm" >&2 - exit 1 - fi - BENCHMARK_FLAGS+=("--sandbox-class=${prescan_args[$((i + 1))]}") - ;; - # The benchmark actor memory has no ate-setup flag; it is read from the - # environment, which is also how the benchmark scripts have always taken it. - --benchmark-actor-memory=*) export BENCHMARK_ACTOR_MEMORY="${prescan_args[i]#*=}" ;; - --benchmark-actor-memory) - if (( i + 1 >= ${#prescan_args[@]} )); then - echo "Error: --benchmark-actor-memory requires a size (e.g. 256Mi)" >&2 - exit 1 - fi - export BENCHMARK_ACTOR_MEMORY="${prescan_args[$((i + 1))]}" - ;; - --setup-csi=*) SETUP_CSI="${prescan_args[i]#*=}" ;; - --setup-csi) - if (( i + 1 < ${#prescan_args[@]} )) && [[ "${prescan_args[$((i + 1))]}" != --* ]]; then - SETUP_CSI="${prescan_args[$((i + 1))]}" - else - SETUP_CSI="nfs" - fi - ;; - esac -done - -# Actions run in command line order, one ate-setup invocation each, so a single -# line can still ask for several of them. -while [[ "$#" -gt 0 ]]; do - case $1 in - # Captured in the pre-scan above; matched here only so the `*)` branch does - # not reject them, and so a separated value is consumed with its flag. - --atenet-dataplane|--podcert-workers-per-signer|--rollout-timeout|--otlp-endpoint) shift ;; - --cluster-size|--ingress-auth-mode|--ingress-allowed-spiffe-ids) shift ;; - --experimental-additional-egress-extproc-service) shift ;; - --credential-provider) shift ;; - --ateapi-log-level) shift ;; - --benchmark-worker-count|--benchmark-sandbox-class|--benchmark-actor-memory) shift ;; - --atenet-dataplane=*|--podcert-workers-per-signer=*|--rollout-timeout=*|--otlp-endpoint=*) ;; - --fake-data-plane|--fake-data-plane=*) ;; - --cluster-size=*|--cordon-control-plane|--cordon-control-plane=*|--ingress-auth-mode=*) ;; - --ingress-allowed-spiffe-ids=*) ;; - --experimental-additional-egress-extproc-service=*) ;; - --credential-provider=*) ;; - --ateapi-log-level=*) ;; - --benchmark-worker-count=*|--benchmark-sandbox-class=*|--benchmark-actor-memory=*) ;; - --preview) shift ;; - --preview=*) ;; - - --deploy-ate-system) ate_setup deploy ate-system "--setup-csi=${SETUP_CSI}" ;; - --setup-csi=*) ate_setup setup csi "${SETUP_CSI}" ;; - --setup-csi) - if [[ "$#" -gt 1 && "$2" != --* ]]; then - shift - fi - ate_setup setup csi "${SETUP_CSI}" - ;; - --delete-ate-system) ate_setup delete ate-system ;; - --delete-all) ate_setup delete all ;; - - --deploy-atelet) ate_setup deploy atelet ;; - --deploy-ate-apiserver) ate_setup deploy apiserver ;; - - --deploy-atenet) ate_setup deploy atenet ;; - --delete-atenet) ate_setup delete atenet ;; - - --deploy-benchmarks) ate_setup deploy benchmarks ${BENCHMARK_FLAGS[@]+"${BENCHMARK_FLAGS[@]}"} ;; - --delete-benchmarks) ate_setup delete benchmarks ${BENCHMARK_FLAGS[@]+"${BENCHMARK_FLAGS[@]}"} ;; - - --create-jwt-authority-pool-secret) ate_setup create jwt-authority-pool ;; - --create-actor-id-ca-pool-secret) ate_setup create actor-id-ca-pool ;; - --create-actor-id-ca-certs-secret) ate_setup create actor-id-ca-certs ;; - --create-egress-mitm-ca-pool-secret) ate_setup create egress-mitm-ca-pool ;; - --create-podcertificate-controller-cas) ate_setup create podcertificate-controller-cas ;; - --create-api-server-env-vars) ate_setup create api-server-env-vars ;; - --create-api-authentication-config) ate_setup create api-authentication-config ;; - --create-api-config) ate_setup create api-config ;; - - # The one demo flag that is not just a demo name. - --deploy-demo-counter-with-external-volume) - ate_setup deploy demo counter --with-external-volume \ - "--storage-class=$(counter_storage_class)" - ;; - --deploy-demo-*) run_demo deploy "$1" ;; - --delete-demo-*) run_demo delete "$1" ;; - - *) - # Invalid option, should usage and exit with an error. - echo "Error: unknown option: $1" >&2 - echo "" - usage - exit 1 - ;; - esac - shift -done +go run ./cmd/ate-setup "$@" diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 875b98a702..eff1360f99 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -82,10 +82,10 @@ log() { log "Deploying the ate control plane (--deploy-ate-system)..." if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh deploy ate-system else # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh depoy ate-system fi # --- 2. install micro-VM deps (assets + cluster-wide SandboxConfig) -------- @@ -105,9 +105,9 @@ fi # dispatch through install-ate.sh like step 1. log "Deploying the counter-microvm demo (--deploy-demo-counter-microvm)..." if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-demo-counter-microvm + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh deploy demo counter-microvm else - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-demo-counter-microvm + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh deploy demo counter-microvm fi log "Demo applied. Next steps:" diff --git a/tools/setup-gcp/cloud-sql.md b/tools/setup-gcp/cloud-sql.md index 578500fdab..561e7093e9 100644 --- a/tools/setup-gcp/cloud-sql.md +++ b/tools/setup-gcp/cloud-sql.md @@ -121,8 +121,8 @@ users need a separate migration of ownership and grants. ```sh export ATE_API_POSTGRES_CLOUDSQL_INSTANCE=:: export ATE_API_POSTGRES_CLOUDSQL_GSA=ate-api-server@.iam.gserviceaccount.com -./hack/install-ate.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' -# Existing installation: --deploy-ate-apiserver instead of --deploy-ate-system +go run ./cmd/ate-setup deploy ate-system --credential-provider='{"name":"k8s.io"}' +# Existing installation: deploy apiserver instead of deploy ate-system ``` What this does differently from a plain install: From 46131a6470fd5902b334812cf836bbe9a0e1513d Mon Sep 17 00:00:00 2001 From: hajiler Date: Tue, 20 Oct 2026 17:39:23 +0000 Subject: [PATCH 08/63] ate-setup: front the NFS CSI controller with an Envoy mTLS proxy (#2937) Fixes #2917. ## Problem `ate-setup setup csi nfs` (and `--setup-csi=nfs|both`) patched the `csi-nfs-controller` Deployment with a privileged `socat` sidecar that exposed the driver's controller socket as plaintext, unauthenticated TCP: `csi-nfs-controller.kube-system.svc:50052`, and `:10000` since the pod is host-networked. Anything with cluster network access could call `CreateVolume` / `DeleteVolume`. ## Change - **Envoy mTLS proxy for the NFS controller**, matching the hostpath driver and `docs/csi-deployment.md`: the sidecar terminates TLS with a serving certificate from `servicednssigner`, requires a client certificate, and accepts only the `ate-api-server` pod identity SPIFFE ID (rendered from the install namespace via `installdefaults.SPIFFEID`). It proxies raw TCP to `/csi/csi.sock`, so gRPC streams are passed end to end with no HTTP route timeout that could cut a slow `CreateVolume` short. No admin port, since the pod runs on the host network. - **`CSIDriverConfig` `nfs.csi.k8s.io`** now sets `tls: {enabled, usePodIdentity, serverName: csi-nfs-controller.kube-system.svc}`. The `.svc` form is the only DNS name `servicednssigner` issues. - **Ordering**: the Service and the Envoy ConfigMap are created before the driver, because the signer refuses to issue a certificate for a pod no Service selects yet (same rule the hostpath step already follows). - **Not privileged**: the proxy runs as root because the driver creates the socket mode 0755, but with `allowPrivilegeEscalation: false`, all capabilities dropped and a read-only root filesystem. The hostpath driver's Envoy loses `privileged: true` the same way. - **Upgrade path**: the patch carries a `$patch: delete` for the `socat` container. The upstream manifest is server-side applied, which leaves containers another field manager added in place, so re-running setup on a cluster that still has the old sidecar removes it. - **Readiness probe** on the proxy, so the Deployment rollout and the Service only count the pod once Envoy has its certificate and is accepting connections. - `hack/setup-csi-nfs-kind.sh` makes the same changes; `docs/csi-deployment.md` gets the `.svc` server name (the example previously named `.svc.cluster.local`, which fails verification), the Service-before-pod note, and a non-privileged security context in the example. ## Verification on kind `ate-setup setup csi both` and `hack/setup-csi-nfs-kind.sh` were run against a kind cluster, including over an install that still had the `socat` sidecar: - `csi-nfs-controller` runs 6/6 with `envoy` in place of `socat`, security context `{runAsUser: 0, allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}}`; the hostpath proxy runs with the same context. - Issued serving certificate SAN: `DNS:csi-nfs-controller.kube-system.svc`. - TLS client without a certificate: `tlsv13 alert certificate required`. Client with a certificate for a different service account: `alert certificate unknown`. Plaintext: no bytes returned. - A gRPC client holding the `ate-api-server` pod identity, using the same TLS settings as `internal/volume/csi` (TLS 1.3, ALPN h2, servicedns trust bundle, server name from the `CSIDriverConfig`), completed `GetPluginInfo`, `ControllerGetCapabilities`, `CreateVolume` and `DeleteVolume` through the proxy. The same client with server name `csi-nfs-controller.kube-system.svc.cluster.local` fails verification, which is why the doc example changed. Not run here: the external-volume e2e suite. The local Docker VM lost outbound network partway through, so atelet could not fetch the gVisor release and no worker could host the probe actor; the suite failed before reaching volumes. CI runs it. --- cmd/ate-setup/internal/steps/csi.go | 249 ++++++++++++++---- cmd/ate-setup/internal/steps/csi_test.go | 95 ++++++- docs/csi-deployment.md | 17 +- docs/csi-volumes.md | 4 +- hack/setup-csi-nfs-kind.sh | 188 +++++++++++-- .../deploy/csi-hostpath-testing.yaml | 15 +- 6 files changed, 481 insertions(+), 87 deletions(-) diff --git a/cmd/ate-setup/internal/steps/csi.go b/cmd/ate-setup/internal/steps/csi.go index 1fccb87793..b05d71842f 100644 --- a/cmd/ate-setup/internal/steps/csi.go +++ b/cmd/ate-setup/internal/steps/csi.go @@ -32,6 +32,7 @@ import ( "go.ate.dev/substrate/cmd/ate-setup/internal/kube" "go.ate.dev/substrate/cmd/ate-setup/internal/kustomize" "go.ate.dev/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/internal/installdefaults" "go.ate.dev/substrate/internal/nodepath" ) @@ -40,10 +41,10 @@ import ( // they mount inside it are visible to atelet. const workerHostDir = nodepath.BasePath -// csiHostpathControllerService fronts the hostpath driver's socat sidecar, -// which bridges the TCP endpoint atelet dials to the driver's unix socket. The -// CSIDriverConfig applied later names this Service in both its -// controllerEndpoint and its TLS server name. +// csiHostpathControllerService fronts the hostpath driver's Envoy sidecar, +// which terminates mTLS on the TCP endpoint ateapi dials and forwards to the +// driver's unix socket. The CSIDriverConfig applied later names this Service +// in both its controllerEndpoint and its TLS server name. const csiHostpathControllerService = ` apiVersion: v1 kind: Service @@ -203,8 +204,8 @@ func (e *Env) setupCSIHostpath(ctx context.Context) error { log.Infof("Using Kind node: %s", node) cleanupKindWorkerDir(node) - // Before the driver, not after: atelet talks to the controller over TCP, - // the socat sidecar bridges that to the driver's unix socket, and + // Before the driver, not after: ateapi talks to the controller over TCP, + // the Envoy sidecar bridges that to the driver's unix socket, and // servicednssigner issues the sidecar's serving certificate from the // Service. A sidecar that starts before the Service exists has no // certificate to be issued against and never becomes ready. @@ -292,6 +293,184 @@ spec: return e.Kube.RolloutRestart(ctx, e.Namespace(), "atelet", time.Now()) } +// csiNFSControllerService fronts the Envoy sidecar csiNFSControllerPatch adds to the NFS controller pod. +const csiNFSControllerService = ` +apiVersion: v1 +kind: Service +metadata: + name: csi-nfs-controller + namespace: kube-system +spec: + selector: + app: csi-nfs-controller + ports: + - port: 50052 + targetPort: 10000 + name: grpc +` + +const csiNFSEnvoyConfigMapName = "csi-nfs-envoy-config" + +// csiNFSEnvoyConfigMap renders the NFS controller's Envoy config: an mTLS listener accepting only apiServerSPIFFEID, proxied to the driver's unix socket. +func csiNFSEnvoyConfigMap(apiServerSPIFFEID string) string { + return strings.ReplaceAll(csiNFSEnvoyConfigMapTemplate, "{{API_SERVER_SPIFFE_ID}}", apiServerSPIFFEID) +} + +const csiNFSEnvoyConfigMapTemplate = ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: ` + csiNFSEnvoyConfigMapName + ` + namespace: kube-system +data: + envoy.yaml: | + node: + id: csi-nfs-envoy + cluster: csi-nfs-envoy + static_resources: + listeners: + - name: csi_tls_listener + address: + socket_address: { address: 0.0.0.0, port_value: 10000 } + filter_chains: + - transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + require_client_certificate: true + common_tls_context: + alpn_protocols: + - h2 + tls_certificate_sds_secret_configs: + - name: servicedns_serving_cert + sds_config: + resource_api_version: V3 + path_config_source: + path: /etc/envoy/sds-servicedns-cert.yaml + combined_validation_context: + default_validation_context: + match_typed_subject_alt_names: + - san_type: URI + matcher: + exact: "{{API_SERVER_SPIFFE_ID}}" + validation_context_sds_secret_config: + name: podidentity_validation_context + sds_config: + resource_api_version: V3 + path_config_source: + path: /etc/envoy/sds-podidentity-validation.yaml + filters: + - name: envoy.filters.network.tcp_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + stat_prefix: csi_proxy + cluster: csi_unix_socket + clusters: + - name: csi_unix_socket + connect_timeout: 0.25s + type: STATIC + load_assignment: + cluster_name: csi_unix_socket + endpoints: + - lb_endpoints: + - endpoint: + address: + pipe: + path: /csi/csi.sock + sds-servicedns-cert.yaml: | + resources: + - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret + name: servicedns_serving_cert + tls_certificate: + certificate_chain: { filename: /run/servicedns/credential-bundle.pem } + private_key: { filename: /run/servicedns/credential-bundle.pem } + watched_directory: { path: /run/servicedns } + sds-podidentity-validation.yaml: | + resources: + - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret + name: podidentity_validation_context + validation_context: + trusted_ca: { filename: /run/podidentity-ca/trust-bundle.pem } + watched_directory: { path: /run/podidentity-ca } +` + +// csiNFSControllerPatch adds the Envoy TLS proxy to the upstream csi-nfs-controller Deployment and removes the socat sidecar earlier installs added. +const csiNFSControllerPatch = ` +spec: + template: + spec: + containers: + - $patch: delete + name: socat + - name: envoy + image: envoyproxy/envoy:v1.34-latest + command: + - /usr/local/bin/envoy + args: + - -c + - /etc/envoy/envoy.yaml + readinessProbe: + tcpSocket: + port: 10000 + periodSeconds: 5 + securityContext: + # Root because the driver creates csi.sock mode 0755; nothing else is needed. + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - mountPath: /etc/envoy + name: envoy-config + readOnly: true + - mountPath: /csi + name: socket-dir + - mountPath: /run/servicedns + name: servicedns-certs + readOnly: true + - mountPath: /run/podidentity-ca + name: podidentity-ca + readOnly: true + volumes: + - name: envoy-config + configMap: + name: ` + csiNFSEnvoyConfigMapName + ` + - name: servicedns-certs + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: podidentity-ca + projected: + sources: + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +` + +// csiNFSDriverConfig registers the NFS driver; its serverName is the one DNS name servicednssigner issues, ..svc. +const csiNFSDriverConfig = ` +apiVersion: ate.dev/v1alpha1 +kind: CSIDriverConfig +metadata: + name: nfs.csi.k8s.io +spec: + driverName: nfs.csi.k8s.io + controllerEndpoint: tcp://csi-nfs-controller.kube-system.svc.cluster.local:50052 + nodeSocketOverride: unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock + tls: + enabled: true + usePodIdentity: true + serverName: csi-nfs-controller.kube-system.svc +` + func (e *Env) setupCSINFS(ctx context.Context) error { log.Step("setup_csi_nfs") @@ -306,6 +485,15 @@ func (e *Env) setupCSINFS(ctx context.Context) error { return err } + // Before the driver: servicednssigner only issues the sidecar's certificate once a Service selects its pod. + log.Infof("Exposing the CSI NFS controller over a TLS Service...") + if err := e.applyInline(ctx, csiNFSControllerService); err != nil { + return err + } + if err := e.applyInline(ctx, csiNFSEnvoyConfigMap(installdefaults.APIServerSPIFFEID(e.Namespace()))); err != nil { + return err + } + log.Infof("Deploying the CSI NFS driver...") for _, name := range []string{ "rbac-csi-nfs.yaml", @@ -344,24 +532,8 @@ spec: return fmt.Errorf("while patching the csi-nfs-node DaemonSet: %w", err) } - log.Infof("Patching the CSI NFS controller Deployment...") - const controllerPatch = ` -spec: - template: - spec: - containers: - - name: socat - image: docker.io/alpine/socat:1.7.4.3-r0 - args: - - tcp-listen:10000,fork,reuseaddr - - unix-connect:/csi/csi.sock - securityContext: - privileged: true - volumeMounts: - - mountPath: /csi - name: socket-dir -` - controllerPatchJSON, err := yaml.YAMLToJSON([]byte(controllerPatch)) + log.Infof("Patching the CSI NFS controller Deployment with the Envoy TLS proxy...") + controllerPatchJSON, err := yaml.YAMLToJSON([]byte(csiNFSControllerPatch)) if err != nil { return fmt.Errorf("while converting csi-nfs-controller patch to JSON: %w", err) } @@ -370,24 +542,6 @@ spec: return fmt.Errorf("while patching the csi-nfs-controller Deployment: %w", err) } - log.Infof("Exposing the CSI NFS controller over a Service...") - if err := e.applyInline(ctx, ` -apiVersion: v1 -kind: Service -metadata: - name: csi-nfs-controller - namespace: kube-system -spec: - selector: - app: csi-nfs-controller - ports: - - port: 50052 - targetPort: 10000 - name: grpc -`); err != nil { - return err - } - // The driver resolves the server's DNS name at provisioning time, so the // StorageClass can be created before the NFS server has an address. log.Infof("Creating the csi-nfs-sc StorageClass...") @@ -407,16 +561,7 @@ spec: } log.Infof("Creating the nfs.csi.k8s.io CSIDriverConfig...") - if err := e.applyInline(ctx, ` -apiVersion: ate.dev/v1alpha1 -kind: CSIDriverConfig -metadata: - name: nfs.csi.k8s.io -spec: - driverName: nfs.csi.k8s.io - controllerEndpoint: tcp://csi-nfs-controller.kube-system.svc.cluster.local:50052 - nodeSocketOverride: unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock -`); err != nil { + if err := e.applyInline(ctx, csiNFSDriverConfig); err != nil { return err } diff --git a/cmd/ate-setup/internal/steps/csi_test.go b/cmd/ate-setup/internal/steps/csi_test.go index e558bfb9a2..695593f140 100644 --- a/cmd/ate-setup/internal/steps/csi_test.go +++ b/cmd/ate-setup/internal/steps/csi_test.go @@ -18,10 +18,13 @@ import ( "context" "os" "path/filepath" + "strconv" "strings" "testing" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" "go.ate.dev/substrate/cmd/ate-setup/internal/config" "go.ate.dev/substrate/cmd/ate-setup/internal/kube" @@ -103,7 +106,7 @@ func functionBody(t *testing.T, file, signature string) string { return body[:end] } -// servicednssigner issues the socat sidecar's serving certificate from the +// servicednssigner issues the Envoy sidecar's serving certificate from the // csi-hostpath-controller Service, so a driver deployed before that Service // exists has nothing to be issued against and never becomes ready. The shell // installer created the Service first and said why. @@ -122,7 +125,7 @@ func TestTheHostpathControllerServiceIsCreatedBeforeTheDriver(t *testing.T) { t.Fatal("setupCSIHostpath no longer applies the driver bundle with ApplyTolerant") case service > driver: t.Error("setupCSIHostpath deploys the hostpath driver before creating the " + - "csi-hostpath-controller Service; the socat sidecar cannot be issued a " + + "csi-hostpath-controller Service; the Envoy sidecar cannot be issued a " + "certificate until the Service exists") } } @@ -197,3 +200,91 @@ func TestCSIDriverSpellings(t *testing.T) { }) } } + +// decodeYAML unmarshals a manifest defined in csi.go. +func decodeYAML(t *testing.T, name, src string, into any) { + t.Helper() + if err := yaml.Unmarshal([]byte(src), into); err != nil { + t.Fatalf("%s does not parse: %v", name, err) + } +} + +// ateapi verifies tls.serverName, which must be the ..svc name servicednssigner issues. +func TestTheNFSDriverConfigEnablesTLSAgainstTheControllerService(t *testing.T) { + var svc corev1.Service + decodeYAML(t, "csiNFSControllerService", csiNFSControllerService, &svc) + var cfg struct { + Spec struct { + ControllerEndpoint string `json:"controllerEndpoint"` + TLS *struct { + Enabled bool `json:"enabled"` + UsePodIdentity bool `json:"usePodIdentity"` + ServerName string `json:"serverName"` + } `json:"tls"` + } `json:"spec"` + } + decodeYAML(t, "csiNFSDriverConfig", csiNFSDriverConfig, &cfg) + + if cfg.Spec.TLS == nil || !cfg.Spec.TLS.Enabled || !cfg.Spec.TLS.UsePodIdentity { + t.Fatalf("csiNFSDriverConfig tls = %+v, want enabled with pod identity", cfg.Spec.TLS) + } + if want := svc.Name + "." + svc.Namespace + ".svc"; cfg.Spec.TLS.ServerName != want { + t.Errorf("tls.serverName = %q, want %q, the name servicednssigner issues", cfg.Spec.TLS.ServerName, want) + } + if want := "tcp://" + svc.Name + "." + svc.Namespace + ".svc.cluster.local:" + strconv.Itoa(int(svc.Spec.Ports[0].Port)); cfg.Spec.ControllerEndpoint != want { + t.Errorf("controllerEndpoint = %q, want %q", cfg.Spec.ControllerEndpoint, want) + } +} + +// hack/setup-csi-nfs-kind.sh is the same install by hand and must keep the same security decisions. +func TestTheNFSShellSetupMatchesTheGoSetup(t *testing.T) { + script, err := os.ReadFile(filepath.Join(repoRoot(t), "hack", "setup-csi-nfs-kind.sh")) + if err != nil { + t.Fatalf("reading the shell setup: %v", err) + } + src := string(script) + + for _, forbidden := range []string{"socat:", "privileged: true"} { + if strings.Contains(src, forbidden) { + t.Errorf("hack/setup-csi-nfs-kind.sh still contains %q", forbidden) + } + } + for _, required := range []string{ + "require_client_certificate: true", + "serverName: csi-nfs-controller.kube-system.svc\n", + "usePodIdentity: true", + "/sa/ate-api-server", + } { + if !strings.Contains(src, required) { + t.Errorf("hack/setup-csi-nfs-kind.sh no longer contains %q", required) + } + } + if service, driver := strings.Index(src, "name: csi-nfs-controller\n namespace: kube-system"), strings.Index(src, "csi-nfs-controller.yaml"); service < 0 || driver < 0 || service > driver { + t.Error("hack/setup-csi-nfs-kind.sh deploys the NFS driver before creating the csi-nfs-controller Service") + } +} + +// The hostpath proxy needs no more than root to reach the driver's socket. +func TestTheHostpathProxyIsNotPrivileged(t *testing.T) { + objs, err := kube.LoadPath(filepath.Join(repoRoot(t), "hack", "third_party", "csi-driver-host-path", "deploy", "csi-hostpath-testing.yaml")) + if err != nil { + t.Fatalf("loading the hostpath proxy manifest: %v", err) + } + var found bool + for _, obj := range objs { + if obj.GetKind() != "StatefulSet" { + continue + } + containers, _, _ := unstructured.NestedSlice(obj.Object, "spec", "template", "spec", "containers") + for _, c := range containers { + c := c.(map[string]any) + found = true + if privileged, _, _ := unstructured.NestedBool(c, "securityContext", "privileged"); privileged { + t.Errorf("%s container %q is privileged", kube.Describe(obj), c["name"]) + } + } + } + if !found { + t.Fatal("csi-hostpath-testing.yaml no longer holds a proxy StatefulSet") + } +} diff --git a/docs/csi-deployment.md b/docs/csi-deployment.md index 11172207cc..21c2382295 100644 --- a/docs/csi-deployment.md +++ b/docs/csi-deployment.md @@ -32,6 +32,8 @@ Standard CSI controller deployments can be made network-accessible and secured i Create a Kubernetes `Service` targeting the CSI Controller pods. This provides a stable DNS name and load balancing across controller replicas. +Create the `Service` before the controller pods when the proxy's serving certificate comes from Substrate's `servicedns.podcert.ate.dev/identity` signer: it issues the certificate for the `..svc` names of the `Service`s selecting the pod, and refuses to issue one for a pod no `Service` selects yet. The `Service` name is also what `spec.tls.serverName` in the `CSIDriverConfig` must be. + ### Securing with mTLS and Pod Identity When exposing the CSI Controller over the network, communication should be secured with mutual TLS (mTLS). In the [`CSIDriverConfig`](csi-volumes.md#2-dynamic-csi-driver-discovery-csidriverconfig) resource, configure the following fields under `spec.tls`: @@ -204,9 +206,19 @@ spec: # Envoy proxy sidecar terminating mTLS and forwarding to /csi/csi.sock - name: envoy-proxy image: envoyproxy/envoy:v1.34-latest + command: + - "/usr/local/bin/envoy" args: - "-c" - "/etc/envoy/envoy.yaml" + # The proxy needs no privilege, only to be able to connect to the + # driver's socket, which the nfs container creates root-only. + securityContext: + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] volumeMounts: - name: socket-dir mountPath: /csi @@ -273,7 +285,10 @@ spec: tls: enabled: true usePodIdentity: true - serverName: csi-nfs-controller.kube-system.svc.cluster.local + # The one DNS name servicednssigner puts in the proxy's certificate is + # ..svc, so that, not the cluster.local form the + # endpoint dials, is the server name to verify. + serverName: csi-nfs-controller.kube-system.svc ``` --- diff --git a/docs/csi-volumes.md b/docs/csi-volumes.md index 0ae8acb403..f9052dbd45 100644 --- a/docs/csi-volumes.md +++ b/docs/csi-volumes.md @@ -41,7 +41,7 @@ spec: tls: enabled: true usePodIdentity: true - serverName: csi-nfs-controller.kube-system.svc.cluster.local + serverName: csi-nfs-controller.kube-system.svc ``` ### Specification (`CSIDriverConfigSpec`) @@ -143,7 +143,7 @@ spec: tls: enabled: true usePodIdentity: true - serverName: csi-nfs-controller.kube-system.svc.cluster.local + serverName: csi-nfs-controller.kube-system.svc ``` ### Step 3: Define WorkerPool and ActorTemplate diff --git a/hack/setup-csi-nfs-kind.sh b/hack/setup-csi-nfs-kind.sh index 624d824ad9..5451da5342 100755 --- a/hack/setup-csi-nfs-kind.sh +++ b/hack/setup-csi-nfs-kind.sh @@ -31,14 +31,116 @@ echo "Deploying NFS server..." kubectl apply -f "${ROOT}/hack/csi/nfs-server.yaml" -# 2. Deploy CSI NFS Driver +# 2. Expose the CSI NFS Controller over a TLS Service, and render the Envoy +# proxy configuration it fronts. Both before the driver: servicednssigner +# issues the proxy's serving certificate from the Services selecting its pod, +# and refuses to issue one for a pod no Service selects yet. +echo "Exposing CSI NFS Controller over Service..." +cat </dev/null 2>&1 || true +kubectl rollout restart daemonset/atelet -n "${ATE_NAMESPACE}" >/dev/null 2>&1 || true echo "CSI NFS setup complete!" diff --git a/hack/third_party/csi-driver-host-path/deploy/csi-hostpath-testing.yaml b/hack/third_party/csi-driver-host-path/deploy/csi-hostpath-testing.yaml index 1a34c06dd7..5a72d350c9 100644 --- a/hack/third_party/csi-driver-host-path/deploy/csi-hostpath-testing.yaml +++ b/hack/third_party/csi-driver-host-path/deploy/csi-hostpath-testing.yaml @@ -162,10 +162,17 @@ spec: - "-c" - "/etc/envoy/envoy.yaml" securityContext: - # This is necessary only for systems with SELinux, where - # non-privileged sidecar containers cannot access unix domain socket - # created by privileged CSI driver container. - privileged: true + # Root, because the driver creates csi.sock mode 0755 and nothing + # else can connect to it; not privileged, because nothing here + # needs more than that. (On SELinux systems an unprivileged sidecar + # cannot reach a socket created by a privileged container, but the + # hostpath driver is Kind only.) + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL volumeMounts: - mountPath: /etc/envoy name: envoy-config From 11afdf6499a0abe9986fe1b654450c6185f82744 Mon Sep 17 00:00:00 2001 From: Dmitry Berkovich <54346214+dberkov@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:53:41 +0000 Subject: [PATCH 09/63] atelet: serve shared snapshot restores from the node-local file cache (#2796) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces #1551, which was four commits from September that no longer applied: the restore source message, the download path and the module path all changed underneath it. Same design, ported onto today's main, after #2756 and #2889, in four commits, plus three from review: the sharing derived from the snapshot's owner, the gVisor worker evicting a shared file's pages before unlinking it, and the manifest read through the cache. Milestone M2 of the node-local artifact cache from #690: an actor restoring from a shared snapshot, the one a tag holds and every actor created from it starts on, gets its files from a node-level cache instead of its own download. **Sharing is the control plane's knowledge.** `ObjectStorage` gains `SnapshotSharing`, the property rather than the snapshot's kind, so new shared snapshot kinds need no atelet change. ate-api-server sets it on Restore from the snapshot's recorded owner: SHARED for a tag's snapshot, borrowed at creation and immutable, PRIVATE for the actor's own, which its next suspend replaces, and for an unrecorded owner; the local-copy fallback object carries it too. Neither side infers it from the URI. **The cache.** A `filecache.Store` under `/var/lib/ate/snapshot-cache`, behind `--snapshot-cache-dir` (empty disables), `--snapshot-cache-min-age` and `--snapshot-cache-fetch-timeout`, opened at startup with crash debris swept before atelet serves. The directory must be under the node state dir: the object-store plugin writes only there, so a cache elsewhere could never be filled, and startup refuses it. Every flag is validated whether or not the cache is enabled, and the help text says which are read only while it is. The image cache's base-path check moved to `nodepath.UnderBasePath`, which both caches use. **Eviction.** The cache runs on the eviction loop every file cache on the node shares, `cmd/atelet/internal/filecachegc` from #2889: `--snapshot-cache-gc-period`, `--snapshot-cache-max-bytes` and `--snapshot-cache-gc-dry-run`, a budget with a 90% low water, least recently used first, never an entry a restore still links. Its size, evictions and passes appear under the existing `ate.filecache.*` instruments as a second value of `ate.filecache.name`, `snapshot`; no new instrument and no new loop. Every flag is validated whether or not the cache is enabled, as the other flags in atelet are, and the help text says which ones are read only while it is; a disabled cache starts no loop and reports no size. **The restore path.** A restore whose durable snapshot is declared SHARED stages each file through the cache; concurrent restores of one snapshot share a single download, later ones read disk. How a hit is served depends on the sandbox class the request's runtime resolved to. ateworker-gvisor gets hard links. Once the restore's page loading is done the worker evicts the memory and filesystem images from the page cache and unlinks them from its restore dir, keeping only the small descriptors linked: a file with another link is treated like a preserved restore dir, since unlinking it would free nothing and would leave its pages charged to the actor's cgroup, the case #1917 fixed. So a link protects an image's entry through the restore and the min age, not for the actor's life; an entry evicted later is still read to the end through the descriptors runsc holds open, and the next restore reads the images from local disk. ateworker-microvm gets private hole-preserving copies, not because a link would corrupt the cache, it replaces `config.json` by rename and its delta merge copies when the inode is shared, but because a link would cost more than it saves: an on-demand restore keeps `memory-ranges` open for the VM's life, which would pin the entry that long and turn every suspend's in-place merge into a full copy, and the worker's page-cache eviction would drop the cached pages for every other restore; #2948 tracks what links would take. EXDEV on a link, from a cache bind-mounted apart from the restore dirs, degrades to a copy, warned once per process. The manifest of a shared snapshot is read through the cache too, so a restore that finds the snapshot on the node makes no request to object storage; a private snapshot's manifest is read fresh, as the commit marker it is. The cache key is the object's address without its query, so a signed URL's credential reaches neither the key, the cache's errors, `meta.json` nor the logs; and a restore that fails after staging removes its restore-state dir, so the links do not hold the cache's entries until the next lifecycle call. The fetch goes through the object-store plugin as the direct path does, one file into a scratch dir on the cache's filesystem; because the cache then writes the restore dir by path rather than through the plugin's `os.Root`, a symlink waiting there fails the restore instead of being followed. **Not here.** Private snapshots stay uncached by design. Links for ateworker-microvm are #2948, a worker change. The open-and-sweep code the two file caches share in shape is still duplicated, since the snapshot cache can be disabled and degrades to copies across mounts where the asset cache refuses to start. **Tests.** ate-api functional: a golden restore declares SHARED, the actor's own snapshot PRIVATE. ate-api unit: the detach-and-resume test from #2852 compares the whole Restore snapshot, so its fixture now puts the snapshot under the actor's own UID, and it expects PRIVATE for the downloaded copy and nothing for a local one. atelet: flag validation, enabled or not, including the eviction flags, the fetch timeout, and a directory outside the node state dir; two shared manifest reads download once and a private one reads fresh; open disabled, creating its root, sweeping debris; the sharing gate and the class modes; through the plugin-backed test atelet, two link restores share one download and one inode and the file is read-only, copy mode gives private inodes and survives a consumer's mutation, a nil cache downloads, a missing file's error reaches the caller with nothing left behind, a plugin error through the cached modes keeps its code and leaks no credential from a signed URI, a restore that fails at the worker after staging leaves no file in restore-state and no link on the cache's entry, four concurrent restores download once, a file in the checkpoint subdirectory lands there, and symlinks in place of the file or the subdirectory fail; on Linux, a restore dir on another mount gets a copy with no second download and one warning for two restores. ateworker-gvisor, on Linux: staged images hard-linked elsewhere are evicted after the restore wait and only then unlinked, with the other links intact. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- .../controlapi/functionaltest/actor_test.go | 10 + .../internal/controlapi/workflow_resume.go | 24 +- .../controlapi/workflow_resume_detach_test.go | 8 +- cmd/atelet/imagegc.go | 20 +- cmd/atelet/imagegc_test.go | 25 - .../apivalidation/zz_generated.validation.go | 31 ++ cmd/atelet/internal/ateletpath/ateletpath.go | 8 + cmd/atelet/lifecycle_test.go | 87 ++++ cmd/atelet/main.go | 51 +- cmd/atelet/main_test.go | 2 +- cmd/atelet/snapshot_transfer_test.go | 22 +- cmd/atelet/snapshotcache.go | 305 ++++++++++++ cmd/atelet/snapshotcache_linux_test.go | 84 ++++ cmd/atelet/snapshotcache_test.go | 464 ++++++++++++++++++ cmd/ateworker-gvisor/ckptcgroup.go | 34 +- cmd/ateworker-gvisor/ckptcgroup_test.go | 66 +++ docs/metrics/registry/metrics.yaml | 16 +- docs/observability.md | 2 +- docs/sandbox-asset-cache.md | 7 +- internal/ateattr/ateattr.go | 3 + internal/nodepath/nodepath.go | 21 +- internal/nodepath/nodepath_test.go | 25 + internal/proto/ateletpb/atelet.pb.go | 388 +++++++++------ internal/proto/ateletpb/atelet.proto | 22 + 24 files changed, 1503 insertions(+), 222 deletions(-) create mode 100644 cmd/atelet/snapshotcache.go create mode 100644 cmd/atelet/snapshotcache_linux_test.go create mode 100644 cmd/atelet/snapshotcache_test.go diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index 597f21c8d3..ee43b6c876 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -3367,6 +3367,11 @@ func TestResumeActor_DataSnapshotIgnoresGolden(t *testing.T) { if got := restoreReq.GetSnapshot().GetObject().GetSnapshotUri(); got != actorSnapshotURI { t.Errorf("restore config snapshot uri = %q, want the actor's data snapshot %q", got, actorSnapshotURI) } + // The actor's own snapshot is replaced by its next suspend: atelet must + // not cache it. + if got := restoreReq.GetSnapshot().GetObject().GetSharing(); got != ateletpb.SnapshotSharing_SNAPSHOT_SHARING_PRIVATE { + t.Errorf("restore of the actor's own snapshot declared sharing %v, want PRIVATE", got) + } } // TestSuspendActor_ReplacedSnapshotReleaseFailure verifies that object storage @@ -7149,6 +7154,11 @@ func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { if got, want := goldenRestore.GetSnapshot().GetObject().GetSnapshotUri(), goldenSnapshotURI(t); got != want { t.Fatalf("golden-actor RestoreRequest.SnapshotUri = %q, want golden %q", got, want) } + // A borrowed golden snapshot is shared by every actor created from it, + // and the request must say so: atelet caches only declared-SHARED sources. + if got := goldenRestore.GetSnapshot().GetObject().GetSharing(); got != ateletpb.SnapshotSharing_SNAPSHOT_SHARING_SHARED { + t.Fatalf("golden-actor RestoreRequest.Sharing = %v, want SHARED", got) + } // Free worker-b and have it advertise rtA in restorable_runtimes while // keeping rtB as its default. Resuming an actor whose snapshot requires rtA diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index 09d081b42d..253e523be5 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -48,6 +48,11 @@ type resumeSnapshotSource struct { // snapshot, which takes precedence at restore). SnapshotURI resources.SnapshotURI Fidelity ateapipb.SnapshotFidelity + // Owner is who the durable snapshot belongs to: the actor, whose next + // suspend replaces it, or a tag, whose snapshot is immutable and read by + // every actor created from it. It decides what the restore declares as + // the snapshot's sharing. + Owner ateapipb.SnapshotOwner // TemplateReplaced is true when the external snapshot's recorded template // UID differs from the actor's current template. TemplateReplaced bool @@ -198,6 +203,7 @@ func resolveResumeSource(actorRef resources.ActorRef, actor *ateapipb.Actor, act src.SnapshotUUID = snap.GetUuid() src.Generation = snap.GetGeneration() src.Fidelity = snap.GetFidelity() + src.Owner = snap.GetOwner() src.TemplateReplaced = templateReplaced(snap, actorTemplate) } // A local restore also names the local snapshot's upload, so a URI that @@ -217,6 +223,18 @@ func templateReplaced(snap *ateapipb.Snapshot, tmpl *ateapipb.ActorTemplate) boo return capturedUnder != "" && capturedUnder != tmpl.GetMetadata().GetUid() } +// snapshotSharing is what a restore declares about a snapshot with the +// given owner. A tag's snapshot is immutable once published and read by +// every actor created from it, so atelet may serve its files from the +// node-local cache; the actor's own is replaced by its next suspend, so +// never, and so is one whose owner is not recorded. +func snapshotSharing(owner ateapipb.SnapshotOwner) ateletpb.SnapshotSharing { + if owner == ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG { + return ateletpb.SnapshotSharing_SNAPSHOT_SHARING_SHARED + } + return ateletpb.SnapshotSharing_SNAPSHOT_SHARING_PRIVATE +} + // ensureVolumesCreated provisions any actor volumes that have no storage // volume ID yet, persisting the resulting volume state (even when creation // partially failed, so progress is not lost) and returning the stored copy. @@ -822,7 +840,10 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou // report of the upload lands. atelet reads it only when it // holds no usable local copy, and the manifest, uploaded // last, tells a complete upload from a partial one. - Object: &ateletpb.ObjectStorage{SnapshotUri: localSnap.GetDurableSnapshot().GetObject().GetSnapshotUri()}, + Object: &ateletpb.ObjectStorage{ + SnapshotUri: localSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(), + Sharing: snapshotSharing(localSnap.GetOwner()), + }, }, SandboxRuntime: sandboxRuntime, ActorUid: actor.GetMetadata().Uid, @@ -863,6 +884,7 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou Fidelity: scope, Object: &ateletpb.ObjectStorage{ SnapshotUri: src.SnapshotURI.String(), + Sharing: snapshotSharing(src.Owner), }, }, SandboxRuntime: sandboxRuntime, diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go index 20ac9e9a95..4620082a66 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go @@ -422,7 +422,6 @@ func TestResumeActor_DetachesFromFullNode(t *testing.T) { a.Status.LastAssignedGeneration = 1 a.Status.Snapshots = []*ateapipb.Snapshot{proto.CloneOf(snap)} }) - w, atelet := newDetachTestWorkflow(t, ctx, st) addAteletOnNode(t, w, "node-2") atelet.detachState = detachedState(snap) @@ -452,7 +451,12 @@ func TestResumeActor_DetachesFromFullNode(t *testing.T) { SnapshotUuid: snap.GetUuid(), Generation: snap.GetGeneration(), Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, - Object: &ateletpb.ObjectStorage{SnapshotUri: snap.GetDurableSnapshot().GetObject().GetSnapshotUri()}, + // The actor's own snapshot, so private, whether atelet + // downloads it or restores the local copy. + Object: &ateletpb.ObjectStorage{ + SnapshotUri: snap.GetDurableSnapshot().GetObject().GetSnapshotUri(), + Sharing: ateletpb.SnapshotSharing_SNAPSHOT_SHARING_PRIVATE, + }, } if !proto.Equal(restore.GetSnapshot(), want) { t.Errorf("Restore snapshot = %v, want %v", restore.GetSnapshot(), want) diff --git a/cmd/atelet/imagegc.go b/cmd/atelet/imagegc.go index 0325c9b74a..6a08e2cf1b 100644 --- a/cmd/atelet/imagegc.go +++ b/cmd/atelet/imagegc.go @@ -29,10 +29,7 @@ import ( "errors" "fmt" "log/slog" - "os" - "path/filepath" "runtime/debug" - "strings" "time" "github.com/spf13/pflag" @@ -76,7 +73,10 @@ func validateImageCacheGCFlags() error { // future), making just-pulled layers evictable. return fmt.Errorf("--image-cache-min-age %v must be >= 0", *imageCacheMinAge) } - if imageCacheDirOutsideBasePath(*imageCacheDir) { + // Warn-worthy, not an error: a separate cache volume is legitimate + // (recommended for IOPS), but the watermarks then measure a different + // volume than actor state. + if !nodepath.UnderBasePath(*imageCacheDir) { slog.Warn("Image cache dir is outside the worker base path; its volume watermarks are measured separately from actor state", slog.String("image_cache_dir", *imageCacheDir), slog.String("actors_dir", nodepath.ActorsDir)) @@ -84,18 +84,6 @@ func validateImageCacheGCFlags() error { return nil } -// imageCacheDirOutsideBasePath reports whether the cache dir is outside -// the worker base path — the watermarks then measure a different volume -// than actor state. Warn-worthy, not an error: a separate cache volume is -// legitimate (recommended for IOPS). -func imageCacheDirOutsideBasePath(dir string) bool { - abs, err := filepath.Abs(dir) - if err != nil { - abs = filepath.Clean(dir) - } - return !strings.HasPrefix(abs, nodepath.BasePath+string(os.PathSeparator)) -} - // imageCacheGCTarget computes the bytes a pass should free: the larger // of the watermark shortfall (kubelet's formula — usage at highPct frees // down to lowPct) and the pool's overage past maxBytes, but never more diff --git a/cmd/atelet/imagegc_test.go b/cmd/atelet/imagegc_test.go index 5555768af1..a23f114a64 100644 --- a/cmd/atelet/imagegc_test.go +++ b/cmd/atelet/imagegc_test.go @@ -26,7 +26,6 @@ import ( "time" "go.ate.dev/substrate/internal/imagecache" - "go.ate.dev/substrate/internal/nodepath" ) func TestImageCacheGCTarget(t *testing.T) { @@ -154,30 +153,6 @@ func TestValidateImageCacheGCFlags(t *testing.T) { } } -func TestImageCacheDirOutsideBasePath(t *testing.T) { - cases := []struct { - name string - dir string - want bool - }{ - {"inside", filepath.Join(nodepath.BasePath, "image-cache"), false}, - {"inside with doubled separator", nodepath.BasePath + "//image-cache", false}, - {"inside via dot-dot", nodepath.BasePath + "/x/../image-cache", false}, - {"base path itself is not inside", nodepath.BasePath, true}, - {"sibling with the base path as name prefix", nodepath.BasePath + "-other/image-cache", true}, - {"outside", "/var/lib/elsewhere/image-cache", true}, - {"dot-dot escaping the base path", nodepath.BasePath + "/../elsewhere/image-cache", true}, - {"relative resolves against the cwd, not the base path", "image-cache", true}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if got := imageCacheDirOutsideBasePath(tc.dir); got != tc.want { - t.Errorf("imageCacheDirOutsideBasePath(%q) = %v, want %v", tc.dir, got, tc.want) - } - }) - } -} - type fakeGCStore struct { size int64 sizeErr error diff --git a/cmd/atelet/internal/apivalidation/zz_generated.validation.go b/cmd/atelet/internal/apivalidation/zz_generated.validation.go index d26f538dd1..c3183d5863 100644 --- a/cmd/atelet/internal/apivalidation/zz_generated.validation.go +++ b/cmd/atelet/internal/apivalidation/zz_generated.validation.go @@ -1711,6 +1711,37 @@ func Validate_ObjectStorage( errs = append(errs, fn(fldPath.Child("snapshot_uri"), &obj.SnapshotUri, oldVal, oldObj != nil)...) } + { // field ateletpb.ObjectStorage.Sharing + fn := func( + fldPath *field.Path, + obj, oldObj *ateletpb.SnapshotSharing, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 2); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.ObjectStorage) *ateletpb.SnapshotSharing { + return &oldObj.Sharing + }) + errs = append(errs, fn(fldPath.Child("sharing"), &obj.Sharing, oldVal, oldObj != nil)...) + } + return errs } diff --git a/cmd/atelet/internal/ateletpath/ateletpath.go b/cmd/atelet/internal/ateletpath/ateletpath.go index dc0dbec4b3..624f790cd5 100644 --- a/cmd/atelet/internal/ateletpath/ateletpath.go +++ b/cmd/atelet/internal/ateletpath/ateletpath.go @@ -55,6 +55,14 @@ var ( // runtime registered before a restart resolves the moment atelet serves // again. SandboxRuntimeIndexFile = filepath.Join(nodepath.BasePath, "sandbox-runtime-index.json") + + // SnapshotCacheDir is the node-local cache of shared snapshot files, the + // snapshots many actors are created from (see + // cmd/atelet/internal/filecache). It lives under BasePath because cache + // hits are served as hard links into the per-actor restore dirs under + // ActorsDir, and link(2) requires both ends on one mounted filesystem, + // the same mount, not merely the same disk. + SnapshotCacheDir = filepath.Join(nodepath.BasePath, "snapshot-cache") ) func ActorPath(actorUID string) string { diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index de49f5692d..fbb0451460 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -30,6 +30,7 @@ import ( "time" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/cmd/atelet/internal/filecache" "go.ate.dev/substrate/cmd/atelet/internal/snapshottracker" "go.ate.dev/substrate/cmd/atelet/internal/trustbundle" "go.ate.dev/substrate/internal/apierror" @@ -76,6 +77,9 @@ type fakeWorker struct { // restored holds the file contents staged into the restore dir by the // most recent RestoreWorkload. restored map[string]string + // restoreErr, if set, fails every RestoreWorkload after it recorded the + // request, as a worker that could not start the sandbox would. + restoreErr error // actorDirs records the ActorDirs each RPC arrived with, by RPC name. actorDirs map[string]*ateworkerpb.ActorDirs // preserveRestoreDir records the PreserveRestoreDir flag from the most @@ -136,6 +140,9 @@ func (f *fakeWorker) RestoreWorkload(_ context.Context, req *ateworkerpb.Restore f.recordActorDirs("RestoreWorkload", req.GetActorDirs()) f.recordFidelity("RestoreWorkload", req.GetFidelity()) f.preserveRestoreDir = req.GetPreserveRestoreDir() + if f.restoreErr != nil { + return nil, f.restoreErr + } dir := req.GetActorDirs().GetRestoreDir() f.restored = map[string]string{} for name := range f.snapshotFiles { @@ -897,6 +904,86 @@ func TestRestorePrefersLocalAndFallsBackToObjectStorage(t *testing.T) { } } +// TestRestoreFailureRemovesStagedSnapshotFiles pins what a shared restore +// that fails after its snapshot files were staged leaves behind: nothing in +// the restore-state directory, so the links into the snapshot cache do not +// hold its entries until a later lifecycle call resets the directory. +func TestRestoreFailureRemovesStagedSnapshotFiles(t *testing.T) { + useTempNodeDirs(t) + const ( + actorUID = "0f6c2b0e-1b7e-4b3f-9a6a-1d2c3e4f5a66" + snapshotName = "pause-snap-1" + ) + worker := &fakeWorker{snapshotFiles: map[string]string{"checkpoint.img": ""}, restoreErr: status.Error(codes.Internal, "sandbox would not start")} + serveFakeWorker(t, worker) + host := imageVolumeTestRegistry(t) + pinnedImage := pushTestImage(t, host+"/actor:v1", singleFileLayer(t, "bin/app", "app")) + release := newGVisorTestRelease(t) + sandboxRec := testSandboxRecord(release, pinnedImage) + manifestRec := sandboxAssetsRecord{ + SandboxClass: "gvisor", + PauseImage: pinnedImage, + Assets: map[string]assetEntry{gvisorAssetName: release.entry}, + SnapshotFiles: []string{"checkpoint.img"}, + } + s := newPluginHerder(t, &recordingObjectStorage{}) + s.workerDialer = newWorkerDialer(1) + s.imageCache = newImageVolumeStore(t) + s.anonGCSClient = fakeObjectStorage{data: release.tarball} + s.systemInfoVolumes = newSystemInfoVolumeRefresher(nil, nil) + // No min age, so the only thing standing between a staged entry and a + // pass is the restore's link. + cache, err := filecache.New(filepath.Join(t.TempDir(), "cache"), filecache.WithMinAge(0)) + if err != nil { + t.Fatal(err) + } + s.snapshotCache = cache + uri, err := resources.ParseSnapshotURI(pausedSnapshotURI) + if err != nil { + t.Fatal(err) + } + srcDir := t.TempDir() + writeLocalSnapshot(t, srcDir, manifestRec, map[string]string{"checkpoint.img": "shared-memory"}) + if err := s.uploadSnapshot(t.Context(), uri, srcDir, &manifestRec); err != nil { + t.Fatalf("uploadSnapshot: %v", err) + } + advertiseRuntime(t, s, sandboxRec) + + _, err = s.Restore(t.Context(), &ateletpb.RestoreRequest{ + Atespace: "ate-demo", + ActorName: "counter", + ActorUid: actorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + WorkerPodUid: "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f", + SandboxRuntime: runtimeFor(t, sandboxRec), + Spec: &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}}, + Snapshot: &ateletpb.Snapshot{ + SnapshotUuid: snapshotName, + Generation: 1, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateletpb.ObjectStorage{SnapshotUri: pausedSnapshotURI, Sharing: ateletpb.SnapshotSharing_SNAPSHOT_SHARING_SHARED}, + }, + }) + if status.Code(err) != codes.Internal { + t.Fatalf("Restore = %v, want the worker's Internal", err) + } + if worker.actorDirs["RestoreWorkload"] == nil { + t.Fatal("the worker was never asked: the failure must come after the snapshot files were staged") + } + if entries, err := os.ReadDir(ateletpath.RestoreStateDir(actorUID)); err == nil && len(entries) != 0 { + t.Errorf("restore-state holds %v after a failed restore, want nothing", entries) + } + stats, err := cache.EvictUnused(t.Context(), 1<<40, true) + if err != nil { + t.Fatal(err) + } + // Two entries, the manifest and checkpoint.img, both unlinked. + if stats.Retired != 2 || stats.SkippedLinked != 0 { + t.Errorf("after the failed restore a pass would retire %d entries and skip %d linked, want 2 and 0: the staged link must be gone", stats.Retired, stats.SkippedLinked) + } +} + // TestRestoreWithoutUsableLocalCopy checks that a Restore with no usable local // copy, naming a snapshot.object that was never uploaded, fails as NotFound // and wraps the local copy's problem. diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 24209b9c3e..5e69327244 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -282,6 +282,18 @@ func main() { if err := filecachegc.Start(ctx, ateattr.FileCacheSandboxAsset, sandboxAssetCache, sandboxAssetCacheGCFlags, fileCacheInstruments); err != nil { serverboot.Fatal(ctx, "Failed to start the sandbox asset cache GC", err) } + if err := validateSnapshotCacheFlags(); err != nil { + serverboot.Fatal(ctx, "Invalid snapshot cache flags", err) + } + snapshotCache, err := openSnapshotCache(ctx, *snapshotCacheDir, *snapshotCacheMinAge, *snapshotCacheFetchTimeout) + if err != nil { + serverboot.Fatal(ctx, "Failed to open snapshot cache", err) + } + if snapshotCache != nil { + if err := filecachegc.Start(ctx, ateattr.FileCacheSnapshot, snapshotCache, snapshotCacheGCFlags, fileCacheInstruments); err != nil { + serverboot.Fatal(ctx, "Failed to start the snapshot cache GC", err) + } + } // Sandbox assets go through the object-store plugin; a gs:// asset it // cannot serve, such as one in the public gs://gvisor bucket, is read @@ -402,6 +414,7 @@ func main() { *objectStorePluginSocket, imageCache, sandboxAssetCache, + snapshotCache, runtimes, instruments, volPlugins, @@ -671,6 +684,14 @@ type Atelet struct { // sandboxAssetCache holds every sandbox asset on the node once; actors get hard // links into their own directories (see ensureSandboxAssets). sandboxAssetCache *filecache.Store + // snapshotCache dedupes and retains shared snapshot files across + // restores. nil means caching is disabled (--snapshot-cache-dir=""): + // every restore downloads its snapshot files directly. + snapshotCache *filecache.Store + // snapshotCacheCrossMount warns once per process when a cache hit + // cannot be linked into a restore dir; every later hit says so at + // debug. + snapshotCacheCrossMount sync.Once // runtimes resolves the runtime a Run or Restore names to the assets to // stage, among those this node's workers advertise. runtimes *runtimeIndex @@ -719,6 +740,7 @@ func NewService( pluginSocket string, imageCache *imagecache.Store, sandboxAssetCache *filecache.Store, + snapshotCache *filecache.Store, runtimes *runtimeIndex, instruments *Instruments, volumePlugins map[string]volume.VolumePluginWorkerPlane, @@ -730,6 +752,7 @@ func NewService( workerDialer: workerDialer, imageCache: imageCache, sandboxAssetCache: sandboxAssetCache, + snapshotCache: snapshotCache, runtimes: runtimes, anonGCSClient: anonGCSClient, objectStorePlugin: objectStorePlugin, @@ -1471,9 +1494,13 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res // The record below is written last, so until then it describes the // previous activation; a failed restore must not leave this one's // asset links behind for a later Terminate to take for that record's. + // Nor its snapshot files: staged ones are links into the snapshot cache + // or copies, and either would hold the cache's entries, or disk, until + // the next lifecycle call resets the directory. defer func() { if err != nil { _ = os.RemoveAll(ateletpath.SandboxAssetsDir(actorUID)) + _ = os.RemoveAll(ateletpath.RestoreStateDir(actorUID)) } }() @@ -1513,7 +1540,7 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res slog.WarnContext(ctx, "Local snapshot is unusable; restoring from object storage", slog.String("local_dir", localDir), slog.Any("err", localErr)) } - manifest, err := s.fetchManifest(ctx, durable.GetSnapshotUri()) + manifest, err := s.fetchManifestCached(ctx, durable.GetSnapshotUri(), snapshotCacheMode(durable, cacheModeLink)) if status.Code(err) == codes.NotFound { return nil, apierror.NotFound("snapshot %q has no usable local copy and no uploaded copy: %w", req.GetSnapshot().GetSnapshotUuid(), localErr) @@ -1565,7 +1592,12 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res // readLocalSnapshot checked; no staging. return nil } - return s.downloadExternalCheckpoint(gctx, durable.GetSnapshotUri(), checkpointDir, sandboxRec.SnapshotFiles) + // Cache-eligible only when the control plane declared the snapshot + // shared; how a hit is served depends on the sandbox class that + // consumes it, the one the request resolved, which is what runs the + // restored workload. + mode := snapshotCacheMode(durable, cacheModeFor(runtimeRec.SandboxClass)) + return s.downloadExternalCheckpoint(gctx, durable.GetSnapshotUri(), checkpointDir, sandboxRec.SnapshotFiles, mode) }) g.Go(func() (err error) { defer func() { prepErr = err }() @@ -1866,8 +1898,19 @@ func copyRootFile(root *os.Root, src, dst string) (int64, error) { return nBytes, errors.Join(err, destination.Close()) } -func (s *Atelet) downloadExternalCheckpoint(ctx context.Context, snapshotURI string, dstDir string, files []string) error { - if err := s.fetchSnapshotFiles(ctx, snapshotURI, dstDir, files); err != nil { +// downloadExternalCheckpoint stages files of one snapshot into dstDir. With +// a mode other than cacheModeOff and the snapshot cache enabled, each file +// is served from the cache (see downloadSnapshotFilesCached): the caller +// asserts the snapshot is immutable at its URI. Otherwise the files are +// downloaded into the restore directory directly. +func (s *Atelet) downloadExternalCheckpoint(ctx context.Context, snapshotURI string, dstDir string, files []string, mode cacheMode) error { + var err error + if mode != cacheModeOff && s.snapshotCache != nil { + err = s.downloadSnapshotFilesCached(ctx, snapshotURI, dstDir, files, mode) + } else { + err = s.fetchSnapshotFiles(ctx, snapshotURI, dstDir, files) + } + if err != nil { return fmt.Errorf("while downloading snapshot files: %w", err) } return nil diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index 9fb7a4e906..a1918db774 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -541,7 +541,7 @@ func TestDownloadExternalCheckpointRejectsSymlinkOutsideRoot(t *testing.T) { t.Fatal(err) } err := newPluginHerder(t, store).downloadExternalCheckpoint( - context.Background(), testSnapshotURI, restoreDir, []string{"checkpoint.img"}) + context.Background(), testSnapshotURI, restoreDir, []string{"checkpoint.img"}, cacheModeOff) if err == nil { t.Fatal("downloadExternalCheckpoint() followed a symlink outside the restore directory") } diff --git a/cmd/atelet/snapshot_transfer_test.go b/cmd/atelet/snapshot_transfer_test.go index d92692998d..62f048919c 100644 --- a/cmd/atelet/snapshot_transfer_test.go +++ b/cmd/atelet/snapshot_transfer_test.go @@ -144,7 +144,7 @@ func TestUploadSnapshotManifestLast(t *testing.T) { func TestSnapshotTransferSkipsEmptyFileLists(t *testing.T) { store := &orderedObjectStorage{} s := newPluginHerder(t, store) - if err := s.downloadExternalCheckpoint(context.Background(), pausedSnapshotURI, t.TempDir(), nil); err != nil { + if err := s.downloadExternalCheckpoint(context.Background(), pausedSnapshotURI, t.TempDir(), nil, cacheModeOff); err != nil { t.Errorf("downloadExternalCheckpoint(no files) = %v, want nil", err) } if err := s.uploadSnapshotFiles(context.Background(), pausedSnapshotURI, t.TempDir(), nil); err != nil { @@ -190,7 +190,13 @@ func TestObjectStorePluginErrorCodeThroughAtelet(t *testing.T) { run func(ctx context.Context, s *Atelet) error }{ {"download", true, func(ctx context.Context, s *Atelet) error { - return s.downloadExternalCheckpoint(ctx, pausedSnapshotURI, t.TempDir(), []string{"a"}) + return s.downloadExternalCheckpoint(ctx, pausedSnapshotURI, t.TempDir(), []string{"a"}, cacheModeOff) + }}, + {"cached download, link", true, func(ctx context.Context, s *Atelet) error { + return s.downloadExternalCheckpoint(ctx, pausedSnapshotURI, t.TempDir(), []string{"a"}, cacheModeLink) + }}, + {"cached download, copy", true, func(ctx context.Context, s *Atelet) error { + return s.downloadExternalCheckpoint(ctx, pausedSnapshotURI, t.TempDir(), []string{"a"}, cacheModeCopy) }}, {"paused upload", true, func(ctx context.Context, s *Atelet) error { dir := t.TempDir() @@ -216,6 +222,7 @@ func TestObjectStorePluginErrorCodeThroughAtelet(t *testing.T) { s := &Atelet{ objectStorePlugin: failingNodePlugin{err: status.Error(pluginCode, "plugin failed")}, snapshotScratchDir: t.TempDir(), + snapshotCache: newTestAssetCache(t), } // atelet's server interceptor decides the code ate-api-server sees. _, err := ateinterceptors.InternalServerUnaryInterceptor(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "/test"}, @@ -249,7 +256,15 @@ func TestObjectStorePluginErrorKeepsCredentialsOut(t *testing.T) { run func(ctx context.Context, s *Atelet) error }{ {"download", signedURI, func(ctx context.Context, s *Atelet) error { - return s.downloadExternalCheckpoint(ctx, signedURI, t.TempDir(), []string{"a"}) + return s.downloadExternalCheckpoint(ctx, signedURI, t.TempDir(), []string{"a"}, cacheModeOff) + }}, + // The cached modes also build a cache key from the URI, which the + // cache quotes in its errors and logs. + {"cached download, link", signedURI, func(ctx context.Context, s *Atelet) error { + return s.downloadExternalCheckpoint(ctx, signedURI, t.TempDir(), []string{"a"}, cacheModeLink) + }}, + {"cached download, copy", signedURI, func(ctx context.Context, s *Atelet) error { + return s.downloadExternalCheckpoint(ctx, signedURI, t.TempDir(), []string{"a"}, cacheModeCopy) }}, {"upload", signedURI, func(ctx context.Context, s *Atelet) error { return s.uploadSnapshotFiles(ctx, signedURI, t.TempDir(), []string{"a"}) @@ -272,6 +287,7 @@ func TestObjectStorePluginErrorKeepsCredentialsOut(t *testing.T) { s := &Atelet{ objectStorePlugin: failingNodePlugin{err: status.Error(pluginCode, "get "+signedURI+": AccountKey="+secret)}, snapshotScratchDir: t.TempDir(), + snapshotCache: newTestAssetCache(t), } _, err := ateinterceptors.InternalServerUnaryInterceptor(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "/test"}, func(ctx context.Context, _ any) (any, error) { return nil, tr.run(ctx, s) }) diff --git a/cmd/atelet/snapshotcache.go b/cmd/atelet/snapshotcache.go new file mode 100644 index 0000000000..9b07457d68 --- /dev/null +++ b/cmd/atelet/snapshotcache.go @@ -0,0 +1,305 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +// The snapshot file cache. +// +// A shared snapshot's files, the snapshot a tag holds and every actor +// created from it restores, are immutable once published, yet every restore +// that needs them downloads them again into its own per-actor dir. This +// cache (a filecache.Store) makes those files a node-level resource: +// concurrent restores of one snapshot share a single download, and later +// restores are served from disk instead of object storage. +// +// Whether a snapshot is cacheable is the control plane's call, carried on +// the request as ObjectStorage.sharing; atelet only maps that property to a +// serving mode per sandbox class. This file holds the cache's flags and +// validation, its opening with the startup debris sweep, and the restore +// path's cached reads. The eviction loop is filecachegc, started from main. + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/url" + "os" + "path/filepath" + "syscall" + "time" + + "github.com/spf13/pflag" + "golang.org/x/sync/errgroup" + + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/cmd/atelet/internal/filecache" + "go.ate.dev/substrate/cmd/atelet/internal/filecachegc" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" +) + +var ( + snapshotCacheDir = pflag.String("snapshot-cache-dir", ateletpath.SnapshotCacheDir, "Directory for the node-local shared snapshot file cache. Empty disables caching: every restore downloads its snapshot files, and the other --snapshot-cache-* flags are validated but not read. Must be under the node state dir, "+nodepath.BasePath+": the object-store plugin writes only there, and cache hits are served as hard links into the per-actor restore dirs on the same mount.") + snapshotCacheMinAge = pflag.Duration("snapshot-cache-min-age", 10*time.Minute, "Cached snapshot files younger than this are never evicted, protecting files fetched but not yet linked into a restore dir. Read only while the cache is enabled.") + snapshotCacheFetchTimeout = pflag.Duration("snapshot-cache-fetch-timeout", 10*time.Minute, "Bound on one cached snapshot file download. Downloads run detached from the restores waiting on them, so this is the only bound on how long one can run; a restore arriving while a download hangs waits on it until then. Sized for multi-GiB memory images on a busy node. Read only while the cache is enabled.") + // snapshotCacheGCFlags are --snapshot-cache-gc-period, + // --snapshot-cache-max-bytes and --snapshot-cache-gc-dry-run. + snapshotCacheGCFlags = filecachegc.NewFlags(pflag.CommandLine, "snapshot-cache", "snapshot cache") +) + +// validateSnapshotCacheFlags checks every flag of the cache, enabled or +// not: a value outside its domain is a mistake whatever else is set, and +// startup is the cheapest moment to say so. +func validateSnapshotCacheFlags() error { + if *snapshotCacheMinAge < 0 { + // A negative min-age inverts the veto (the cutoff lands in the + // future), making just-fetched files evictable mid-restore. + return fmt.Errorf("--snapshot-cache-min-age %v must be >= 0", *snapshotCacheMinAge) + } + if *snapshotCacheFetchTimeout <= 0 { + return fmt.Errorf("--snapshot-cache-fetch-timeout %v must be > 0", *snapshotCacheFetchTimeout) + } + if err := snapshotCacheGCFlags.Validate(); err != nil { + return err + } + if *snapshotCacheDir != "" && !nodepath.UnderBasePath(*snapshotCacheDir) { + // The object-store plugin confines its writes to the base path, so + // a cache elsewhere could never be filled: every shared restore + // would fail at the fetch. + return fmt.Errorf("--snapshot-cache-dir %q must be under %s, the only tree the object-store plugin writes to", *snapshotCacheDir, nodepath.BasePath) + } + return nil +} + +// openSnapshotCache opens the snapshot cache rooted at dir and clears crash +// debris before the store serves any restore. An empty dir disables +// caching: the returned store is nil and restores download their snapshot +// files directly. +func openSnapshotCache(ctx context.Context, dir string, minAge, fetchTimeout time.Duration) (*filecache.Store, error) { + if dir == "" { + slog.InfoContext(ctx, "Snapshot cache disabled; every restore downloads its snapshot files") + return nil, nil + } + store, err := filecache.New(dir, filecache.WithMinAge(minAge), filecache.WithFetchTimeout(fetchTimeout)) + if err != nil { + return nil, fmt.Errorf("while opening snapshot cache at %s: %w", dir, err) + } + stats, err := store.SweepDebris(ctx) + if err != nil { + // Leftover debris wastes space but affects no lookup, so the store + // is fully usable: log and carry on rather than failing startup. + slog.WarnContext(ctx, "Snapshot cache debris sweep incomplete", slog.Any("err", err)) + } + slog.InfoContext(ctx, "Snapshot cache open", + slog.String("dir", dir), + slog.Duration("min_age", minAge), + slog.Duration("fetch_timeout", fetchTimeout), + slog.Int("tmp_removed", stats.TmpRemoved), + slog.Int("retired_removed", stats.RetiredRemoved)) + return store, nil +} + +// cacheMode is how a restore may materialize snapshot files from the +// cache. The mode exists because the fast path, a hard link, shares the +// cached inode with the consumer, which is only safe when the consumer never +// writes the staged file in place. +type cacheMode int + +const ( + // cacheModeOff downloads fresh, bypassing the cache. + cacheModeOff cacheMode = iota + // cacheModeLink hard-links the cached copy: zero-cost hits, but the + // consumer must treat the staged file as read-only. + cacheModeLink + // cacheModeCopy stages a private, hole-preserving copy: costlier per hit + // than a link, but the inode is the consumer's alone, to write in place, + // to drop from the page cache, or to keep open without pinning the + // cache's entry, and a copy can cross mounts. + cacheModeCopy +) + +// cacheModeFor returns how this sandbox class's restores may use the +// snapshot cache. ateworker-gvisor reads restore state and gets hard +// links. Once the restore's page loading is done it drops the memory and +// filesystem images from the page cache and unlinks them from its restore +// dir, keeping only the small descriptors linked, so the actor's cgroup is +// charged for its guest memory alone, as for any restore, and a link +// protects an image's entry through the restore and the min age, not for +// the actor's life: an entry evicted later is still read to the end by the +// descriptors runsc holds open, and the next restore of the snapshot reads +// the images from local disk. ateworker-microvm gets private copies. Not because a link would +// corrupt the cache: it replaces config.json by rename, and its delta +// merge copies instead of writing in place when the memory file's inode +// is shared. Because a link would cost more than it saves: an on-demand +// restore keeps memory-ranges open for the VM's whole life, so a link +// would pin the cache's entry that long and turn every suspend from the +// in-place delta merge into a full copy, and the worker drops its restore +// files from the page cache, which on a shared inode would drop them for +// every other restore too. A copy is still one download per snapshot per +// node. +func cacheModeFor(sandboxClass string) cacheMode { + switch atev1alpha1.SandboxClass(sandboxClass) { + case atev1alpha1.SandboxClassGvisor: + return cacheModeLink + case atev1alpha1.SandboxClassMicroVM: + return cacheModeCopy + default: + return cacheModeOff + } +} + +// snapshotCacheMode returns the cache mode for a restore's durable +// snapshot: classMode when the control plane declared the snapshot shared, +// cacheModeOff otherwise, which covers private snapshots and a caller that +// did not say. atelet never infers cacheability from the URI. +func snapshotCacheMode(object *ateletpb.ObjectStorage, classMode cacheMode) cacheMode { + if object.GetSharing() == ateletpb.SnapshotSharing_SNAPSHOT_SHARING_SHARED { + return classMode + } + return cacheModeOff +} + +// downloadSnapshotFilesCached stages the named files of the snapshot at +// snapshotURI into dstDir through the snapshot cache, each per mode. The +// caller asserts the snapshot is immutable at its URI. Files sit directly +// in dstDir or in its one checkpoint subdirectory, as the plugin's own +// download allows; a destination that already exists, a symlink included, +// fails rather than being followed. +func (s *Atelet) downloadSnapshotFilesCached(ctx context.Context, snapshotURI, dstDir string, files []string, mode cacheMode) error { + if err := resources.ValidateSnapshotFileNames(files); err != nil { + return err + } + // Subdirectories first, once each: the files are fetched in parallel, + // and several of them share the checkpoint subdirectory. + made := map[string]bool{} + for _, fileName := range files { + dir := filepath.Dir(fileName) + if dir == "." || made[dir] { + continue + } + if err := mkdirNoFollow(filepath.Join(dstDir, dir)); err != nil { + return fmt.Errorf("while creating %s in restore directory: %w", dir, err) + } + made[dir] = true + } + g, gCtx := errgroup.WithContext(ctx) + for _, fileName := range files { + g.Go(func() error { + if err := s.fetchSnapshotFile(gCtx, snapshotURI, fileName, filepath.Join(dstDir, fileName), mode); err != nil { + return fmt.Errorf("while downloading %s: %w", fileName, err) + } + return nil + }) + } + return g.Wait() +} + +// mkdirNoFollow creates dir unless it already is a directory. Anything else +// in its place, a symlink above all, is an error: the restore directory is +// written by path here, outside the os.Root the plugin's own download uses. +func mkdirNoFollow(dir string) error { + fi, err := os.Lstat(dir) + switch { + case errors.Is(err, os.ErrNotExist): + return os.Mkdir(dir, 0o700) + case err != nil: + return err + case !fi.IsDir(): + return fmt.Errorf("%s exists and is not a directory", dir) + } + return nil +} + +// fetchManifestCached reads the manifest of the snapshot at snapshotURI, +// through the snapshot cache when mode allows it: the manifest of a shared +// snapshot is as immutable as its files, and a restore that finds both on +// the node makes no request to object storage at all. atelet reads the +// manifest itself, so a link serves whatever class the files will take; +// it is read from a scratch dir and the link dropped again. +func (s *Atelet) fetchManifestCached(ctx context.Context, snapshotURI string, mode cacheMode) ([]byte, error) { + if mode == cacheModeOff || s.snapshotCache == nil { + return s.fetchManifest(ctx, snapshotURI) + } + dir, err := s.snapshotScratch() + if err != nil { + return nil, err + } + defer func() { _ = os.RemoveAll(dir) }() + local := filepath.Join(dir, sandboxManifestName) + if err := s.fetchSnapshotFile(ctx, snapshotURI, sandboxManifestName, local, cacheModeLink); err != nil { + return nil, err + } + return os.ReadFile(local) +} + +// fetchSnapshotFile stages one file of the snapshot at snapshotURI at +// local, through the snapshot cache per mode. A cacheModeLink hit is a hard +// link, so local stays valid regardless of later eviction; a cacheModeCopy +// hit is a private copy. Fetch errors pass through the cache wrapped, so +// the plugin's code classification survives. +func (s *Atelet) fetchSnapshotFile(ctx context.Context, snapshotURI, fileName, local string, mode cacheMode) error { + // Keyed by the object's address without its query: a signed URL's + // query is a credential, and the key is logged and written to meta.json. + u, err := url.Parse(snapshotURI) + if err != nil { + return fmt.Errorf("while parsing the snapshot URI: %w", err) + } + objectURI, err := url.JoinPath(redactURL(u), fileName) + if err != nil { + return fmt.Errorf("while addressing snapshot file %q: %w", fileName, err) + } + key := filecache.URIKey(objectURI) + fetch := func(ctx context.Context, dst string) error { + return s.fetchSnapshotFileTo(ctx, snapshotURI, fileName, dst) + } + if mode == cacheModeCopy { + return s.snapshotCache.GetFileCopyTo(ctx, key, local, fetch) + } + err = s.snapshotCache.GetFileTo(ctx, key, local, fetch) + if err == nil || !errors.Is(err, syscall.EXDEV) { + return err + } + // The cache sits on a different mount than the restore dir, a bind + // mount under the node state dir, so a link cannot work, but a copy + // can, and the failed link's fetch already published the entry, so + // this is a local read rather than a second download. It is the node's + // configuration, so it is said once, with the detail per file at debug. + s.snapshotCacheCrossMount.Do(func() { + slog.WarnContext(ctx, "Snapshot cache is on a different filesystem than the restore dirs; serving every hit as a copy instead of a hard link", + slog.String("snapshot_cache_dir", *snapshotCacheDir), slog.String("actors_dir", nodepath.ActorsDir)) + }) + slog.DebugContext(ctx, "Snapshot cache hit served as a copy across mounts", + slog.String("snapshot_uri", redactURL(u)), slog.String("file", fileName)) + return s.snapshotCache.GetFileCopyTo(ctx, key, local, fetch) +} + +// fetchSnapshotFileTo downloads one file of a snapshot to dst through the +// object-store plugin, which writes a directory of files: it fetches into +// a scratch dir beside dst, on the cache's filesystem, and moves the file +// into place. The plugin accepts a write path under the node state dir, +// which is where the cache lives. +func (s *Atelet) fetchSnapshotFileTo(ctx context.Context, snapshotURI, fileName, dst string) error { + scratch, err := os.MkdirTemp(filepath.Dir(dst), ".fetch-*") + if err != nil { + return fmt.Errorf("while creating a snapshot fetch dir: %w", err) + } + defer func() { _ = os.RemoveAll(scratch) }() + if err := s.fetchSnapshotFiles(ctx, snapshotURI, scratch, []string{fileName}); err != nil { + return err + } + return os.Rename(filepath.Join(scratch, fileName), dst) +} diff --git a/cmd/atelet/snapshotcache_linux_test.go b/cmd/atelet/snapshotcache_linux_test.go new file mode 100644 index 0000000000..59dfb30442 --- /dev/null +++ b/cmd/atelet/snapshotcache_linux_test.go @@ -0,0 +1,84 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestDownloadExternalCheckpointLinkFallsBackToCopyAcrossMounts needs two +// real filesystems, which /dev/shm gives on Linux: a link-mode restore into +// a directory on another mount than the cache must degrade to a private +// copy, not to a download. An unusable /dev/shm fails the test rather than +// skipping it; this is where the fallback is proven. +func TestDownloadExternalCheckpointLinkFallsBackToCopyAcrossMounts(t *testing.T) { + s, store, base := newSnapshotCacheHerder(t, map[string]string{"memory-ranges": "golden memory"}) + ctx := context.Background() + files := []string{"memory-ranges"} + logs := captureLogs(t) + + otherFS, err := os.MkdirTemp("/dev/shm", "snapshot-cache-test-") + if err != nil { + t.Fatalf("creating a directory on /dev/shm: %v", err) + } + t.Cleanup(func() { os.RemoveAll(otherFS) }) + // Prove the two dirs really are different mounts; same-mount tmpdirs + // would silently test the plain link path. + probe := filepath.Join(base, "probe") + if err := os.WriteFile(probe, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Link(probe, filepath.Join(otherFS, "probe")); err == nil { + t.Fatal("test dirs share a filesystem; cannot exercise EXDEV") + } + + first := filepath.Join(otherFS, "first") + if err := os.Mkdir(first, 0o700); err != nil { + t.Fatal(err) + } + if err := s.downloadExternalCheckpoint(ctx, testSnapshotURI, first, files, cacheModeLink); err != nil { + t.Fatalf("cross-mount link-mode restore: %v", err) + } + if got, err := os.ReadFile(filepath.Join(first, "memory-ranges")); err != nil || string(got) != "golden memory" { + t.Fatalf("staged %q, %v", got, err) + } + if n := store.gets.Load(); n != 1 { + t.Errorf("cross-mount fallback performed %d downloads, want 1 (the copy must reuse the published entry)", n) + } + + // And the fallback stays cache-served: a second cross-mount restore is a + // local copy, not a download. + second := filepath.Join(otherFS, "second") + if err := os.Mkdir(second, 0o700); err != nil { + t.Fatal(err) + } + if err := s.downloadExternalCheckpoint(ctx, testSnapshotURI, second, files, cacheModeLink); err != nil { + t.Fatal(err) + } + if n := store.gets.Load(); n != 1 { + t.Errorf("second cross-mount restore downloaded again (gets=%d, want 1)", n) + } + // The misconfiguration is the node's, not the restore's: one warning per + // process, not one per file per restore. + if n := strings.Count(logs.String(), "serving every hit as a copy"); n != 1 { + t.Errorf("cross-mount warning logged %d times over two restores, want once", n) + } +} diff --git a/cmd/atelet/snapshotcache_test.go b/cmd/atelet/snapshotcache_test.go new file mode 100644 index 0000000000..f1d947d041 --- /dev/null +++ b/cmd/atelet/snapshotcache_test.go @@ -0,0 +1,464 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "go.ate.dev/substrate/cmd/atelet/internal/filecache" + "go.ate.dev/substrate/internal/proto/ateletpb" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/objectstorage" +) + +func TestValidateSnapshotCacheFlags(t *testing.T) { + origDir, origMinAge, origPeriod, origFetch := *snapshotCacheDir, *snapshotCacheMinAge, *snapshotCacheGCFlags.Period, *snapshotCacheFetchTimeout + t.Cleanup(func() { + *snapshotCacheDir, *snapshotCacheMinAge, *snapshotCacheGCFlags.Period, *snapshotCacheFetchTimeout = origDir, origMinAge, origPeriod, origFetch + }) + + cases := []struct { + name string + dir string + minAge time.Duration + wantErr bool + }{ + {"defaults", origDir, 10 * time.Minute, false}, + {"disabled", "", 10 * time.Minute, false}, + {"disabled still rejects a bad min-age", "", -time.Second, true}, + {"zero min-age", origDir, 0, false}, + {"outside base path: the plugin could not fill it", "/var/lib/elsewhere/snapshot-cache", 10 * time.Minute, true}, + {"negative min-age inverts the veto", origDir, -time.Second, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + *snapshotCacheDir, *snapshotCacheMinAge = tc.dir, tc.minAge + err := validateSnapshotCacheFlags() + if (err != nil) != tc.wantErr { + t.Errorf("dir=%q minAge=%v: err=%v, wantErr=%v", tc.dir, tc.minAge, err, tc.wantErr) + } + }) + } + // The eviction flags are validated here too, enabled or not. + for _, dir := range []string{origDir, ""} { + *snapshotCacheDir, *snapshotCacheMinAge, *snapshotCacheGCFlags.Period = dir, 10*time.Minute, -time.Second + if err := validateSnapshotCacheFlags(); err == nil { + t.Errorf("validateSnapshotCacheFlags() = nil with a negative GC period and dir %q, want an error", dir) + } + } + // So is the fetch timeout: a zero or negative bound would end every + // download at once. + *snapshotCacheGCFlags.Period = origPeriod + for _, timeout := range []time.Duration{0, -time.Second} { + *snapshotCacheFetchTimeout = timeout + if err := validateSnapshotCacheFlags(); err == nil { + t.Errorf("validateSnapshotCacheFlags() = nil with fetch timeout %v, want an error", timeout) + } + } +} + +func TestOpenSnapshotCacheDisabled(t *testing.T) { + store, err := openSnapshotCache(context.Background(), "", 10*time.Minute, time.Minute) + if err != nil { + t.Fatalf("openSnapshotCache(\"\") error: %v", err) + } + if store != nil { + t.Errorf("openSnapshotCache(\"\") = %v, want nil (disabled)", store) + } +} + +func TestOpenSnapshotCacheCreatesRoot(t *testing.T) { + dir := filepath.Join(t.TempDir(), "nested", "snapshot-cache") + store, err := openSnapshotCache(context.Background(), dir, 10*time.Minute, time.Minute) + if err != nil { + t.Fatalf("openSnapshotCache(%q) error: %v", dir, err) + } + if store == nil { + t.Fatal("openSnapshotCache returned a nil store for a non-empty dir") + } + if fi, err := os.Stat(dir); err != nil || !fi.IsDir() { + t.Errorf("cache root %s not created: %v", dir, err) + } +} + +// TestOpenSnapshotCacheSweepsDebris pins the startup ordering contract: crash +// debris (unfinished fetches in tmp/, interrupted evictions as .rm-*) is gone +// by the time openSnapshotCache returns, while published entries survive. +func TestOpenSnapshotCacheSweepsDebris(t *testing.T) { + dir := t.TempDir() + // A prior life's layout: one published entry, plus debris of both kinds. + entryDir := filepath.Join(dir, "entries", "aaaa") + for _, d := range []string{ + filepath.Join(dir, "tmp", "aaaa-12345"), + filepath.Join(dir, ".rm-bbbb-xyz"), + entryDir, + } { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(d, "data"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + + store, err := openSnapshotCache(context.Background(), dir, 10*time.Minute, time.Minute) + if err != nil { + t.Fatalf("openSnapshotCache error: %v", err) + } + if store == nil { + t.Fatal("openSnapshotCache returned a nil store for a non-empty dir") + } + + if children, err := os.ReadDir(filepath.Join(dir, "tmp")); err != nil || len(children) != 0 { + t.Errorf("tmp debris not swept: %d children, %v", len(children), err) + } + if _, err := os.Stat(filepath.Join(dir, ".rm-bbbb-xyz")); !os.IsNotExist(err) { + t.Errorf("retired debris not swept: %v", err) + } + if _, err := os.Stat(filepath.Join(entryDir, "data")); err != nil { + t.Errorf("published entry did not survive the sweep: %v", err) + } +} + +// TestSnapshotCacheMode pins the caching gate: only a snapshot the control +// plane declared shared is served from the cache. Private, unstated, and +// absent sources all download fresh; atelet never infers cacheability from +// the URI. +func TestSnapshotCacheMode(t *testing.T) { + withSharing := func(sh ateletpb.SnapshotSharing) *ateletpb.ObjectStorage { + return &ateletpb.ObjectStorage{SnapshotUri: testSnapshotURI, Sharing: sh} + } + cases := []struct { + name string + object *ateletpb.ObjectStorage + want cacheMode + }{ + {"shared uses the class mode", withSharing(ateletpb.SnapshotSharing_SNAPSHOT_SHARING_SHARED), cacheModeLink}, + {"private is never cached", withSharing(ateletpb.SnapshotSharing_SNAPSHOT_SHARING_PRIVATE), cacheModeOff}, + {"unstated is never cached", withSharing(ateletpb.SnapshotSharing_SNAPSHOT_SHARING_UNSPECIFIED), cacheModeOff}, + {"no object", nil, cacheModeOff}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := snapshotCacheMode(tc.object, cacheModeLink); got != tc.want { + t.Errorf("snapshotCacheMode() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestCacheModeFor(t *testing.T) { + if got := cacheModeFor(string(atev1alpha1.SandboxClassGvisor)); got != cacheModeLink { + t.Errorf("gVisor consumes restore state read-only; want cacheModeLink, got %v", got) + } + // ateworker-microvm keeps memory-ranges open for the VM's life, merges + // deltas into it, and drops it from the page cache; through a link each + // of those would reach the cache's entry, so it gets private copies. + if got := cacheModeFor(string(atev1alpha1.SandboxClassMicroVM)); got != cacheModeCopy { + t.Errorf("micro-VM keeps and reworks its restore files; want cacheModeCopy, got %v", got) + } + if got := cacheModeFor(""); got != cacheModeOff { + t.Errorf("unknown sandbox class: want cacheModeOff, got %v", got) + } +} + +// countingObjectStorage counts reads, so a test can pin how many downloads +// a sequence of restores performed. +type countingObjectStorage struct { + objectstorage.ObjectStorage + gets atomic.Int32 +} + +func (c *countingObjectStorage) GetObject(ctx context.Context, bucket, object string) (io.ReadCloser, error) { + c.gets.Add(1) + return c.ObjectStorage.GetObject(ctx, bucket, object) +} + +// newSnapshotCacheHerder returns an atelet whose plugin serves a snapshot +// holding the named files with the given contents, with a snapshot cache +// open under base, plus base itself, which is on the cache's filesystem, +// for restore directories. +func newSnapshotCacheHerder(t *testing.T, files map[string]string) (*Atelet, *countingObjectStorage, string) { + t.Helper() + base := t.TempDir() + store := &countingObjectStorage{ObjectStorage: &recordingObjectStorage{}} + for name, content := range files { + payload := filepath.Join(base, "payload-"+filepath.Base(name)) + if err := os.WriteFile(payload, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + if err := objectstorage.SendLocalFileToGCSWithZstd(context.Background(), store.ObjectStorage, testSnapshotURI+"/"+name+".zstd", payload); err != nil { + t.Fatal(err) + } + } + s := newPluginHerder(t, store) + cache, err := filecache.New(filepath.Join(base, "cache")) + if err != nil { + t.Fatal(err) + } + s.snapshotCache = cache + return s, store, base +} + +// restoreDir makes a restore directory under base, as Restore does before +// it downloads. +func restoreDir(t *testing.T, base, name string) string { + t.Helper() + dir := filepath.Join(base, name) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + return dir +} + +// TestFetchManifestCached pins that a shared snapshot's manifest is read +// through the cache, one download for any number of restores, with the +// scratch link dropped again, while a private one is read fresh each time. +func TestFetchManifestCached(t *testing.T) { + const manifest = `{"sandboxClass":"gvisor"}` + s, store, _ := newSnapshotCacheHerder(t, nil) + ctx := context.Background() + // The plugin stores the manifest as is, not zstd-compressed like the + // data files the herder's helper uploads. + if err := store.ObjectStorage.PutObject(ctx, "bucket", strings.TrimPrefix(testSnapshotPath, "bucket/")+"/"+sandboxManifestName, strings.NewReader(manifest)); err != nil { + t.Fatal(err) + } + + for i := range 2 { + got, err := s.fetchManifestCached(ctx, testSnapshotURI, cacheModeLink) + if err != nil { + t.Fatalf("fetchManifestCached #%d: %v", i+1, err) + } + if string(got) != manifest { + t.Fatalf("manifest #%d = %q, want %q", i+1, got, manifest) + } + } + if n := store.gets.Load(); n != 1 { + t.Errorf("two cached manifest reads downloaded %d times, want 1", n) + } + if left, err := os.ReadDir(s.snapshotScratchDir); err != nil || len(left) != 0 { + t.Errorf("scratch dir holds %v (%v) after cached manifest reads, want nothing", left, err) + } + + // Not shared: read from object storage, as before the cache existed. + if _, err := s.fetchManifestCached(ctx, testSnapshotURI, cacheModeOff); err != nil { + t.Fatal(err) + } + if n := store.gets.Load(); n != 2 { + t.Errorf("a private manifest read downloaded %d times in total, want 2", n) + } +} + +func TestDownloadExternalCheckpointServesFromCache(t *testing.T) { + s, store, base := newSnapshotCacheHerder(t, map[string]string{"memory-ranges": "golden memory"}) + ctx := context.Background() + files := []string{"memory-ranges"} + + // Two cached restores: one download, both destinations share the inode. + first, second := restoreDir(t, base, "first"), restoreDir(t, base, "second") + for _, dir := range []string{first, second} { + if err := s.downloadExternalCheckpoint(ctx, testSnapshotURI, dir, files, cacheModeLink); err != nil { + t.Fatalf("downloadExternalCheckpoint into %s: %v", dir, err) + } + if got, err := os.ReadFile(filepath.Join(dir, "memory-ranges")); err != nil || string(got) != "golden memory" { + t.Fatalf("staged in %s = %q, %v", dir, got, err) + } + } + if n := store.gets.Load(); n != 1 { + t.Errorf("two cached restores performed %d downloads, want 1", n) + } + fi1, err1 := os.Stat(filepath.Join(first, "memory-ranges")) + fi2, err2 := os.Stat(filepath.Join(second, "memory-ranges")) + if err1 != nil || err2 != nil || !os.SameFile(fi1, fi2) { + t.Errorf("cached destinations do not share an inode (%v, %v)", err1, err2) + } + if fi1.Mode().Perm()&0o222 != 0 { + t.Errorf("a linked restore file is writable (%v); the shared bytes must be read-only", fi1.Mode()) + } + + // A non-cacheable restore downloads fresh and shares nothing. + direct := restoreDir(t, base, "direct") + if err := s.downloadExternalCheckpoint(ctx, testSnapshotURI, direct, files, cacheModeOff); err != nil { + t.Fatalf("direct downloadExternalCheckpoint: %v", err) + } + if n := store.gets.Load(); n != 2 { + t.Errorf("direct restore did not download (gets=%d, want 2)", n) + } + if fi3, err := os.Stat(filepath.Join(direct, "memory-ranges")); err != nil || os.SameFile(fi1, fi3) { + t.Errorf("direct destination shares the cache inode (%v)", err) + } +} + +// TestDownloadExternalCheckpointCopyMode pins the micro-VM serving mode: one +// download per snapshot file, every destination a private writable inode, +// and a consumer's in-place mutation never reaches the next consumer. +func TestDownloadExternalCheckpointCopyMode(t *testing.T) { + s, store, base := newSnapshotCacheHerder(t, map[string]string{"memory-ranges": "golden memory"}) + ctx := context.Background() + files := []string{"memory-ranges"} + + first, second := restoreDir(t, base, "first"), restoreDir(t, base, "second") + for _, dir := range []string{first, second} { + if err := s.downloadExternalCheckpoint(ctx, testSnapshotURI, dir, files, cacheModeCopy); err != nil { + t.Fatalf("downloadExternalCheckpoint into %s: %v", dir, err) + } + } + if n := store.gets.Load(); n != 1 { + t.Errorf("two copy-mode restores performed %d downloads, want 1", n) + } + fi1, err1 := os.Stat(filepath.Join(first, "memory-ranges")) + fi2, err2 := os.Stat(filepath.Join(second, "memory-ranges")) + if err1 != nil || err2 != nil { + t.Fatal(err1, err2) + } + if os.SameFile(fi1, fi2) { + t.Error("copy-mode destinations share an inode; each consumer must own its own") + } + + // The consumer may rewrite its staged file (ateworker-microvm does, to + // config.json) without poisoning what the next restore receives. + if err := os.WriteFile(filepath.Join(first, "memory-ranges"), []byte("rewritten by the worker"), 0o600); err != nil { + t.Fatalf("consumer write to its copy: %v", err) + } + third := restoreDir(t, base, "third") + if err := s.downloadExternalCheckpoint(ctx, testSnapshotURI, third, files, cacheModeCopy); err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(filepath.Join(third, "memory-ranges")); err != nil || string(got) != "golden memory" { + t.Errorf("copy after consumer mutation = %q, %v; the cached bytes were poisoned", got, err) + } + if n := store.gets.Load(); n != 1 { + t.Errorf("gets=%d, want 1 (mutation must not invalidate the cache)", n) + } +} + +func TestDownloadExternalCheckpointWithoutCacheDownloads(t *testing.T) { + s, store, base := newSnapshotCacheHerder(t, map[string]string{"memory-ranges": "golden memory"}) + s.snapshotCache = nil // caching disabled (--snapshot-cache-dir="") + + dir := restoreDir(t, base, "out") + if err := s.downloadExternalCheckpoint(context.Background(), testSnapshotURI, dir, []string{"memory-ranges"}, cacheModeLink); err != nil { + t.Fatalf("downloadExternalCheckpoint with nil cache: %v", err) + } + if got, err := os.ReadFile(filepath.Join(dir, "memory-ranges")); err != nil || string(got) != "golden memory" { + t.Fatalf("staged content = %q, %v", got, err) + } + if n := store.gets.Load(); n != 1 { + t.Errorf("gets=%d, want 1", n) + } +} + +func TestDownloadExternalCheckpointCachedFetchErrorReachesCaller(t *testing.T) { + s, _, base := newSnapshotCacheHerder(t, map[string]string{"memory-ranges": "golden memory"}) + dir := restoreDir(t, base, "out") + err := s.downloadExternalCheckpoint(context.Background(), testSnapshotURI, dir, []string{"missing"}, cacheModeLink) + if err == nil { + t.Fatal("downloadExternalCheckpoint succeeded for a missing file") + } + if _, statErr := os.Stat(filepath.Join(dir, "missing")); !os.IsNotExist(statErr) { + t.Errorf("failed fetch left a destination file: %v", statErr) + } +} + +// TestDownloadExternalCheckpointSharesOneDownload pins the exit criterion at +// the unit level: concurrent restores staging the same shared snapshot +// perform its downloads once. +func TestDownloadExternalCheckpointSharesOneDownload(t *testing.T) { + s, store, base := newSnapshotCacheHerder(t, map[string]string{"memory-ranges": "golden memory"}) + files := []string{"memory-ranges"} + + const restores = 4 + errs := make([]error, restores) + var wg sync.WaitGroup + for i := range restores { + dir := restoreDir(t, base, fmt.Sprintf("actor-%d", i)) + wg.Go(func() { + errs[i] = s.downloadExternalCheckpoint(context.Background(), testSnapshotURI, dir, files, cacheModeLink) + }) + } + wg.Wait() + for i, err := range errs { + if err != nil { + t.Fatalf("restore %d: %v", i, err) + } + got, err := os.ReadFile(filepath.Join(base, fmt.Sprintf("actor-%d", i), "memory-ranges")) + if err != nil || string(got) != "golden memory" { + t.Fatalf("restore %d staged %q, %v", i, got, err) + } + } + if n := store.gets.Load(); n != 1 { + t.Errorf("%d concurrent restores performed %d downloads, want 1", restores, n) + } +} + +// TestDownloadExternalCheckpointCachedSubdirAndSymlinks covers the two ways +// the cached path differs from the plugin writing through an os.Root: a +// file in the checkpoint subdirectory is placed there, and a symlink waiting +// in the restore directory, as a directory or as the file itself, fails the +// restore instead of being followed outside it. +func TestDownloadExternalCheckpointCachedSubdirAndSymlinks(t *testing.T) { + s, _, base := newSnapshotCacheHerder(t, map[string]string{"memory-ranges": "golden memory", "fs/data.tar": "fs delta", "fs/rootfs-upper-app.tar": "rootfs delta"}) + ctx := context.Background() + + // Two files share the subdirectory and are fetched in parallel: the + // directory is made once, before either, not raced into by both. + dir := restoreDir(t, base, "subdir") + if err := s.downloadExternalCheckpoint(ctx, testSnapshotURI, dir, []string{"memory-ranges", "fs/data.tar", "fs/rootfs-upper-app.tar"}, cacheModeLink); err != nil { + t.Fatalf("downloadExternalCheckpoint with subdirectory files: %v", err) + } + for name, want := range map[string]string{"fs/data.tar": "fs delta", "fs/rootfs-upper-app.tar": "rootfs delta"} { + if got, err := os.ReadFile(filepath.Join(dir, name)); err != nil || string(got) != want { + t.Fatalf("subdirectory file %s = %q, %v", name, got, err) + } + } + + outside := filepath.Join(base, "outside") + if err := os.MkdirAll(outside, 0o700); err != nil { + t.Fatal(err) + } + keep := filepath.Join(outside, "memory-ranges") + if err := os.WriteFile(keep, []byte("keep me"), 0o600); err != nil { + t.Fatal(err) + } + fileLink := restoreDir(t, base, "file-link") + if err := os.Symlink(keep, filepath.Join(fileLink, "memory-ranges")); err != nil { + t.Fatal(err) + } + if err := s.downloadExternalCheckpoint(ctx, testSnapshotURI, fileLink, []string{"memory-ranges"}, cacheModeLink); err == nil { + t.Error("a symlink in place of the restore file was followed") + } + dirLink := restoreDir(t, base, "dir-link") + if err := os.Symlink(outside, filepath.Join(dirLink, "fs")); err != nil { + t.Fatal(err) + } + if err := s.downloadExternalCheckpoint(ctx, testSnapshotURI, dirLink, []string{"fs/data.tar"}, cacheModeCopy); err == nil { + t.Error("a symlink in place of the checkpoint subdirectory was followed") + } + if got, err := os.ReadFile(keep); err != nil || string(got) != "keep me" { + t.Errorf("file outside the restore directory = %q, %v; want untouched", got, err) + } + if entries, _ := os.ReadDir(outside); len(entries) != 1 { + t.Errorf("the directory outside the restore directory holds %d entries, want only the original file", len(entries)) + } +} diff --git a/cmd/ateworker-gvisor/ckptcgroup.go b/cmd/ateworker-gvisor/ckptcgroup.go index 7660da6207..d3f332c985 100644 --- a/cmd/ateworker-gvisor/ckptcgroup.go +++ b/cmd/ateworker-gvisor/ckptcgroup.go @@ -27,6 +27,7 @@ import ( "slices" "strconv" "strings" + "syscall" "go.ate.dev/substrate/internal/ocispec" "go.ate.dev/substrate/internal/proto/ateworkerpb" @@ -224,17 +225,21 @@ func isCheckpointDataFile(name string) bool { var evictCheckpointCache = snapshotcache.EvictWithRetries // dropActorCheckpointCacheAsync pins the snapshot data files in restoreDir -// while the caller holds the actor lock. When the restore dir is not preserved, -// it unlinks them immediately (the kernel frees pages and blocks once the -// sentry closes its open FDs). When the restore dir is preserved, it waits in -// the background (bounded by actorCtx) for async restore page loading to finish -// and evicts clean pages from the host page cache. +// while the caller holds the actor lock. When the restore dir is not preserved +// and nothing else links the files, it unlinks them immediately: the kernel +// frees pages and blocks once the sentry closes its open FDs, and the actor's +// cgroup is no longer charged for them. When the restore dir is preserved, or +// a file has another link, such as atelet's snapshot cache entry it was staged +// from, so that unlinking would free nothing, it waits in the background +// (bounded by actorCtx) for async restore page loading to finish and evicts +// clean pages from the host page cache, then unlinks the files unless the +// dir is preserved. func dropActorCheckpointCacheAsync(actorCtx context.Context, actorUID, restoreDir string, preserveRestoreDir bool, waitRestore func(context.Context) error) { targets := snapshotcache.FindFiles(restoreDir, isCheckpointDataFile) if len(targets) == 0 { return } - if !preserveRestoreDir { + if !preserveRestoreDir && !anyLinkedElsewhere(targets) { snapshotcache.DropFiles(targets) return } @@ -243,9 +248,26 @@ func dropActorCheckpointCacheAsync(actorCtx context.Context, actorUID, restoreDi if err := waitRestore(actorCtx); err != nil { slog.Warn("Skipping checkpoint page cache eviction after restore wait error", slog.String("actorUID", actorUID), slog.Any("err", err)) + if !preserveRestoreDir { + snapshotcache.DropFiles(targets) + } return } } evictCheckpointCache(targets) + if !preserveRestoreDir { + snapshotcache.DropFiles(targets) + } }() } + +// anyLinkedElsewhere reports whether any of files has a hard link besides +// the one in the restore dir. +func anyLinkedElsewhere(files []snapshotcache.File) bool { + for _, f := range files { + if st, ok := f.Info.Sys().(*syscall.Stat_t); ok && st.Nlink > 1 { + return true + } + } + return false +} diff --git a/cmd/ateworker-gvisor/ckptcgroup_test.go b/cmd/ateworker-gvisor/ckptcgroup_test.go index adaa737838..7cb2c83ac9 100644 --- a/cmd/ateworker-gvisor/ckptcgroup_test.go +++ b/cmd/ateworker-gvisor/ckptcgroup_test.go @@ -462,6 +462,72 @@ func TestDropStagedRestoreFiles(t *testing.T) { } } +// TestDropSharedRestoreFilesAfterEviction pins the case of a restore dir +// whose files are hard links into atelet's snapshot cache: unlinking them at +// once would free nothing, since the cache's link keeps the inode and its +// pages charged to the actor's cgroup, so the files are evicted from the page +// cache after the restore's page loading, as a preserved dir is, and only +// then unlinked. +func TestDropSharedRestoreFilesAfterEviction(t *testing.T) { + restoreDir := newRestoreDir(t) + cache := filepath.Join(t.TempDir(), "cache") + mustMkdir(t, cache) + for _, name := range []string{"checkpoint.img", "pages.img"} { + if err := os.Link(filepath.Join(restoreDir, name), filepath.Join(cache, name)); err != nil { + t.Fatal(err) + } + } + evicted := captureEviction(t) + restoreDone := make(chan struct{}) + + dropActorCheckpointCacheAsync(context.Background(), testActorUID, restoreDir, false, func(ctx context.Context) error { + select { + case <-restoreDone: + return nil + case <-ctx.Done(): + return ctx.Err() + } + }) + // Nothing happens while the restore still loads pages from the files. + select { + case <-evicted: + t.Fatal("evicted before the restore wait completed") + case <-time.After(50 * time.Millisecond): + } + for _, name := range []string{"checkpoint.img", "pages.img", "fs/pages.img"} { + if _, err := os.Stat(filepath.Join(restoreDir, name)); err != nil { + t.Fatalf("%s unlinked before the restore wait completed: %v", name, err) + } + } + + close(restoreDone) + select { + case targets := <-evicted: + if len(targets) != 3 { + t.Errorf("evicted %d files, want the 3 data files", len(targets)) + } + case <-time.After(5 * time.Second): + t.Fatal("eviction never ran after the restore wait completed") + } + deadline := time.Now().Add(5 * time.Second) + for _, name := range []string{"checkpoint.img", "pages.img", "fs/pages.img"} { + for { + if _, err := os.Stat(filepath.Join(restoreDir, name)); os.IsNotExist(err) { + break + } + if time.Now().After(deadline) { + t.Fatalf("%s still exists after eviction", name) + } + time.Sleep(10 * time.Millisecond) + } + } + for _, name := range []string{"checkpoint.img", "pages.img"} { + if _, err := os.Stat(filepath.Join(cache, name)); err != nil { + t.Errorf("the cache's link to %s went with the restore dir's: %v", name, err) + } + } +} + func TestEvictPreservedRestoreDir(t *testing.T) { t.Run("evicts once the restore wait completes", func(t *testing.T) { restoreDir := newRestoreDir(t) diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index 0fe9265ae9..9204478f96 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -589,6 +589,13 @@ groups: The sandbox asset cache: the runsc binaries, the kernels and the guest images the workers run actors with. Each actor on the node links its assets from this one cache. + - id: snapshot + stability: development + value: snapshot + brief: > + The snapshot cache: the files of snapshots many actors + restore from, a tag's or a golden's, served to each restore + as hard links or private copies. - id: ate.filecache.gc.outcome stability: development brief: > @@ -1354,10 +1361,11 @@ groups: note: > atelet reads the value on each collection as the sum of the allocated size of each entry in the cache. Compare it with the budget that the - cache's max-bytes flag sets, --sandbox-asset-cache-max-bytes for the - sandbox asset cache. A value that stays over the budget with shortfall - passes means that the cache's consumers hold more than the budget: for - the sandbox asset cache, the actors and the workers' default runtimes. + cache's max-bytes flag sets, --sandbox-asset-cache-max-bytes or + --snapshot-cache-max-bytes. A value that stays over the budget with + shortfall passes means that the cache's consumers hold more than the + budget: for the sandbox asset cache, the actors and the workers' + default runtimes; for the snapshot cache, the restores in progress. The budget is too small, or the node hosts too many runtimes. annotations: substrate: diff --git a/docs/observability.md b/docs/observability.md index ff8fd79ac6..41096503d8 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -315,7 +315,7 @@ For `ate.imagecache.requests`: * `ate.imagecache.outcome` is `hit` when the node holds a complete image record — every layer directory the record names is present — and `miss` when the lookup must pull. A failed lookup is neither: `error` is a failed lookup whatever the cause, and `cancelled` or `timeout` is the caller giving up, as on `ate.router.outcome`. So the hit ratio is `hit / (hit + miss)`, with failures and abandoned lookups out of the denominator. * `error.type` is present only on the `error` outcome, and carries the registry's own HTTP status for its rejection, from a fixed set: `401`, `403`, `404`, `429`, `500`, `502`, `503`, `504`. The set is an allow-list because the registry client reports whatever the remote returned. Each other status, and each failure that carries no status, reports `_OTHER`. -For the `ate.filecache.*` instruments, one series per cache under `ate.filecache.name`, today `sandbox_asset` for the sandbox asset cache (see the [Sandbox Asset Cache](sandbox-asset-cache.md) page): +For the `ate.filecache.*` instruments, one series per cache under `ate.filecache.name`: `sandbox_asset` for the sandbox asset cache and `snapshot` for the snapshot cache (see the [Sandbox Asset Cache](sandbox-asset-cache.md) page): * The cache evicts only what nothing links: an actor's staged assets and each worker's default runtime are hard links into the cache, and an entry with a link stays whatever the pressure. `ate.filecache.gc.outcome` is `shortfall` when a pass could not reach the budget for that reason; the size then measures what the actors and the defaults on the node need, not waste. `skipped` means atelet could not read the cache directory and did nothing. `ate.workerpool.namespace` and `ate.workerpool.name` identify a pool together, on every instrument that names one. A WorkerPool is a namespaced resource, so the name on its own merges same-named pools from different namespaces into one series. The pair means that capacity (`ate.workerpool.workers`, `ate.workerpool.desired_workers`, `ate.workerpool.ready_workers`) joins to demand (`ate.scheduler.assignment.duration`, `ate.actor.lifecycle.operation.duration`, `ate.actor.crashes`) by pool. diff --git a/docs/sandbox-asset-cache.md b/docs/sandbox-asset-cache.md index 4b1976fab3..f1e5131aad 100644 --- a/docs/sandbox-asset-cache.md +++ b/docs/sandbox-asset-cache.md @@ -205,9 +205,10 @@ metrics suite asserts that one. ## Not covered here, deliberately -- **Snapshots.** The same `filecache` store is designed to serve shared - snapshot restores, the golden and clone images many actors start from. - That milestone is separate and not yet merged. +- **Snapshots.** The same `filecache` store, the same eviction loop and the + same instruments serve shared snapshot restores, the images many actors + start from, as the `snapshot` value of `ate.filecache.name`. This page is + about the asset cache only. - **One GC driver for both caches.** The image layer cache has its own eviction loop with the same shape but no metrics. Sharing the driver would give it the pass and eviction instruments; it is a follow-up. diff --git a/internal/ateattr/ateattr.go b/internal/ateattr/ateattr.go index 5c5097cf76..0fb9f5df97 100644 --- a/internal/ateattr/ateattr.go +++ b/internal/ateattr/ateattr.go @@ -244,6 +244,9 @@ const ( // FileCacheSandboxAsset is the sandbox asset cache: the runsc binaries, // kernels and guest images the workers run actors with. FileCacheSandboxAsset = "sandbox_asset" + // FileCacheSnapshot is the snapshot cache: the files of shared snapshots, + // the ones many actors restore from. + FileCacheSnapshot = "snapshot" ) // Values for FileCacheGCOutcomeKey. A pass is quiet when the cache is within diff --git a/internal/nodepath/nodepath.go b/internal/nodepath/nodepath.go index 7a4d563f85..d9976efafe 100644 --- a/internal/nodepath/nodepath.go +++ b/internal/nodepath/nodepath.go @@ -18,7 +18,11 @@ // stages runtime binaries. package nodepath -import "path/filepath" +import ( + "os" + "path/filepath" + "strings" +) // BasePath is the root shared folder on the host filesystem, mounted at the // same path into the atelet and worker containers. @@ -27,6 +31,21 @@ const BasePath = "/var/lib/ate" // ActorsDir is the parent of the per-actor directories atelet prepares. var ActorsDir = filepath.Join(BasePath, "actors") +// UnderBasePath reports whether dir resolves to a directory strictly under +// BasePath; BasePath itself does not count. A relative dir resolves against +// the process working directory. Callers use it to tell a cache directory +// configured off the shared mount, and judge the consequence themselves: the +// image cache's GC watermarks then measure a different volume than actor +// state, and the snapshot cache cannot be filled at all, since the +// object-store plugin writes only under the base path. +func UnderBasePath(dir string) bool { + abs, err := filepath.Abs(dir) + if err != nil { + abs = filepath.Clean(dir) + } + return strings.HasPrefix(abs, BasePath+string(os.PathSeparator)) +} + // ActorSandboxAssetsDir is where atelet stages an actor's sandbox runtime // assets, hard-linked from its asset cache, and the only location a worker // runs a sandbox binary from. Both sides derive it from the actor UID, so a diff --git a/internal/nodepath/nodepath_test.go b/internal/nodepath/nodepath_test.go index 28a84cbdc1..62c9a78d65 100644 --- a/internal/nodepath/nodepath_test.go +++ b/internal/nodepath/nodepath_test.go @@ -15,6 +15,7 @@ package nodepath import ( + "path/filepath" "strings" "testing" ) @@ -80,3 +81,27 @@ func TestWorkerPathUniqueness(t *testing.T) { t.Errorf("expected different paths for different pod UIDs, got %q", path1) } } + +func TestUnderBasePath(t *testing.T) { + cases := []struct { + name string + dir string + want bool + }{ + {"inside", filepath.Join(BasePath, "image-cache"), true}, + {"inside with doubled separator", BasePath + "//image-cache", true}, + {"inside via dot-dot", BasePath + "/x/../image-cache", true}, + {"base path itself is not inside", BasePath, false}, + {"sibling with the base path as name prefix", BasePath + "-other/image-cache", false}, + {"outside", "/var/lib/elsewhere/image-cache", false}, + {"dot-dot escaping the base path", BasePath + "/../elsewhere/image-cache", false}, + {"relative resolves against the cwd, not the base path", "image-cache", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := UnderBasePath(tc.dir); got != tc.want { + t.Errorf("UnderBasePath(%q) = %v, want %v", tc.dir, got, tc.want) + } + }) + } +} diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index b174379c53..fc781298ee 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -150,6 +150,62 @@ func (SnapshotFidelity) EnumDescriptor() ([]byte, []int) { return file_atelet_proto_rawDescGZIP(), []int{1} } +// SnapshotSharing is how a snapshot in object storage may be consumed, as +// known by the control plane. atelet never infers it from the URI. +type SnapshotSharing int32 + +const ( + // Not stated: treated as PRIVATE. + SnapshotSharing_SNAPSHOT_SHARING_UNSPECIFIED SnapshotSharing = 0 + // Read by one actor and replaced by its next suspend. Never cached. + SnapshotSharing_SNAPSHOT_SHARING_PRIVATE SnapshotSharing = 1 + // Immutable once published and read by many actors, such as a tag's + // snapshot borrowed at creation. atelet may serve its files from the + // node-local snapshot cache. + SnapshotSharing_SNAPSHOT_SHARING_SHARED SnapshotSharing = 2 +) + +// Enum value maps for SnapshotSharing. +var ( + SnapshotSharing_name = map[int32]string{ + 0: "SNAPSHOT_SHARING_UNSPECIFIED", + 1: "SNAPSHOT_SHARING_PRIVATE", + 2: "SNAPSHOT_SHARING_SHARED", + } + SnapshotSharing_value = map[string]int32{ + "SNAPSHOT_SHARING_UNSPECIFIED": 0, + "SNAPSHOT_SHARING_PRIVATE": 1, + "SNAPSHOT_SHARING_SHARED": 2, + } +) + +func (x SnapshotSharing) Enum() *SnapshotSharing { + p := new(SnapshotSharing) + *p = x + return p +} + +func (x SnapshotSharing) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SnapshotSharing) Descriptor() protoreflect.EnumDescriptor { + return file_atelet_proto_enumTypes[2].Descriptor() +} + +func (SnapshotSharing) Type() protoreflect.EnumType { + return &file_atelet_proto_enumTypes[2] +} + +func (x SnapshotSharing) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use SnapshotSharing.Descriptor instead. +func (SnapshotSharing) EnumDescriptor() ([]byte, []int) { + return file_atelet_proto_rawDescGZIP(), []int{2} +} + // UploadMode is when a checkpoint's snapshot is uploaded. type UploadMode int32 @@ -199,11 +255,11 @@ func (x UploadMode) String() string { } func (UploadMode) Descriptor() protoreflect.EnumDescriptor { - return file_atelet_proto_enumTypes[2].Descriptor() + return file_atelet_proto_enumTypes[3].Descriptor() } func (UploadMode) Type() protoreflect.EnumType { - return &file_atelet_proto_enumTypes[2] + return &file_atelet_proto_enumTypes[3] } func (x UploadMode) Number() protoreflect.EnumNumber { @@ -212,7 +268,7 @@ func (x UploadMode) Number() protoreflect.EnumNumber { // Deprecated: Use UploadMode.Descriptor instead. func (UploadMode) EnumDescriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{2} + return file_atelet_proto_rawDescGZIP(), []int{3} } type RegisterWorkerRequest struct { @@ -2940,7 +2996,15 @@ type ObjectStorage struct { // +k8s:required // +k8s:maxLength=2048 // +k8s:customValidation # a parseable snapshot URI - SnapshotUri string `protobuf:"bytes,1,opt,name=snapshot_uri,json=snapshotUri,proto3" json:"snapshot_uri,omitempty"` + SnapshotUri string `protobuf:"bytes,1,opt,name=snapshot_uri,json=snapshotUri,proto3" json:"snapshot_uri,omitempty"` + // How the snapshot at snapshot_uri may be consumed, as the control plane + // knows it. Read on Restore, where it decides whether atelet may serve the + // snapshot's files from its node-local cache; meaningless where atelet + // writes to the snapshot. + // + // +k8s:optional + // +k8s:maximum=2 # keep this in sync with the SnapshotSharing enum + Sharing SnapshotSharing `protobuf:"varint,2,opt,name=sharing,proto3,enum=atelet.SnapshotSharing" json:"sharing,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2982,6 +3046,13 @@ func (x *ObjectStorage) GetSnapshotUri() string { return "" } +func (x *ObjectStorage) GetSharing() SnapshotSharing { + if x != nil { + return x.Sharing + } + return SnapshotSharing_SNAPSHOT_SHARING_UNSPECIFIED +} + // Snapshot identifies one snapshot of an actor, and where its durable copy // lives. type Snapshot struct { @@ -4110,9 +4181,10 @@ const file_atelet_proto_rawDesc = "" + "\rHttpGetAction\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x12\n" + "\x04port\x18\x02 \x01(\x05R\x04port\"\r\n" + - "\vRunResponse\"2\n" + + "\vRunResponse\"e\n" + "\rObjectStorage\x12!\n" + - "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\"\xb4\x01\n" + + "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\x121\n" + + "\asharing\x18\x02 \x01(\x0e2\x17.atelet.SnapshotSharingR\asharing\"\xb4\x01\n" + "\bSnapshot\x12#\n" + "\rsnapshot_uuid\x18\x01 \x01(\tR\fsnapshotUuid\x12\x1e\n" + "\n" + @@ -4186,7 +4258,11 @@ const file_atelet_proto_rawDesc = "" + "\x1dSNAPSHOT_FIDELITY_UNSPECIFIED\x10\x00\x12\x1d\n" + "\x19SNAPSHOT_FIDELITY_VOLUMES\x10\x01\x12\x1c\n" + "\x18SNAPSHOT_FIDELITY_ROOTFS\x10\x02\x12\x1c\n" + - "\x18SNAPSHOT_FIDELITY_MEMORY\x10\x03*^\n" + + "\x18SNAPSHOT_FIDELITY_MEMORY\x10\x03*n\n" + + "\x0fSnapshotSharing\x12 \n" + + "\x1cSNAPSHOT_SHARING_UNSPECIFIED\x10\x00\x12\x1c\n" + + "\x18SNAPSHOT_SHARING_PRIVATE\x10\x01\x12\x1b\n" + + "\x17SNAPSHOT_SHARING_SHARED\x10\x02*^\n" + "\n" + "UploadMode\x12\x1b\n" + "\x17UPLOAD_MODE_UNSPECIFIED\x10\x00\x12\x18\n" + @@ -4219,160 +4295,162 @@ func file_atelet_proto_rawDescGZIP() []byte { return file_atelet_proto_rawDescData } -var file_atelet_proto_enumTypes = make([]protoimpl.EnumInfo, 3) +var file_atelet_proto_enumTypes = make([]protoimpl.EnumInfo, 4) var file_atelet_proto_msgTypes = make([]protoimpl.MessageInfo, 58) var file_atelet_proto_goTypes = []any{ (ActorMetadataField)(0), // 0: atelet.ActorMetadataField (SnapshotFidelity)(0), // 1: atelet.SnapshotFidelity - (UploadMode)(0), // 2: atelet.UploadMode - (*RegisterWorkerRequest)(nil), // 3: atelet.RegisterWorkerRequest - (*SandboxRuntime)(nil), // 4: atelet.SandboxRuntime - (*VersionedSandboxCompat)(nil), // 5: atelet.VersionedSandboxCompat - (*AttributeEntry)(nil), // 6: atelet.AttributeEntry - (*WorkerResources)(nil), // 7: atelet.WorkerResources - (*ResourceRequirements)(nil), // 8: atelet.ResourceRequirements - (*Resources)(nil), // 9: atelet.Resources - (*RegisterWorkerResponse)(nil), // 10: atelet.RegisterWorkerResponse - (*RequestActorSuspendRequest)(nil), // 11: atelet.RequestActorSuspendRequest - (*RequestActorSuspendResponse)(nil), // 12: atelet.RequestActorSuspendResponse - (*ReportActorCrashedRequest)(nil), // 13: atelet.ReportActorCrashedRequest - (*ReportActorCrashedResponse)(nil), // 14: atelet.ReportActorCrashedResponse - (*GetActorEgressPolicyRequest)(nil), // 15: atelet.GetActorEgressPolicyRequest - (*GetActorEgressPolicyResponse)(nil), // 16: atelet.GetActorEgressPolicyResponse - (*EgressPolicy)(nil), // 17: atelet.EgressPolicy - (*EgressRule)(nil), // 18: atelet.EgressRule - (*EgressDestinations)(nil), // 19: atelet.EgressDestinations - (*EgressPorts)(nil), // 20: atelet.EgressPorts - (*EgressAllPorts)(nil), // 21: atelet.EgressAllPorts - (*MintActorCertificateRequest)(nil), // 22: atelet.MintActorCertificateRequest - (*MintActorCertificateResponse)(nil), // 23: atelet.MintActorCertificateResponse - (*TerminateRequest)(nil), // 24: atelet.TerminateRequest - (*TerminateResponse)(nil), // 25: atelet.TerminateResponse - (*RunRequest)(nil), // 26: atelet.RunRequest - (*EgressGateway)(nil), // 27: atelet.EgressGateway - (*WorkloadSpec)(nil), // 28: atelet.WorkloadSpec - (*DurableDirVolumeSource)(nil), // 29: atelet.DurableDirVolumeSource - (*ExternalVolumeSource)(nil), // 30: atelet.ExternalVolumeSource - (*ImageVolumeSource)(nil), // 31: atelet.ImageVolumeSource - (*ActorMetadataItem)(nil), // 32: atelet.ActorMetadataItem - (*ActorMetadataDataSource)(nil), // 33: atelet.ActorMetadataDataSource - (*TrustBundleDataSource)(nil), // 34: atelet.TrustBundleDataSource - (*SystemInfoDataSource)(nil), // 35: atelet.SystemInfoDataSource - (*SystemInfoVolumeSource)(nil), // 36: atelet.SystemInfoVolumeSource - (*Volume)(nil), // 37: atelet.Volume - (*VolumeMount)(nil), // 38: atelet.VolumeMount - (*Container)(nil), // 39: atelet.Container - (*SecurityContext)(nil), // 40: atelet.SecurityContext - (*Capabilities)(nil), // 41: atelet.Capabilities - (*EnvVar)(nil), // 42: atelet.EnvVar - (*WakeupProbe)(nil), // 43: atelet.WakeupProbe - (*HttpGetAction)(nil), // 44: atelet.HttpGetAction - (*RunResponse)(nil), // 45: atelet.RunResponse - (*ObjectStorage)(nil), // 46: atelet.ObjectStorage - (*Snapshot)(nil), // 47: atelet.Snapshot - (*SnapshotReport)(nil), // 48: atelet.SnapshotReport - (*ActorSnapshotState)(nil), // 49: atelet.ActorSnapshotState - (*UploadConfig)(nil), // 50: atelet.UploadConfig - (*CheckpointRequest)(nil), // 51: atelet.CheckpointRequest - (*CheckpointResponse)(nil), // 52: atelet.CheckpointResponse - (*RestoreRequest)(nil), // 53: atelet.RestoreRequest - (*RestoreResponse)(nil), // 54: atelet.RestoreResponse - (*PromoteSnapshotRequest)(nil), // 55: atelet.PromoteSnapshotRequest - (*PromoteSnapshotResponse)(nil), // 56: atelet.PromoteSnapshotResponse - (*DetachActorRequest)(nil), // 57: atelet.DetachActorRequest - (*DetachActorResponse)(nil), // 58: atelet.DetachActorResponse - nil, // 59: atelet.ExternalVolumeSource.VolumeContextEntry - nil, // 60: atelet.ExternalVolumeSource.PublishContextEntry - (*timestamppb.Timestamp)(nil), // 61: google.protobuf.Timestamp + (SnapshotSharing)(0), // 2: atelet.SnapshotSharing + (UploadMode)(0), // 3: atelet.UploadMode + (*RegisterWorkerRequest)(nil), // 4: atelet.RegisterWorkerRequest + (*SandboxRuntime)(nil), // 5: atelet.SandboxRuntime + (*VersionedSandboxCompat)(nil), // 6: atelet.VersionedSandboxCompat + (*AttributeEntry)(nil), // 7: atelet.AttributeEntry + (*WorkerResources)(nil), // 8: atelet.WorkerResources + (*ResourceRequirements)(nil), // 9: atelet.ResourceRequirements + (*Resources)(nil), // 10: atelet.Resources + (*RegisterWorkerResponse)(nil), // 11: atelet.RegisterWorkerResponse + (*RequestActorSuspendRequest)(nil), // 12: atelet.RequestActorSuspendRequest + (*RequestActorSuspendResponse)(nil), // 13: atelet.RequestActorSuspendResponse + (*ReportActorCrashedRequest)(nil), // 14: atelet.ReportActorCrashedRequest + (*ReportActorCrashedResponse)(nil), // 15: atelet.ReportActorCrashedResponse + (*GetActorEgressPolicyRequest)(nil), // 16: atelet.GetActorEgressPolicyRequest + (*GetActorEgressPolicyResponse)(nil), // 17: atelet.GetActorEgressPolicyResponse + (*EgressPolicy)(nil), // 18: atelet.EgressPolicy + (*EgressRule)(nil), // 19: atelet.EgressRule + (*EgressDestinations)(nil), // 20: atelet.EgressDestinations + (*EgressPorts)(nil), // 21: atelet.EgressPorts + (*EgressAllPorts)(nil), // 22: atelet.EgressAllPorts + (*MintActorCertificateRequest)(nil), // 23: atelet.MintActorCertificateRequest + (*MintActorCertificateResponse)(nil), // 24: atelet.MintActorCertificateResponse + (*TerminateRequest)(nil), // 25: atelet.TerminateRequest + (*TerminateResponse)(nil), // 26: atelet.TerminateResponse + (*RunRequest)(nil), // 27: atelet.RunRequest + (*EgressGateway)(nil), // 28: atelet.EgressGateway + (*WorkloadSpec)(nil), // 29: atelet.WorkloadSpec + (*DurableDirVolumeSource)(nil), // 30: atelet.DurableDirVolumeSource + (*ExternalVolumeSource)(nil), // 31: atelet.ExternalVolumeSource + (*ImageVolumeSource)(nil), // 32: atelet.ImageVolumeSource + (*ActorMetadataItem)(nil), // 33: atelet.ActorMetadataItem + (*ActorMetadataDataSource)(nil), // 34: atelet.ActorMetadataDataSource + (*TrustBundleDataSource)(nil), // 35: atelet.TrustBundleDataSource + (*SystemInfoDataSource)(nil), // 36: atelet.SystemInfoDataSource + (*SystemInfoVolumeSource)(nil), // 37: atelet.SystemInfoVolumeSource + (*Volume)(nil), // 38: atelet.Volume + (*VolumeMount)(nil), // 39: atelet.VolumeMount + (*Container)(nil), // 40: atelet.Container + (*SecurityContext)(nil), // 41: atelet.SecurityContext + (*Capabilities)(nil), // 42: atelet.Capabilities + (*EnvVar)(nil), // 43: atelet.EnvVar + (*WakeupProbe)(nil), // 44: atelet.WakeupProbe + (*HttpGetAction)(nil), // 45: atelet.HttpGetAction + (*RunResponse)(nil), // 46: atelet.RunResponse + (*ObjectStorage)(nil), // 47: atelet.ObjectStorage + (*Snapshot)(nil), // 48: atelet.Snapshot + (*SnapshotReport)(nil), // 49: atelet.SnapshotReport + (*ActorSnapshotState)(nil), // 50: atelet.ActorSnapshotState + (*UploadConfig)(nil), // 51: atelet.UploadConfig + (*CheckpointRequest)(nil), // 52: atelet.CheckpointRequest + (*CheckpointResponse)(nil), // 53: atelet.CheckpointResponse + (*RestoreRequest)(nil), // 54: atelet.RestoreRequest + (*RestoreResponse)(nil), // 55: atelet.RestoreResponse + (*PromoteSnapshotRequest)(nil), // 56: atelet.PromoteSnapshotRequest + (*PromoteSnapshotResponse)(nil), // 57: atelet.PromoteSnapshotResponse + (*DetachActorRequest)(nil), // 58: atelet.DetachActorRequest + (*DetachActorResponse)(nil), // 59: atelet.DetachActorResponse + nil, // 60: atelet.ExternalVolumeSource.VolumeContextEntry + nil, // 61: atelet.ExternalVolumeSource.PublishContextEntry + (*timestamppb.Timestamp)(nil), // 62: google.protobuf.Timestamp } var file_atelet_proto_depIdxs = []int32{ - 7, // 0: atelet.RegisterWorkerRequest.capacity:type_name -> atelet.WorkerResources - 4, // 1: atelet.RegisterWorkerRequest.default_runtime:type_name -> atelet.SandboxRuntime - 4, // 2: atelet.RegisterWorkerRequest.restorable_runtimes:type_name -> atelet.SandboxRuntime - 5, // 3: atelet.SandboxRuntime.version:type_name -> atelet.VersionedSandboxCompat - 6, // 4: atelet.VersionedSandboxCompat.attributes:type_name -> atelet.AttributeEntry - 9, // 5: atelet.ResourceRequirements.limits:type_name -> atelet.Resources - 17, // 6: atelet.GetActorEgressPolicyResponse.egress_policy:type_name -> atelet.EgressPolicy - 18, // 7: atelet.EgressPolicy.rules:type_name -> atelet.EgressRule - 19, // 8: atelet.EgressRule.http:type_name -> atelet.EgressDestinations - 19, // 9: atelet.EgressRule.https:type_name -> atelet.EgressDestinations - 19, // 10: atelet.EgressRule.tls_passthrough:type_name -> atelet.EgressDestinations - 20, // 11: atelet.EgressDestinations.ports:type_name -> atelet.EgressPorts - 21, // 12: atelet.EgressPorts.all:type_name -> atelet.EgressAllPorts - 28, // 13: atelet.TerminateRequest.spec:type_name -> atelet.WorkloadSpec - 28, // 14: atelet.RunRequest.spec:type_name -> atelet.WorkloadSpec - 27, // 15: atelet.RunRequest.egress_gateway:type_name -> atelet.EgressGateway - 8, // 16: atelet.RunRequest.resources:type_name -> atelet.ResourceRequirements - 4, // 17: atelet.RunRequest.sandbox_runtime:type_name -> atelet.SandboxRuntime - 39, // 18: atelet.WorkloadSpec.containers:type_name -> atelet.Container - 37, // 19: atelet.WorkloadSpec.volumes:type_name -> atelet.Volume - 59, // 20: atelet.ExternalVolumeSource.volume_context:type_name -> atelet.ExternalVolumeSource.VolumeContextEntry - 60, // 21: atelet.ExternalVolumeSource.publish_context:type_name -> atelet.ExternalVolumeSource.PublishContextEntry + 8, // 0: atelet.RegisterWorkerRequest.capacity:type_name -> atelet.WorkerResources + 5, // 1: atelet.RegisterWorkerRequest.default_runtime:type_name -> atelet.SandboxRuntime + 5, // 2: atelet.RegisterWorkerRequest.restorable_runtimes:type_name -> atelet.SandboxRuntime + 6, // 3: atelet.SandboxRuntime.version:type_name -> atelet.VersionedSandboxCompat + 7, // 4: atelet.VersionedSandboxCompat.attributes:type_name -> atelet.AttributeEntry + 10, // 5: atelet.ResourceRequirements.limits:type_name -> atelet.Resources + 18, // 6: atelet.GetActorEgressPolicyResponse.egress_policy:type_name -> atelet.EgressPolicy + 19, // 7: atelet.EgressPolicy.rules:type_name -> atelet.EgressRule + 20, // 8: atelet.EgressRule.http:type_name -> atelet.EgressDestinations + 20, // 9: atelet.EgressRule.https:type_name -> atelet.EgressDestinations + 20, // 10: atelet.EgressRule.tls_passthrough:type_name -> atelet.EgressDestinations + 21, // 11: atelet.EgressDestinations.ports:type_name -> atelet.EgressPorts + 22, // 12: atelet.EgressPorts.all:type_name -> atelet.EgressAllPorts + 29, // 13: atelet.TerminateRequest.spec:type_name -> atelet.WorkloadSpec + 29, // 14: atelet.RunRequest.spec:type_name -> atelet.WorkloadSpec + 28, // 15: atelet.RunRequest.egress_gateway:type_name -> atelet.EgressGateway + 9, // 16: atelet.RunRequest.resources:type_name -> atelet.ResourceRequirements + 5, // 17: atelet.RunRequest.sandbox_runtime:type_name -> atelet.SandboxRuntime + 40, // 18: atelet.WorkloadSpec.containers:type_name -> atelet.Container + 38, // 19: atelet.WorkloadSpec.volumes:type_name -> atelet.Volume + 60, // 20: atelet.ExternalVolumeSource.volume_context:type_name -> atelet.ExternalVolumeSource.VolumeContextEntry + 61, // 21: atelet.ExternalVolumeSource.publish_context:type_name -> atelet.ExternalVolumeSource.PublishContextEntry 0, // 22: atelet.ActorMetadataItem.field:type_name -> atelet.ActorMetadataField - 32, // 23: atelet.ActorMetadataDataSource.items:type_name -> atelet.ActorMetadataItem - 33, // 24: atelet.SystemInfoDataSource.actor_metadata:type_name -> atelet.ActorMetadataDataSource - 34, // 25: atelet.SystemInfoDataSource.trust_bundle:type_name -> atelet.TrustBundleDataSource - 35, // 26: atelet.SystemInfoVolumeSource.data_sources:type_name -> atelet.SystemInfoDataSource - 29, // 27: atelet.Volume.durable_dir:type_name -> atelet.DurableDirVolumeSource - 30, // 28: atelet.Volume.external:type_name -> atelet.ExternalVolumeSource - 36, // 29: atelet.Volume.system_info:type_name -> atelet.SystemInfoVolumeSource - 31, // 30: atelet.Volume.image:type_name -> atelet.ImageVolumeSource - 42, // 31: atelet.Container.env:type_name -> atelet.EnvVar - 43, // 32: atelet.Container.wakeup_probe:type_name -> atelet.WakeupProbe - 38, // 33: atelet.Container.volume_mounts:type_name -> atelet.VolumeMount - 40, // 34: atelet.Container.security_context:type_name -> atelet.SecurityContext - 8, // 35: atelet.Container.resources:type_name -> atelet.ResourceRequirements - 41, // 36: atelet.SecurityContext.capabilities:type_name -> atelet.Capabilities - 44, // 37: atelet.WakeupProbe.http_get:type_name -> atelet.HttpGetAction - 1, // 38: atelet.Snapshot.fidelity:type_name -> atelet.SnapshotFidelity - 46, // 39: atelet.Snapshot.object:type_name -> atelet.ObjectStorage - 47, // 40: atelet.SnapshotReport.snapshot:type_name -> atelet.Snapshot - 61, // 41: atelet.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp - 61, // 42: atelet.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp - 48, // 43: atelet.ActorSnapshotState.latest_local:type_name -> atelet.SnapshotReport - 48, // 44: atelet.ActorSnapshotState.uploading:type_name -> atelet.SnapshotReport - 48, // 45: atelet.ActorSnapshotState.latest_durable:type_name -> atelet.SnapshotReport - 61, // 46: atelet.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp - 2, // 47: atelet.UploadConfig.mode:type_name -> atelet.UploadMode - 28, // 48: atelet.CheckpointRequest.spec:type_name -> atelet.WorkloadSpec - 47, // 49: atelet.CheckpointRequest.snapshot:type_name -> atelet.Snapshot - 50, // 50: atelet.CheckpointRequest.upload_config:type_name -> atelet.UploadConfig - 49, // 51: atelet.CheckpointResponse.state:type_name -> atelet.ActorSnapshotState - 28, // 52: atelet.RestoreRequest.spec:type_name -> atelet.WorkloadSpec - 47, // 53: atelet.RestoreRequest.snapshot:type_name -> atelet.Snapshot - 27, // 54: atelet.RestoreRequest.egress_gateway:type_name -> atelet.EgressGateway - 8, // 55: atelet.RestoreRequest.resources:type_name -> atelet.ResourceRequirements - 4, // 56: atelet.RestoreRequest.sandbox_runtime:type_name -> atelet.SandboxRuntime - 47, // 57: atelet.PromoteSnapshotRequest.snapshot:type_name -> atelet.Snapshot - 49, // 58: atelet.PromoteSnapshotResponse.state:type_name -> atelet.ActorSnapshotState - 49, // 59: atelet.DetachActorResponse.state:type_name -> atelet.ActorSnapshotState - 22, // 60: atelet.WorkerSupport.MintActorCertificate:input_type -> atelet.MintActorCertificateRequest - 3, // 61: atelet.WorkerSupport.RegisterWorker:input_type -> atelet.RegisterWorkerRequest - 11, // 62: atelet.WorkerSupport.RequestActorSuspend:input_type -> atelet.RequestActorSuspendRequest - 15, // 63: atelet.WorkerSupport.GetActorEgressPolicy:input_type -> atelet.GetActorEgressPolicyRequest - 13, // 64: atelet.WorkerSupport.ReportActorCrashed:input_type -> atelet.ReportActorCrashedRequest - 26, // 65: atelet.Atelet.Run:input_type -> atelet.RunRequest - 51, // 66: atelet.Atelet.Checkpoint:input_type -> atelet.CheckpointRequest - 53, // 67: atelet.Atelet.Restore:input_type -> atelet.RestoreRequest - 55, // 68: atelet.Atelet.PromoteSnapshot:input_type -> atelet.PromoteSnapshotRequest - 57, // 69: atelet.Atelet.DetachActor:input_type -> atelet.DetachActorRequest - 24, // 70: atelet.Atelet.Terminate:input_type -> atelet.TerminateRequest - 23, // 71: atelet.WorkerSupport.MintActorCertificate:output_type -> atelet.MintActorCertificateResponse - 10, // 72: atelet.WorkerSupport.RegisterWorker:output_type -> atelet.RegisterWorkerResponse - 12, // 73: atelet.WorkerSupport.RequestActorSuspend:output_type -> atelet.RequestActorSuspendResponse - 16, // 74: atelet.WorkerSupport.GetActorEgressPolicy:output_type -> atelet.GetActorEgressPolicyResponse - 14, // 75: atelet.WorkerSupport.ReportActorCrashed:output_type -> atelet.ReportActorCrashedResponse - 45, // 76: atelet.Atelet.Run:output_type -> atelet.RunResponse - 52, // 77: atelet.Atelet.Checkpoint:output_type -> atelet.CheckpointResponse - 54, // 78: atelet.Atelet.Restore:output_type -> atelet.RestoreResponse - 56, // 79: atelet.Atelet.PromoteSnapshot:output_type -> atelet.PromoteSnapshotResponse - 58, // 80: atelet.Atelet.DetachActor:output_type -> atelet.DetachActorResponse - 25, // 81: atelet.Atelet.Terminate:output_type -> atelet.TerminateResponse - 71, // [71:82] is the sub-list for method output_type - 60, // [60:71] is the sub-list for method input_type - 60, // [60:60] is the sub-list for extension type_name - 60, // [60:60] is the sub-list for extension extendee - 0, // [0:60] is the sub-list for field type_name + 33, // 23: atelet.ActorMetadataDataSource.items:type_name -> atelet.ActorMetadataItem + 34, // 24: atelet.SystemInfoDataSource.actor_metadata:type_name -> atelet.ActorMetadataDataSource + 35, // 25: atelet.SystemInfoDataSource.trust_bundle:type_name -> atelet.TrustBundleDataSource + 36, // 26: atelet.SystemInfoVolumeSource.data_sources:type_name -> atelet.SystemInfoDataSource + 30, // 27: atelet.Volume.durable_dir:type_name -> atelet.DurableDirVolumeSource + 31, // 28: atelet.Volume.external:type_name -> atelet.ExternalVolumeSource + 37, // 29: atelet.Volume.system_info:type_name -> atelet.SystemInfoVolumeSource + 32, // 30: atelet.Volume.image:type_name -> atelet.ImageVolumeSource + 43, // 31: atelet.Container.env:type_name -> atelet.EnvVar + 44, // 32: atelet.Container.wakeup_probe:type_name -> atelet.WakeupProbe + 39, // 33: atelet.Container.volume_mounts:type_name -> atelet.VolumeMount + 41, // 34: atelet.Container.security_context:type_name -> atelet.SecurityContext + 9, // 35: atelet.Container.resources:type_name -> atelet.ResourceRequirements + 42, // 36: atelet.SecurityContext.capabilities:type_name -> atelet.Capabilities + 45, // 37: atelet.WakeupProbe.http_get:type_name -> atelet.HttpGetAction + 2, // 38: atelet.ObjectStorage.sharing:type_name -> atelet.SnapshotSharing + 1, // 39: atelet.Snapshot.fidelity:type_name -> atelet.SnapshotFidelity + 47, // 40: atelet.Snapshot.object:type_name -> atelet.ObjectStorage + 48, // 41: atelet.SnapshotReport.snapshot:type_name -> atelet.Snapshot + 62, // 42: atelet.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp + 62, // 43: atelet.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp + 49, // 44: atelet.ActorSnapshotState.latest_local:type_name -> atelet.SnapshotReport + 49, // 45: atelet.ActorSnapshotState.uploading:type_name -> atelet.SnapshotReport + 49, // 46: atelet.ActorSnapshotState.latest_durable:type_name -> atelet.SnapshotReport + 62, // 47: atelet.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp + 3, // 48: atelet.UploadConfig.mode:type_name -> atelet.UploadMode + 29, // 49: atelet.CheckpointRequest.spec:type_name -> atelet.WorkloadSpec + 48, // 50: atelet.CheckpointRequest.snapshot:type_name -> atelet.Snapshot + 51, // 51: atelet.CheckpointRequest.upload_config:type_name -> atelet.UploadConfig + 50, // 52: atelet.CheckpointResponse.state:type_name -> atelet.ActorSnapshotState + 29, // 53: atelet.RestoreRequest.spec:type_name -> atelet.WorkloadSpec + 48, // 54: atelet.RestoreRequest.snapshot:type_name -> atelet.Snapshot + 28, // 55: atelet.RestoreRequest.egress_gateway:type_name -> atelet.EgressGateway + 9, // 56: atelet.RestoreRequest.resources:type_name -> atelet.ResourceRequirements + 5, // 57: atelet.RestoreRequest.sandbox_runtime:type_name -> atelet.SandboxRuntime + 48, // 58: atelet.PromoteSnapshotRequest.snapshot:type_name -> atelet.Snapshot + 50, // 59: atelet.PromoteSnapshotResponse.state:type_name -> atelet.ActorSnapshotState + 50, // 60: atelet.DetachActorResponse.state:type_name -> atelet.ActorSnapshotState + 23, // 61: atelet.WorkerSupport.MintActorCertificate:input_type -> atelet.MintActorCertificateRequest + 4, // 62: atelet.WorkerSupport.RegisterWorker:input_type -> atelet.RegisterWorkerRequest + 12, // 63: atelet.WorkerSupport.RequestActorSuspend:input_type -> atelet.RequestActorSuspendRequest + 16, // 64: atelet.WorkerSupport.GetActorEgressPolicy:input_type -> atelet.GetActorEgressPolicyRequest + 14, // 65: atelet.WorkerSupport.ReportActorCrashed:input_type -> atelet.ReportActorCrashedRequest + 27, // 66: atelet.Atelet.Run:input_type -> atelet.RunRequest + 52, // 67: atelet.Atelet.Checkpoint:input_type -> atelet.CheckpointRequest + 54, // 68: atelet.Atelet.Restore:input_type -> atelet.RestoreRequest + 56, // 69: atelet.Atelet.PromoteSnapshot:input_type -> atelet.PromoteSnapshotRequest + 58, // 70: atelet.Atelet.DetachActor:input_type -> atelet.DetachActorRequest + 25, // 71: atelet.Atelet.Terminate:input_type -> atelet.TerminateRequest + 24, // 72: atelet.WorkerSupport.MintActorCertificate:output_type -> atelet.MintActorCertificateResponse + 11, // 73: atelet.WorkerSupport.RegisterWorker:output_type -> atelet.RegisterWorkerResponse + 13, // 74: atelet.WorkerSupport.RequestActorSuspend:output_type -> atelet.RequestActorSuspendResponse + 17, // 75: atelet.WorkerSupport.GetActorEgressPolicy:output_type -> atelet.GetActorEgressPolicyResponse + 15, // 76: atelet.WorkerSupport.ReportActorCrashed:output_type -> atelet.ReportActorCrashedResponse + 46, // 77: atelet.Atelet.Run:output_type -> atelet.RunResponse + 53, // 78: atelet.Atelet.Checkpoint:output_type -> atelet.CheckpointResponse + 55, // 79: atelet.Atelet.Restore:output_type -> atelet.RestoreResponse + 57, // 80: atelet.Atelet.PromoteSnapshot:output_type -> atelet.PromoteSnapshotResponse + 59, // 81: atelet.Atelet.DetachActor:output_type -> atelet.DetachActorResponse + 26, // 82: atelet.Atelet.Terminate:output_type -> atelet.TerminateResponse + 72, // [72:83] is the sub-list for method output_type + 61, // [61:72] is the sub-list for method input_type + 61, // [61:61] is the sub-list for extension type_name + 61, // [61:61] is the sub-list for extension extendee + 0, // [0:61] is the sub-list for field type_name } func init() { file_atelet_proto_init() } @@ -4387,7 +4465,7 @@ func file_atelet_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_atelet_proto_rawDesc), len(file_atelet_proto_rawDesc)), - NumEnums: 3, + NumEnums: 4, NumMessages: 58, NumExtensions: 0, NumServices: 2, diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index 495a4e0cae..b1ab598586 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -832,6 +832,28 @@ message ObjectStorage { // +k8s:maxLength=2048 // +k8s:customValidation # a parseable snapshot URI string snapshot_uri = 1; + + // How the snapshot at snapshot_uri may be consumed, as the control plane + // knows it. Read on Restore, where it decides whether atelet may serve the + // snapshot's files from its node-local cache; meaningless where atelet + // writes to the snapshot. + // + // +k8s:optional + // +k8s:maximum=2 # keep this in sync with the SnapshotSharing enum + SnapshotSharing sharing = 2; +} + +// SnapshotSharing is how a snapshot in object storage may be consumed, as +// known by the control plane. atelet never infers it from the URI. +enum SnapshotSharing { + // Not stated: treated as PRIVATE. + SNAPSHOT_SHARING_UNSPECIFIED = 0; + // Read by one actor and replaced by its next suspend. Never cached. + SNAPSHOT_SHARING_PRIVATE = 1; + // Immutable once published and read by many actors, such as a tag's + // snapshot borrowed at creation. atelet may serve its files from the + // node-local snapshot cache. + SNAPSHOT_SHARING_SHARED = 2; } // Snapshot identifies one snapshot of an actor, and where its durable copy From c0944b76285ecbfa981a6dae3e3323129d92c029 Mon Sep 17 00:00:00 2001 From: Da Huang <127548902+git286@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:10:43 +0000 Subject: [PATCH 10/63] docs/observability: fix stale and wrong information (#2886) - kubectl ate logs uses the singular 'actor' subcommand; the not-running error names atespace/name; quote the crash log message exactly. - GKE telemetry goes to the GKE managed OTel collector, not straight to GMP. - Workers normally export through the atelet relay, so an endpoint change reaches them when atelet restarts; the controller copy is the fallback. - List why atelet may start no relay (https, conflicting per-signal endpoints or compression, unparsable headers), per-signal endpoints as a fallback destination, and that ate-controller injects the endpoint only when it has one. A socket left by an earlier atelet keeps ateworker on the relay, failing every export instead of falling back. - The probes moved to :9091 (#2424); #1558 was closed unmerged. - Drop the removed pause from the lifecycle and snapshot-kind text. - The registry table now lists events.yaml. - Kind Prometheus does not show the metrics that ateworker and atecontroller only push over OTLP; the credential provider does not read ate-otel-config, so its OTLP push never reaches the collector. - Tracing is head-sampled, and the roadmap metrics already exist. --- cmd/atelet/metrics.go | 5 ++-- docs/metrics/registry/metrics.yaml | 13 +++++++---- docs/observability.md | 37 ++++++++++++++++-------------- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/cmd/atelet/metrics.go b/cmd/atelet/metrics.go index ed31bad2f5..84f8ff835d 100644 --- a/cmd/atelet/metrics.go +++ b/cmd/atelet/metrics.go @@ -166,8 +166,9 @@ func restoreSnapshotKind(directLocal bool, rec *sandboxAssetsRecord) string { return ateattr.SnapshotKindLatest } -// checkpointSnapshotKind classifies which snapshot a checkpoint writes: a pause -// writes the node-local one, and a suspend writes a durable latest snapshot. +// checkpointSnapshotKind classifies which snapshot a checkpoint writes: an +// asynchronous suspend writes the node-local one, and a synchronous suspend +// writes a durable latest snapshot. func checkpointSnapshotKind(req *ateletpb.CheckpointRequest) string { if req.GetUploadConfig().GetMode() == ateletpb.UploadMode_UPLOAD_MODE_DEFERRED { return ateattr.SnapshotKindLocal diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index 9204478f96..4121648c17 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -104,14 +104,14 @@ groups: stability: development value: suspending brief: > - ateapi is writing a snapshot: durable for a suspend, - node-local for a pause. + ateapi is writing a snapshot: node-local for an asynchronous + suspend, durable for a synchronous suspend. - id: suspended stability: development value: suspended brief: > - The actor holds no worker. Its latest snapshot is durable, or - node-local after a pause. + The actor holds no worker. Its latest snapshot is durable, + node-local, or both. - id: crashed stability: development value: crashed @@ -298,7 +298,10 @@ groups: - id: local stability: development value: local - brief: A snapshot on the node. A pause writes it. + brief: > + The copy on the node. An asynchronous suspend writes it. A + resume reads it while the node still has it, after either kind + of suspend and whether or not the upload has finished. - id: latest stability: development value: latest diff --git a/docs/observability.md b/docs/observability.md index 41096503d8..60fa15aae8 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -14,7 +14,7 @@ To make underlying infrastructure transitions transparent, Agent Substrate estab * `ate.template.atespace`: The atespace of the actor's ActorTemplate (e.g., `ate-demo-counter`). * `ate.actor.container.name`: The name of the container within the actor that produced the log line (e.g., `counter`), so a multi-container actor's logs can be demultiplexed by container. Absent on the synthetic lifecycle records (`Actor starting`, `Actor restored`, …): those are about the actor, so no container produced them. The `runsc create`, `start`, and `restore` CLI output and the pause container's stdio are wrapped under `_pause`. This does not capture sentry or gofer output, which runsc directs to `/dev/null` when debug logging is disabled. -Currently, Agent Substrate automatically wraps container output and injects these metadata labels into **container logs**. For metrics and distributed tracing, Agent Substrate provides foundational system telemetry and on-demand request tracing, with roadmap plans to fully integrate actor-level correlation. +Currently, Agent Substrate automatically wraps container output and injects these metadata labels into **container logs**. For metrics and distributed tracing, Agent Substrate provides foundational system telemetry and head-sampled request tracing (10% by default, 1% at the router), with roadmap plans to fully integrate actor-level correlation. `ate.*` is reserved for Substrate. An actor's own log lines pass through untouched except for keys in that namespace, which are dropped before the record is written, so nothing a workload emits can be read as platform-issued attribution. @@ -41,7 +41,7 @@ If an actor is suspended or not assigned to a worker pod, the CLI informs you im ```bash $ kubectl ate logs actor test -a demo -Error: actor test is not currently running on any worker pod +Error: actor demo/test is not currently running on any worker pod ``` #### Example 2: Default Clean JSON Lines Output @@ -77,7 +77,7 @@ kubectl ate logs actor -a -c To view the pause container's runsc diagnostics explicitly: ```bash -kubectl ate logs actors -a -c _pause +kubectl ate logs actor -a -c _pause ``` `--source` selects a class of lines instead of a container. `containers` shows application container output with the lifecycle events removed; `_pause` remains excluded unless selected with `-c _pause`. `lifecycle` shows only the lifecycle events, which is the quickest way to see how often an actor has been suspended and restored on its current worker (a migration moves the actor to another pod, whose log holds none of the earlier workers' events): @@ -193,7 +193,7 @@ Creating an actor counts as a change. A new actor is born suspended, so it gets "trace_id":"4bf92f…","span_id":"00f067…","trace_flags":"01"} ``` -The counter carries no actor identity, so this record is the only way to attribute a crash to one agent. The decision-point line that precedes it (`Setting Actor to crashed due to error`) carries only `ate.atespace` and `ate.actor.name`: it is written before the Actor is loaded, so no uid exists yet. +The counter carries no actor identity, so this record is the only way to attribute a crash to one agent. The decision-point line that precedes it (`Setting Actor to crashed due to Atelet error`) carries only `ate.atespace` and `ate.actor.name`: it is written before the Actor is loaded, so no uid exists yet. ateworker-gvisor's `Sandbox has no processes left while the actor is hosted` is a WARN written when a running actor's gVisor sandbox has died: the sentry crashed, was killed, or was OOM-killed. The ateworker's usage sweep finds it, so it shows up within one sweep (`--usage-sample-interval`, a minute by default), once per activation. `ate.sandbox.oom_kills` is the cgroup's `oom_kill` count, so a non-zero value points at the kernel OOM killer: @@ -285,7 +285,7 @@ Agent Substrate emits foundational OpenTelemetry system and server metrics to mo | `ate.workerpool.ready_workers` | atecontroller | up/down counter | number of worker pods currently ready for a WorkerPool, from `status.readyReplicas` (labels `ate.workerpool.namespace`, `ate.workerpool.name`) | | `ate.workerpool.workers` | ateapi | up/down counter | live worker count per pool, split by state (`idle`/`partial`/`at_capacity`/`unschedulable`) and sandbox class to provide fleet capacity and saturation at a glance | -| `ate.actor.lifecycle.operation.duration` | ateapi | histogram | how long each actor operation (create/resume/suspend/delete/revert; a pause is labeled suspend) takes and whether it failed (`error.type` present = failure, absent = success); labeled by operation, template, pool (`ate.workerpool.namespace` + `ate.workerpool.name`), sandbox class, snapshot kind on resume, and snapshot fidelity on resume and suspend; already-running resume no-ops are not recorded so the histogram tracks actual activations, not router traffic | +| `ate.actor.lifecycle.operation.duration` | ateapi | histogram | how long each actor operation (create/resume/suspend/delete/revert) takes and whether it failed (`error.type` present = failure, absent = success); labeled by operation, template, pool (`ate.workerpool.namespace` + `ate.workerpool.name`), sandbox class, snapshot kind on resume, and snapshot fidelity on resume and suspend; already-running resume no-ops are not recorded so the histogram tracks actual activations, not router traffic | | `ate.scheduler.assignment.duration` | ateapi | histogram | time of one attempt to assign a worker to an actor, with one record for each resume (ateapi retries an attempt that loses a race for a worker and records only the last attempt), with the outcome (`assigned` / `no_capacity` / `error`), the assigned pool (`ate.workerpool.namespace` + `ate.workerpool.name`) and sandbox class to catch scheduling latency and capacity starvation problems | | `ate.actor.restore.duration` | atelet | histogram | how long each phase of a restore takes on the worker node, which is where cold-start latency actually goes once ateapi hands off (labels `ate.snapshot.phase`, `ate.snapshot.kind`, `ate.snapshot.fidelity`, `ate.template.atespace`, `ate.template.name`, `ate.sandbox.class`) | | `ate.actor.checkpoint.duration` | atelet | histogram | the same phase breakdown for writing a snapshot, so a slow suspend can be attributed to ateworker or to the upload (same labels as the restore histogram) | @@ -323,7 +323,7 @@ For the `ate.filecache.*` instruments, one series per cache under `ate.filecache **No keys** means the operation has no pool. The actor-centric instruments omit the pair, so a crash before the actor reached a worker, or the `no_capacity` outcome, names no pool. No instrument sends the pair with both keys empty. The three snapshot labels are orthogonal and mean the same thing on every histogram that carries them: -* `ate.snapshot.kind`: which snapshot the operation reads or writes. `local` (node-local, written by a pause), `latest` (a durable snapshot, including a template's golden snapshot), or `boot` (from scratch, so it never appears on the atelet histograms). +* `ate.snapshot.kind`: which snapshot the operation reads or writes. `local` (the copy on the node: an asynchronous suspend writes it, and a resume reads it while the node still has it, whether or not its upload to durable storage has finished. A synchronous suspend is `latest`, but it also keeps a copy on the node, so the resume after it can be `local`), `latest` (a durable snapshot, including a template's golden snapshot), or `boot` (from scratch, so it never appears on the atelet histograms). * `ate.snapshot.fidelity`: what state it holds. `volumes`, `rootfs`, or `memory`. * `ate.snapshot.phase`: which step was timed. `volume_mount`, `manifest_fetch`, `sandbox_assets`, `download`, `oci_unpack`, `worker_restore` on restore; `sandbox_assets`, `worker_checkpoint`, `persist`, `cleanup` on checkpoint; `total` on both. @@ -354,12 +354,13 @@ weaver registry check -r docs/metrics/registry # the same command, direct **To add or change an instrument:** follow [Metrics Best Practices](dev/best-practices/metrics.md), which covers the instrument choice, the labels, the Go, the test, and the registry entry; then change `metrics.yaml` and run the script. -Two files, and not one: +Four files, and not one: | File | Content | Who reads it | |---|---|---| | `docs/metrics/registry/manifest.yaml` | The name and the schema URL of the registry. | Weaver | | `docs/metrics/registry/metrics.yaml` | The `groups`: each metric, each attribute. | Weaver | +| `docs/metrics/registry/events.yaml` | The [actor events](#the-same-records-over-otlp). | Weaver, `TestEventsMatchTheRegistry` | | `docs/metrics/substrate.yaml` | The rules of Substrate that Weaver cannot hold. | A person, an agent | Weaver permits only `groups` and `imports` at the top level of a registry file, and it refuses a file that has any other top-level key. Thus `substrate.yaml` is beside the registry directory and not in it. It records: @@ -386,7 +387,7 @@ A queue that has never processed an item bridges as an exponential histogram wit ### Scraping instead of pushing -ateapi, atelet, atenet-router and the credential provider also serve every instrument on their Prometheus `/metrics` endpoint. A cluster that scrapes those endpoints sets `OTEL_METRICS_EXPORTER=prometheus` on the components, so each series reaches the backend once. With `prometheus` the components install no OTLP metric reader and keep the Prometheus one; traces and logs are unaffected. `none` stops the push too. For now `/metrics` stays on for every value, including `otlp` and `none`. This is temporary until the probes move off `:9090`; `otlp` and `none` will then turn it off ([#2067](https://github.com/agent-substrate/substrate/issues/2067)). The variable takes a comma-separated list, with the same rules as `OTEL_LOGS_EXPORTER`. Unset is `otlp,prometheus`, or `otlp` alone on ateworker. `otlp` keeps the push, and `prometheus` or `none` stops it. ateworker serves no `/metrics`, so there `prometheus` is skipped as unknown with a warning, and the push stays on unless the variable is `none`. +ateapi, atelet, atenet-router and the credential provider also serve every instrument on their Prometheus `/metrics` endpoint. A cluster that scrapes those endpoints sets `OTEL_METRICS_EXPORTER=prometheus` on the components, so each series reaches the backend once. With `prometheus` the components install no OTLP metric reader and keep the Prometheus one; traces and logs are unaffected. `none` stops the push too. For now `/metrics` stays on for every value, including `otlp` and `none`. The health probes have moved to their own server on `:9091`, so `:9090` serves only `/metrics`; turning it off for `otlp` and `none` is tracked in [#2067](https://github.com/agent-substrate/substrate/issues/2067). The variable takes a comma-separated list, with the same rules as `OTEL_LOGS_EXPORTER`. Unset is `otlp,prometheus`, or `otlp` alone on ateworker. `otlp` keeps the push, and `prometheus` or `none` stops it. ateworker serves no `/metrics`, so there `prometheus` is skipped as unknown with a warning, and the push stays on unless the variable is `none`. `ate-controller` does not propagate its own value to the worker pods: ateworker serves no `/metrics`, and `none` would leave it exporting no metrics at all. To set the variable on ateworker, set `ATE_WORKER_OTEL_METRICS_EXPORTER` in `ate-otel-config` (or the controller's `--otel-metrics-exporter` flag). @@ -406,11 +407,13 @@ To explore metrics locally: 2. **Open the Prometheus UI** in your web browser: [http://localhost:9090](http://localhost:9090) -3. **Query metrics**: Run `up` to confirm each component is scraped (one series per target, value `1`), then explore the `rpc_*` series via the expression browser's autocomplete. **Status > Targets** lists the discovered pods. +3. **Query metrics**: Run `up` to confirm each scraped target is up (one series per target, value `1`), then explore the `rpc_*` series via the expression browser's autocomplete. **Status > Targets** lists the discovered pods. > **Note:** Storage is ephemeral (`emptyDir`), so metrics are lost when the Prometheus pod restarts. -> **Roadmap Note (Actor-Level Metrics):** A comprehensive metrics roadmap is under active development to support both system operators and workload analysis. Planned OpenTelemetry instrumentation focuses on control plane latency, state snapshot performance, fleet utilization density, and enriching metrics with standardized actor labels for seamless aggregation across pod transitions. +> **Note:** This Prometheus does not see every instrument. It scrapes the `/metrics` endpoint (`:9090`) of the pods annotated `prometheus.io/scrape`, which are `ate-api-server`, `atelet`, and `atenet-router`. From the collector it keeps only the `substrate_*` counts of the count connector and the collector's own `otelcol_*` counters ([`manifests/ate-install/kind/prometheus.yaml`](../manifests/ate-install/kind/prometheus.yaml)), so that the scraped services are not counted twice. The metrics that ateworker (including its `rpc.server.call.duration`) and atecontroller (`ate.workerpool.desired_workers`, `ate.workerpool.ready_workers`, and the bridged controller-runtime families) push over OTLP reach the collector but do not appear in this UI. The credential provider's metrics appear in neither: its pod carries no scrape annotation and does not read `ate-otel-config`, so its OTLP push goes to the SDK default `localhost:4317` and fails. + +> **Roadmap Note (Actor-Level Metrics):** The control plane latency, snapshot performance, and fleet utilization instruments above are in place. What remains is actor-level correlation: telemetry that an actor emits itself carries no Substrate-injected identity yet ([#853](https://github.com/agent-substrate/substrate/issues/853), [#761](https://github.com/agent-substrate/substrate/issues/761)). --- @@ -460,17 +463,17 @@ Telemetry is emitted the same way everywhere; only the backend differs between a | | Kind | GKE (Google Cloud) | |---|---|---| -| Path | service → in-cluster `opentelemetry-collector` | service → Google Managed Prometheus (GMP) | +| Path | service → in-cluster `opentelemetry-collector` | service → GKE managed OTel collector (`opentelemetry-collector.gke-managed-otel`) | | Metrics | collector Prometheus exporter on `:8889` | Google Cloud Monitoring | | Traces | Jaeger UI | Google Cloud Trace | | Logs | pod stdout; the [actor events](#the-same-records-over-otlp), ateapi's lifecycle records and the ateworkers' usage samples, also to the collector's `debug` exporter | pod stdout. No OTLP logs: `OTEL_LOGS_EXPORTER` is unset | | Dashboards | Not supported | Google Cloud Monitoring (see [Dashboards](#5-dashboards)) | -> In Kind, `ateapi`, `atelet`, `ate-controller`, and `atenet-router` are pointed at the in-cluster collector, and the controller propagates the endpoint to the worker pods it creates, so all component telemetry lands locally. +> In Kind, `ateapi`, `atelet`, `ate-controller`, `atenet-router`, and `atenet-egress` are pointed at the in-cluster collector, and the controller propagates the endpoint to the worker pods it creates, so the telemetry of these components and of the workers lands locally. The Kubernetes credential provider is the exception: it does not read the endpoint (see the [note on the Kind Prometheus](#2-metrics)). > -> Every component reads that endpoint from the shared `ate-otel-config` ConfigMap ([`manifests/ate-install/ate-otel-config.yaml`](../manifests/ate-install/ate-otel-config.yaml), with a Kind replacement of the same name under [`manifests/ate-install/kind/`](../manifests/ate-install/kind/ate-otel-config.yaml)). Editing it does not restart the pods that consume it — follow a change with `kubectl rollout restart`. +> Each of these components reads that endpoint from the shared `ate-otel-config` ConfigMap ([`manifests/ate-install/ate-otel-config.yaml`](../manifests/ate-install/ate-otel-config.yaml), with a Kind replacement of the same name under [`manifests/ate-install/kind/`](../manifests/ate-install/kind/ate-otel-config.yaml)). Editing it does not restart the pods that consume it — follow a change with `kubectl rollout restart`. > -> Workers don't read the ConfigMap at all — `ate-controller` copies the value into each worker pod at creation. A new endpoint reaches them only once the controller itself restarts, and that restart then rolls every WorkerPool Deployment, replacing the running workers along with the actors on them. +> Workers don't read the ConfigMap at all. They normally export through the [`atelet` relay](#the-ateworker-otlp-relay), which forwards to `atelet`'s own endpoint, so restarting `atelet` moves the relayed workers to a new endpoint. `ate-controller` also copies the value into each worker pod at creation, for the direct fallback. That copy changes only once the controller itself restarts, and that restart then rolls every WorkerPool Deployment, replacing the running workers along with the actors on them. ### The ateworker OTLP relay @@ -482,9 +485,9 @@ ateworker ──OTLP/gRPC over unix socket──► atelet relay ──OTLP/gRPC The socket sits in the `BasePath` hostPath already mounted into both, so nothing new is mounted. `atelet` is a DaemonSet, so every ateworker on a node shares one relay, and the many per-pod collector connections collapse into one per node. Four things motivate it: the worker pod runs untrusted agent code and will not need egress to the collector once direct fallback is phased out; the connection count drops; ateworker's own telemetry stays clear of the transparent egress redirect it installs for the actor; and `atelet` outlives the worker pod, so spans still queued at teardown are not lost with it. -The relay is best-effort. If the socket is absent when ateworker starts — `atelet` not up yet, `--otlp-relay-socket=""`, or no collector configured for the relay to forward to — ateworker logs it and exports directly to `OTEL_EXPORTER_OTLP_ENDPOINT`. Without that variable either, ateworker exports no traces, metrics, or logs, rather than dialing the SDK default `localhost:4317`, where no collector runs, and logs a warning per signal with `reason="no OTLP relay or endpoint"`. That fallback is decided once at startup, not per export, and is stamped on telemetry as the `ate.otlp.relay` resource attribute (`relay` vs `direct`). On an install that sets no endpoint for `ate-controller` to propagate, an ateworker that starts before its node's relay socket exists therefore exports nothing until the worker pod restarts. +The relay is best-effort. If the socket is absent when ateworker starts — `atelet` not up yet, `--otlp-relay-socket=""`, or no collector configured for the relay to forward to — ateworker logs it and exports directly to `OTEL_EXPORTER_OTLP_ENDPOINT` (or the per-signal `OTEL_EXPORTER_OTLP_{TRACES,METRICS,LOGS}_ENDPOINT`). `atelet` logs `Failed to create the OTLP relay` and starts no relay when its endpoint uses `https` (the relay forwards over plaintext gRPC and refuses to downgrade), when its per-signal endpoints or compression settings disagree (the relay carries every signal over one connection), or when its `OTEL_EXPORTER_OTLP_*_HEADERS` cannot be parsed. ateworker checks only that the socket file exists, so a socket left on the node by an earlier `atelet` still sends it to the relay: nothing serves the socket, and every export fails with `connection refused` instead of falling back. Without any endpoint either, ateworker exports no traces, metrics, or logs, rather than dialing the SDK default `localhost:4317`, where no collector runs, and logs a warning per signal with `reason="no OTLP relay or endpoint"`. That fallback is decided once at startup, not per export, and is stamped on telemetry as the `ate.otlp.relay` resource attribute (`relay` vs `direct`). On an install that sets no endpoint for `ate-controller` to propagate, an ateworker that starts before its node's relay socket exists therefore exports nothing until the worker pod restarts. -> **Note on Network Egress Lockdown:** Complete network policy lockdown of worker pod egress to the collector is planned as a Phase 2 milestone once the relay path is fully proven and direct fallback is deprecated. While the fallback path remains active, worker pods retain network egress to the collector and `ate-controller` continues to inject `OTEL_EXPORTER_OTLP_ENDPOINT`. +> **Note on Network Egress Lockdown:** Complete network policy lockdown of worker pod egress to the collector is planned as a Phase 2 milestone once the relay path is fully proven and direct fallback is deprecated. While the fallback path remains active, worker pods retain network egress to the collector and `ate-controller` continues to inject `OTEL_EXPORTER_OTLP_ENDPOINT` when it has one. For verified ateworker sources, the relay forwards each request verbatim rather than decoding and re-exporting, which is what keeps every ateworker its own service in Jaeger/GCP Trace instead of being absorbed into `atelet`'s. `ate-controller` injects `k8s.pod.name`, `k8s.namespace.name`, `k8s.pod.uid`, `k8s.node.name`, and `service.instance.id` directly into `OTEL_RESOURCE_ATTRIBUTES` via the Kubernetes Downward API, whether or not it has an endpoint to propagate; because the relay preserves resources verbatim, Kubernetes attributes remain intact even though the TCP connection to the collector originates from `atelet` rather than the worker pod IP (bypassing reliance on collector-side IP-based `k8sattributes` enrichment). @@ -520,7 +523,7 @@ An actor that already has a policy for other destinations keeps those rules and Three things to know when applying this: -* **It is per actor.** The policy is a resource under the actor, and nothing applies one by default today. Every actor that exports telemetry needs its own, created after the actor and before, or shortly after, it is resumed. A template-level default policy is proposed in [#1558](https://github.com/agent-substrate/substrate/pull/1558); the underlying ask is [#1430](https://github.com/agent-substrate/substrate/issues/1430). +* **It is per actor.** The policy is a resource under the actor, and nothing applies one by default today. Every actor that exports telemetry needs its own, created after the actor and before, or shortly after, it is resumed. A template-level default policy is requested in [#1430](https://github.com/agent-substrate/substrate/issues/1430). * **A running actor recovers slowly.** If the policy is created while the actor is already exporting and failing, the SDK's gRPC client is in reconnect backoff, the sandbox's DNS relay takes up to 5s to notice the policy, and the gateway caches the missing policy for `--egress-policy-cache-ttl` (10s by default). Expect the first successful export 45s to two minutes after the policy appears. An actor whose policy exists before it starts has no such delay. * **The collector sees the gateway, not the actor.** The TCP connection to the collector comes from the egress gateway pod, so a collector that enriches by source IP (the `k8sattributes` processor, which the GKE managed collector runs) labels the actor's spans and metrics with the gateway's `k8s.pod.name`, `k8s.deployment.name=atenet-egress`, and `k8s.namespace.name`. The actor's `service.name` and anything it puts in `OTEL_RESOURCE_ATTRIBUTES` survive. Set the actor's identity there yourself if you need it on the backend; the relay-based path that will do this for actors is tracked in [#853](https://github.com/agent-substrate/substrate/issues/853) and [#761](https://github.com/agent-substrate/substrate/issues/761). From 0d7f0d36aa7ea39128780147ab7acd5a49b6dad2 Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Tue, 20 Oct 2026 18:30:32 +0000 Subject: [PATCH 11/63] TestSyncer_Lifecycle no longer flakes on a worker that is still UNAVAILABLE (#2824) Part of #799. The test could read the worker after registration but before it was marked ACTIVE. It failed twice on main this week. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- .../internal/workersync/syncer_test.go | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/atecontroller/internal/workersync/syncer_test.go b/cmd/atecontroller/internal/workersync/syncer_test.go index f331bfecba..7b1623ecd2 100644 --- a/cmd/atecontroller/internal/workersync/syncer_test.go +++ b/cmd/atecontroller/internal/workersync/syncer_test.go @@ -194,6 +194,13 @@ func waitForWorker(t *testing.T, ctx context.Context, api *fakeControl, name str return got } +// workerActive reports whether w is registered and ACTIVE. The syncer registers +// a Worker UNAVAILABLE and marks it available in a later call, so a Worker that +// exists is not yet one that takes Actors. +func workerActive(w *ateapipb.Worker) bool { + return w.GetStatus().GetState() == ateapipb.WorkerState_WORKER_STATE_ACTIVE +} + func TestSyncer_Lifecycle(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -226,7 +233,7 @@ func TestSyncer_Lifecycle(t *testing.T) { t.Fatalf("update pod: %v", err) } - got := waitForWorker(t, ctx, api, testPodUID, func(w *ateapipb.Worker) bool { return w != nil }) + got := waitForWorker(t, ctx, api, testPodUID, workerActive) if want := []string{"127.0.0.1"}; !slices.Equal(got.GetIps(), want) { t.Errorf("worker ips = %q, want %q", got.GetIps(), want) } @@ -236,9 +243,6 @@ func TestSyncer_Lifecycle(t *testing.T) { if !maps.Equal(got.GetLabels(), map[string]string{"foo": "bar"}) { t.Errorf("worker labels = %v, want map[foo:bar]", got.GetLabels()) } - if got.GetStatus().GetState() != ateapipb.WorkerState_WORKER_STATE_ACTIVE { - t.Errorf("worker state = %v, want ACTIVE", got.GetStatus().GetState()) - } if err := fakeK8s.CoreV1().Pods(ns).Delete(ctx, podName, metav1.DeleteOptions{}); err != nil { t.Fatalf("delete pod: %v", err) @@ -1057,9 +1061,7 @@ func TestSyncer_SoftDelete_ViaInformer(t *testing.T) { if _, err := fakeK8s.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil { t.Fatalf("create pod: %v", err) } - waitForWorker(t, ctx, api, testPodUID, func(w *ateapipb.Worker) bool { - return w.GetStatus().GetState() == ateapipb.WorkerState_WORKER_STATE_ACTIVE - }) + waitForWorker(t, ctx, api, testPodUID, workerActive) // The fake clientset stores updates verbatim, so setting DeletionTimestamp // simulates a pod in graceful termination that is still in the cache. @@ -1164,9 +1166,7 @@ func TestSyncer_TerminalPod_ViaInformer(t *testing.T) { if _, err := fakeK8s.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil { t.Fatalf("create pod: %v", err) } - waitForWorker(t, ctx, api, testPodUID, func(w *ateapipb.Worker) bool { - return w.GetStatus().GetState() == ateapipb.WorkerState_WORKER_STATE_ACTIVE - }) + waitForWorker(t, ctx, api, testPodUID, workerActive) failed := pod.DeepCopy() failed.Status.Phase = corev1.PodFailed From 872ab2077df188368261eddd2cb7634d98a0d47d Mon Sep 17 00:00:00 2001 From: Bowei Du Date: Tue, 20 Oct 2026 18:32:18 +0000 Subject: [PATCH 12/63] Move internal/ateworker/tunnel to internal/atunnel (#2981) The tunnel package wires the atunnel dns, egress and ingress packages into a worker, so it belongs next to them. The package is now atunnel. - [x] Tests pass - [x] Docs updated --- benchmarking/atunnel/cmd/atunnel-bench/tunnel.go | 8 ++++---- benchmarking/atunnel/internal/workerhost/workerhost.go | 10 +++++----- .../internal/workerhost/workerhost_linux_test.go | 6 +++--- cmd/ateworker-gvisor/main.go | 10 +++++----- cmd/ateworker-microvm/main.go | 10 +++++----- cmd/ateworker-microvm/net_linux_test.go | 4 ++-- internal/{ateworker/tunnel => atunnel}/egresspolicy.go | 2 +- .../{ateworker/tunnel => atunnel}/egresspolicy_test.go | 2 +- internal/{ateworker/tunnel => atunnel}/tunnel.go | 4 ++-- internal/{ateworker/tunnel => atunnel}/tunnel_test.go | 2 +- 10 files changed, 29 insertions(+), 29 deletions(-) rename internal/{ateworker/tunnel => atunnel}/egresspolicy.go (99%) rename internal/{ateworker/tunnel => atunnel}/egresspolicy_test.go (99%) rename internal/{ateworker/tunnel => atunnel}/tunnel.go (99%) rename internal/{ateworker/tunnel => atunnel}/tunnel_test.go (99%) diff --git a/benchmarking/atunnel/cmd/atunnel-bench/tunnel.go b/benchmarking/atunnel/cmd/atunnel-bench/tunnel.go index 4dd997446a..cbc4bbea24 100644 --- a/benchmarking/atunnel/cmd/atunnel-bench/tunnel.go +++ b/benchmarking/atunnel/cmd/atunnel-bench/tunnel.go @@ -29,7 +29,7 @@ import ( "go.ate.dev/substrate/benchmarking/atunnel/internal/procstats" "go.ate.dev/substrate/benchmarking/atunnel/internal/workerhost" - "go.ate.dev/substrate/internal/ateworker/tunnel" + "go.ate.dev/substrate/internal/atunnel" ) // tunnelCommand is process B of the benchmark: atunnel started and hosting @@ -40,7 +40,7 @@ import ( func tunnelCommand() *cobra.Command { var actors int var gateway, adminListen, logLevel string - var cfg *tunnel.Config + var cfg *atunnel.Config cmd := &cobra.Command{ Use: "tunnel", Short: "Run atunnel, hosting actors with the worker's own code", @@ -56,7 +56,7 @@ func tunnelCommand() *cobra.Command { return runTunnel(cmd.Context(), *cfg, actors, gateway, adminListen) }, } - cfg = tunnel.RegisterFlags(cmd.Flags()) + cfg = atunnel.RegisterFlags(cmd.Flags()) cmd.Flags().IntVar(&actors, "actors", 10, "actors to host") cmd.Flags().StringVar(&gateway, "gateway", "atunnel-sink:9443", "egress gateway every actor tunnels to, host:port") cmd.Flags().StringVar(&adminListen, "admin", ":9090", "address for /stats, /healthz, and /debug/pprof") @@ -64,7 +64,7 @@ func tunnelCommand() *cobra.Command { return cmd } -func runTunnel(ctx context.Context, cfg tunnel.Config, actors int, gateway, adminListen string) error { +func runTunnel(ctx context.Context, cfg atunnel.Config, actors int, gateway, adminListen string) error { host, err := workerhost.Start(ctx, cfg, actors, gateway) if err != nil { return err diff --git a/benchmarking/atunnel/internal/workerhost/workerhost.go b/benchmarking/atunnel/internal/workerhost/workerhost.go index fc82d9974d..ff7bc107f7 100644 --- a/benchmarking/atunnel/internal/workerhost/workerhost.go +++ b/benchmarking/atunnel/internal/workerhost/workerhost.go @@ -15,7 +15,7 @@ //go:build linux // Package workerhost hosts atunnel the way an ateworker does, with the -// worker's own code: tunnel.Start once, then per actor the calls +// worker's own code: atunnel.Start once, then per actor the calls // cmd/ateworker-gvisor makes to run one, in its order. Only the sandbox itself // is missing: each actor's network namespaces, veth, nftables redirect, egress // listener, and DNS relay are real, but no gVisor runs in them. @@ -30,7 +30,7 @@ import ( "sync" "go.ate.dev/substrate/internal/ateworker/actornet" - "go.ate.dev/substrate/internal/ateworker/tunnel" + "go.ate.dev/substrate/internal/atunnel" "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/internal/resources" @@ -62,7 +62,7 @@ type Info struct { // Host is a running atunnel with its actors. type Host struct { - tunnel *tunnel.Tunnel + tunnel *atunnel.Tunnel mu sync.Mutex actors []*actor @@ -75,8 +75,8 @@ type actor struct { // Start starts atunnel from cfg, as a worker's main does, and hosts actors // actors whose egress tunnels to gatewayAddress. -func Start(ctx context.Context, cfg tunnel.Config, actors int, gatewayAddress string) (*Host, error) { - t, err := tunnel.Start(ctx, cfg, actornet.ActorHTTPUpstream) +func Start(ctx context.Context, cfg atunnel.Config, actors int, gatewayAddress string) (*Host, error) { + t, err := atunnel.Start(ctx, cfg, actornet.ActorHTTPUpstream) if err != nil { return nil, err } diff --git a/benchmarking/atunnel/internal/workerhost/workerhost_linux_test.go b/benchmarking/atunnel/internal/workerhost/workerhost_linux_test.go index 5cfb9caa43..1885d159f0 100644 --- a/benchmarking/atunnel/internal/workerhost/workerhost_linux_test.go +++ b/benchmarking/atunnel/internal/workerhost/workerhost_linux_test.go @@ -30,7 +30,7 @@ import ( "go.ate.dev/substrate/benchmarking/atunnel/internal/pki" "go.ate.dev/substrate/benchmarking/atunnel/internal/sink" "go.ate.dev/substrate/internal/atenet" - "go.ate.dev/substrate/internal/ateworker/tunnel" + "go.ate.dev/substrate/internal/atunnel" "go.ate.dev/substrate/internal/credbundle" "go.ate.dev/substrate/internal/egresspolicy" "go.ate.dev/substrate/internal/installdefaults" @@ -73,7 +73,7 @@ func TestHostsActorsWithTheWorkersCode(t *testing.T) { go func() { _ = gateway.Serve(ctx, gatewayLis) }() ingressAddr := freeAddress(t) - host, err := Start(ctx, tunnel.Config{ + host, err := Start(ctx, atunnel.Config{ ListenAddress: ingressAddr, ConnectListenAddress: freeAddress(t), CredentialBundle: filepath.Join(dir, pki.WorkerBundle), @@ -180,7 +180,7 @@ func listen(t *testing.T) net.Listener { } // freeAddress returns a loopback address with a port free a moment ago, for -// listeners tunnel.Start binds itself. +// listeners atunnel.Start binds itself. func freeAddress(t *testing.T) string { t.Helper() lis, err := net.Listen("tcp", "127.0.0.1:0") diff --git a/cmd/ateworker-gvisor/main.go b/cmd/ateworker-gvisor/main.go index 660c48054e..d876d92e7f 100644 --- a/cmd/ateworker-gvisor/main.go +++ b/cmd/ateworker-gvisor/main.go @@ -44,8 +44,8 @@ import ( "go.ate.dev/substrate/internal/ateworker/phaselog" "go.ate.dev/substrate/internal/ateworker/pid1" "go.ate.dev/substrate/internal/ateworker/register" - "go.ate.dev/substrate/internal/ateworker/tunnel" "go.ate.dev/substrate/internal/ateworkervalidation" + "go.ate.dev/substrate/internal/atunnel" "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/ocispec" @@ -76,7 +76,7 @@ func flagsConfigPath() string { var ( podUID = pflag.String("pod-uid", "", "The UID of the current pod") - tunnelConfig = tunnel.RegisterFlags(pflag.CommandLine) + tunnelConfig = atunnel.RegisterFlags(pflag.CommandLine) readinessListenAddress = pflag.String("readiness-listen-address", "0.0.0.0:8080", "Address for the HTTP /readyz and /healthz probes") maxActors = pflag.Int("max-actors", 1000, "How many actors this worker will host at once") @@ -234,7 +234,7 @@ func do(ctx context.Context) error { } actorLogger := actorlog.NewActorLogger(syncedWriter, metadata.OnGCE()) - tunnel, err := tunnel.Start(ctx, *tunnelConfig, actornet.ActorHTTPUpstream) + tunnel, err := atunnel.Start(ctx, *tunnelConfig, actornet.ActorHTTPUpstream) if err != nil { return err } @@ -345,7 +345,7 @@ type AteWorkerService struct { maxActors int actorLogger *actorlog.ActorLogger - tunnel *tunnel.Tunnel + tunnel *atunnel.Tunnel // usage writes the usage records. Nil writes none. usage *workerstats.UsageEmitter @@ -375,7 +375,7 @@ type AteWorkerService struct { var _ ateworkerpb.WorkerServer = (*AteWorkerService)(nil) // NewService creates a new AteWorkerService. -func NewService(tunnel *tunnel.Tunnel, actorLogger *actorlog.ActorLogger, maxActors int, flags *flagStore) *AteWorkerService { +func NewService(tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger, maxActors int, flags *flagStore) *AteWorkerService { return &AteWorkerService{ locks: actorlock.New(), inFlight: actorlock.NewInFlight(), diff --git a/cmd/ateworker-microvm/main.go b/cmd/ateworker-microvm/main.go index 930ae4b93d..b1cd7a3d74 100644 --- a/cmd/ateworker-microvm/main.go +++ b/cmd/ateworker-microvm/main.go @@ -46,7 +46,7 @@ import ( "go.ate.dev/substrate/internal/ateworker/cgroup" "go.ate.dev/substrate/internal/ateworker/pid1" "go.ate.dev/substrate/internal/ateworker/register" - "go.ate.dev/substrate/internal/ateworker/tunnel" + "go.ate.dev/substrate/internal/atunnel" "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/otlprelay" "go.ate.dev/substrate/internal/preview" @@ -75,7 +75,7 @@ var ( otlpRelaySocket = pflag.String("otlp-relay-socket", nodepath.AteletOTLPSocketPath(), "Unix socket of atelet's OTLP relay to export telemetry through, keeping it off the pod network. Empty, or absent at startup, exports directly to OTEL_EXPORTER_OTLP_ENDPOINT instead.") - tunnelConfig = tunnel.RegisterFlags(pflag.CommandLine) + tunnelConfig = atunnel.RegisterFlags(pflag.CommandLine) readinessListenAddress = pflag.String("readiness-listen-address", "0.0.0.0:8080", "Address for the HTTP /readyz and /healthz probes") maxActors = pflag.Int("max-actors", 1000, "How many actors this worker will host at once") @@ -229,7 +229,7 @@ func do(ctx context.Context) error { return fmt.Errorf("while delegating the worker cgroup: %w", err) } - tunnel, err := tunnel.Start(ctx, *tunnelConfig, actornet.ActorHTTPUpstream) + tunnel, err := atunnel.Start(ctx, *tunnelConfig, actornet.ActorHTTPUpstream) if err != nil { return err } @@ -376,7 +376,7 @@ type AteWorkerService struct { // stdout as ate.dev/*-labeled JSON and emits actor lifecycle events (parity // with ateworker-gvisor). actorLogger *actorlog.ActorLogger - tunnel *tunnel.Tunnel + tunnel *atunnel.Tunnel // usage writes the usage records. Nil writes none. usage *workerstats.UsageEmitter @@ -398,7 +398,7 @@ type AteWorkerService struct { var _ ateworkerpb.WorkerServer = (*AteWorkerService)(nil) // NewService creates a new AteWorkerService. -func NewService(podUID, chBinary string, guestDebug bool, memReserveMiB, maxActors int, tunnel *tunnel.Tunnel, actorLogger *actorlog.ActorLogger) *AteWorkerService { +func NewService(podUID, chBinary string, guestDebug bool, memReserveMiB, maxActors int, tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger) *AteWorkerService { return &AteWorkerService{ locks: actorlock.New(), inFlight: actorlock.NewInFlight(), diff --git a/cmd/ateworker-microvm/net_linux_test.go b/cmd/ateworker-microvm/net_linux_test.go index 7831a7cd99..a6e156cc00 100644 --- a/cmd/ateworker-microvm/net_linux_test.go +++ b/cmd/ateworker-microvm/net_linux_test.go @@ -23,7 +23,7 @@ import ( "github.com/vishvananda/netlink" "go.ate.dev/substrate/internal/ateworker/actornet" - "go.ate.dev/substrate/internal/ateworker/tunnel" + "go.ate.dev/substrate/internal/atunnel" "go.ate.dev/substrate/internal/atunnel/egress" "go.ate.dev/substrate/internal/linuxsys" "go.ate.dev/substrate/internal/netns" @@ -42,7 +42,7 @@ func TestHostActorReplacesSameActor(t *testing.T) { t.Fatal(err) } service := &AteWorkerService{ - tunnel: &tunnel.Tunnel{Egress: egressProxy, EgressPort: 15001}, + tunnel: &atunnel.Tunnel{Egress: egressProxy, EgressPort: 15001}, actors: map[string]*hostedActor{}, maxActors: 1, } diff --git a/internal/ateworker/tunnel/egresspolicy.go b/internal/atunnel/egresspolicy.go similarity index 99% rename from internal/ateworker/tunnel/egresspolicy.go rename to internal/atunnel/egresspolicy.go index 10d4c3ca9e..34c0907fed 100644 --- a/internal/ateworker/tunnel/egresspolicy.go +++ b/internal/atunnel/egresspolicy.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package tunnel +package atunnel import ( "context" diff --git a/internal/ateworker/tunnel/egresspolicy_test.go b/internal/atunnel/egresspolicy_test.go similarity index 99% rename from internal/ateworker/tunnel/egresspolicy_test.go rename to internal/atunnel/egresspolicy_test.go index 906e62c964..5fd8ecd3ed 100644 --- a/internal/ateworker/tunnel/egresspolicy_test.go +++ b/internal/atunnel/egresspolicy_test.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package tunnel +package atunnel import ( "context" diff --git a/internal/ateworker/tunnel/tunnel.go b/internal/atunnel/tunnel.go similarity index 99% rename from internal/ateworker/tunnel/tunnel.go rename to internal/atunnel/tunnel.go index 142f044067..484b288995 100644 --- a/internal/ateworker/tunnel/tunnel.go +++ b/internal/atunnel/tunnel.go @@ -14,10 +14,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package tunnel wires atunnel into a worker: its flags, listeners, DNS +// Package atunnel wires the tunnel into a worker: its flags, listeners, DNS // relay, and per-actor ingress and egress. Both workers build their tunnel here // so they cannot drift. -package tunnel +package atunnel import ( "context" diff --git a/internal/ateworker/tunnel/tunnel_test.go b/internal/atunnel/tunnel_test.go similarity index 99% rename from internal/ateworker/tunnel/tunnel_test.go rename to internal/atunnel/tunnel_test.go index 022ee9dc0e..8a0b7aa5e6 100644 --- a/internal/ateworker/tunnel/tunnel_test.go +++ b/internal/atunnel/tunnel_test.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package tunnel +package atunnel import ( "context" From 3c9a3cd366722323e2e27385283e4f72c58e31a1 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Tue, 20 Oct 2026 19:00:25 +0000 Subject: [PATCH 13/63] Upgrade Go to 1.27.3 (#2984) Go has a critical security fix. Longterm we're going to automate these but dependabot can't handle it and we're undecided on solution, @MushuEE / @stshrive are looking. NOTE: we don't _have_ to bump the dockerfiles for go bumps, but in this case TLS is vulnerable so the download mechanism for gotoolchain is impacted, so I'm being extra-comprehensive. AI use: Yes, I reviewed everything. Trivial diff, and tests were run - [x] Tests pass - [x] Docs updated --- .github/workflows/pr-workflow.yaml | 4 ++-- benchmarking/automation/Dockerfile | 2 +- benchmarking/locust/Dockerfile | 2 +- go.mod | 2 ++ hack/tools/code-generator/go.mod | 2 ++ hack/tools/controller-gen/go.mod | 2 ++ hack/tools/go-licenses/go.mod | 2 ++ hack/tools/golangci-lint/go.mod | 2 ++ hack/tools/gotestsum/go.mod | 2 ++ hack/tools/kind/go.mod | 2 ++ hack/tools/ko/go.mod | 2 ++ hack/tools/kube-api-linter/go.mod | 2 ++ hack/tools/protoc-gen-go-grpc/go.mod | 2 ++ hack/tools/protoc-gen-go/go.mod | 2 ++ hack/tools/setup-envtest/go.mod | 2 ++ internal/plugins/gcp-secret-manager/go.mod | 2 ++ tools/apitool/go.mod | 4 +++- tools/junittool/go.mod | 2 ++ 18 files changed, 35 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 052794438a..28154ee609 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -48,7 +48,7 @@ jobs: with: fetch-depth: 0 - name: Setup Go - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: 'go.mod' # Unconditional: a released migration becoming permanent is exactly what @@ -216,7 +216,7 @@ jobs: - name: Checkout uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - name: Setup Go - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: 'go.mod' # Use the e2e cache below instead. setup-go's cache key is shared with diff --git a/benchmarking/automation/Dockerfile b/benchmarking/automation/Dockerfile index 45330795e0..acbecda152 100644 --- a/benchmarking/automation/Dockerfile +++ b/benchmarking/automation/Dockerfile @@ -39,7 +39,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # Go must be new enough to honor the `tool` directive in # hack/tools/*/go.mod (Go 1.24+). Pin a version known to match the # substrate project's pinned ko (see hack/tools/ko/go.mod). -ARG GO_VERSION=1.26.3 +ARG GO_VERSION=1.27.3 RUN curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" \ | tar -C /usr/local -xz ENV PATH=/usr/local/go/bin:$PATH diff --git a/benchmarking/locust/Dockerfile b/benchmarking/locust/Dockerfile index 67791e75fc..a1c85050db 100644 --- a/benchmarking/locust/Dockerfile +++ b/benchmarking/locust/Dockerfile @@ -35,7 +35,7 @@ RUN PYTHON=python3 OUT_DIR=/out benchmarking/locust/codegen/generate.sh # Stage 3: Go build of the boomer-worker worker. Lives in the same image so # runner.py can spawn it as a subprocess during headless runs of glutton. # CGO_ENABLED=0 keeps us on boomer's pure-Go ZMQ (gomq), no libzmq needed. -FROM golang:1.26-bookworm AS goboomer +FROM golang:1.27.3-bookworm@sha256:5c3e9b43cfd3bb95c7937462e29d87ada57b481a96fa23ad0eb61778d7a00110 AS goboomer ENV GOTOOLCHAIN=auto WORKDIR /src COPY go.mod go.sum ./ diff --git a/go.mod b/go.mod index 5322e5a32c..deca33e7cd 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module go.ate.dev/substrate go 1.27.0 +toolchain go1.27.3 + require ( cloud.google.com/go/compute/metadata v0.9.0 cloud.google.com/go/container v1.49.0 diff --git a/hack/tools/code-generator/go.mod b/hack/tools/code-generator/go.mod index a66da5b8ef..ef5cfe76bc 100644 --- a/hack/tools/code-generator/go.mod +++ b/hack/tools/code-generator/go.mod @@ -2,6 +2,8 @@ module go.ate.dev/substrate/hack/tools/code-generator go 1.27.0 +toolchain go1.27.3 + require ( github.com/fxamacker/cbor/v2 v2.9.2 // indirect github.com/go-logr/logr v1.4.4 // indirect diff --git a/hack/tools/controller-gen/go.mod b/hack/tools/controller-gen/go.mod index 3c6ea3b272..383580c8fb 100644 --- a/hack/tools/controller-gen/go.mod +++ b/hack/tools/controller-gen/go.mod @@ -2,6 +2,8 @@ module go.ate.dev/substrate/hack/tools/controller-gen go 1.27.0 +toolchain go1.27.3 + require ( github.com/fatih/color v1.18.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect diff --git a/hack/tools/go-licenses/go.mod b/hack/tools/go-licenses/go.mod index 7b23d3742d..ae54969887 100644 --- a/hack/tools/go-licenses/go.mod +++ b/hack/tools/go-licenses/go.mod @@ -2,6 +2,8 @@ module go.ate.dev/substrate/hack/tools/go-licenses go 1.27.0 +toolchain go1.27.3 + require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-logr/logr v1.2.0 // indirect diff --git a/hack/tools/golangci-lint/go.mod b/hack/tools/golangci-lint/go.mod index 8f19eb233c..7a393833e8 100644 --- a/hack/tools/golangci-lint/go.mod +++ b/hack/tools/golangci-lint/go.mod @@ -2,6 +2,8 @@ module go.ate.dev/substrate/hack/tools/golangci-lint go 1.27.0 +toolchain go1.27.3 + tool github.com/golangci/golangci-lint/v2/cmd/golangci-lint require ( diff --git a/hack/tools/gotestsum/go.mod b/hack/tools/gotestsum/go.mod index f32423efa8..10e6d6eb19 100644 --- a/hack/tools/gotestsum/go.mod +++ b/hack/tools/gotestsum/go.mod @@ -2,6 +2,8 @@ module go.ate.dev/substrate/hack/tools/gotestsum go 1.27.0 +toolchain go1.27.3 + tool gotest.tools/gotestsum require ( diff --git a/hack/tools/kind/go.mod b/hack/tools/kind/go.mod index 5545361cdb..d83d209889 100644 --- a/hack/tools/kind/go.mod +++ b/hack/tools/kind/go.mod @@ -2,6 +2,8 @@ module go.ate.dev/substrate/hack/tools/kind go 1.27.0 +toolchain go1.27.3 + tool sigs.k8s.io/kind require ( diff --git a/hack/tools/ko/go.mod b/hack/tools/ko/go.mod index 315dd667e7..68d4123a63 100644 --- a/hack/tools/ko/go.mod +++ b/hack/tools/ko/go.mod @@ -2,6 +2,8 @@ module go.ate.dev/substrate/hack/tools/ko go 1.27.0 +toolchain go1.27.3 + require ( al.essio.dev/pkg/shellescape v1.6.0 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect diff --git a/hack/tools/kube-api-linter/go.mod b/hack/tools/kube-api-linter/go.mod index e62bf3e968..3030880bc9 100644 --- a/hack/tools/kube-api-linter/go.mod +++ b/hack/tools/kube-api-linter/go.mod @@ -2,6 +2,8 @@ module go.ate.dev/substrate/hack/tools/kube-api-linter go 1.27.0 +toolchain go1.27.3 + tool sigs.k8s.io/kube-api-linter/cmd/golangci-lint-kube-api-linter require ( diff --git a/hack/tools/protoc-gen-go-grpc/go.mod b/hack/tools/protoc-gen-go-grpc/go.mod index 75ec153d5a..e0193e6078 100644 --- a/hack/tools/protoc-gen-go-grpc/go.mod +++ b/hack/tools/protoc-gen-go-grpc/go.mod @@ -2,6 +2,8 @@ module go.ate.dev/substrate/hack/tools/protoc-gen-go-grpc go 1.27.0 +toolchain go1.27.3 + require ( google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.2 // indirect google.golang.org/protobuf v1.36.11 // indirect diff --git a/hack/tools/protoc-gen-go/go.mod b/hack/tools/protoc-gen-go/go.mod index 6a2e087cf7..b4c96ef31e 100644 --- a/hack/tools/protoc-gen-go/go.mod +++ b/hack/tools/protoc-gen-go/go.mod @@ -2,6 +2,8 @@ module go.ate.dev/substrate/hack/tools/protoc-gen-go go 1.27.0 +toolchain go1.27.3 + require google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect tool google.golang.org/protobuf/cmd/protoc-gen-go diff --git a/hack/tools/setup-envtest/go.mod b/hack/tools/setup-envtest/go.mod index 22eb248db2..0aa8b73f0b 100644 --- a/hack/tools/setup-envtest/go.mod +++ b/hack/tools/setup-envtest/go.mod @@ -2,6 +2,8 @@ module go.ate.dev/substrate/hack/tools/setup-envtest go 1.27.0 +toolchain go1.27.3 + require ( github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/zapr v1.3.0 // indirect diff --git a/internal/plugins/gcp-secret-manager/go.mod b/internal/plugins/gcp-secret-manager/go.mod index b45e90bcf4..b1dc19e179 100644 --- a/internal/plugins/gcp-secret-manager/go.mod +++ b/internal/plugins/gcp-secret-manager/go.mod @@ -2,6 +2,8 @@ module go.ate.dev/substrate/internal/plugins/gcp-secret-manager go 1.27.0 +toolchain go1.27.3 + require ( cloud.google.com/go/secretmanager v1.16.0 github.com/googleapis/gax-go/v2 v2.21.0 diff --git a/tools/apitool/go.mod b/tools/apitool/go.mod index 2c138c034d..d602bd6476 100644 --- a/tools/apitool/go.mod +++ b/tools/apitool/go.mod @@ -2,13 +2,15 @@ module go.ate.dev/substrate/tools/apitool go 1.27.0 +toolchain go1.27.3 + replace go.ate.dev/substrate => ../.. require ( - go.ate.dev/substrate v0.0.0-00010101000000-000000000000 github.com/bufbuild/protocompile v0.14.1 github.com/google/go-cmp v0.7.0 github.com/spf13/cobra v1.10.2 + go.ate.dev/substrate v0.0.0-00010101000000-000000000000 google.golang.org/protobuf v1.36.12 ) diff --git a/tools/junittool/go.mod b/tools/junittool/go.mod index 57e3b7110a..eebd3aa280 100644 --- a/tools/junittool/go.mod +++ b/tools/junittool/go.mod @@ -1,3 +1,5 @@ module go.ate.dev/substrate/tools/junittool go 1.27.0 + +toolchain go1.27.3 From cad0ac2cd20ed4a527401975721ce0b8dd7e5dae Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Tue, 20 Oct 2026 19:43:06 +0000 Subject: [PATCH 14/63] Fix an image cache eviction race that could drop an image record after a failed pull (#2989) Follow-up to https://github.com/agent-substrate/substrate/pull/1509#discussion_r4319320553. `retireLayer` stats a layer before checking its pin. If the layer lands, and its pull then fails and unpins in between, eviction reports the layer gone and drops the image record, so the unpacked layer has no record until the image is pulled again. A missing layer now goes through `renameAsideUnlessPinned`, which checks the pin and stats again under `pinMu`. `dryRunRetire` already checks the pin first. There is no new test, since the window can't be hit without a hook between the two checks. `TestRetireLayerStatuses` still covers the missing and missing+pinned cases. AI use: AI-assisted, I reviewed the change. - [x] Tests pass --- internal/imagecache/imagecache.go | 2 +- internal/imagecache/retire.go | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/internal/imagecache/imagecache.go b/internal/imagecache/imagecache.go index c93e19c398..9afaa78ab9 100644 --- a/internal/imagecache/imagecache.go +++ b/internal/imagecache/imagecache.go @@ -704,7 +704,7 @@ func (s *Store) pinLayers(hexes []string) (unpin func()) { } // pinned reports whether an in-flight pull holds the layer. A snapshot: -// retirement itself re-checks under pinMu instead. +// retirement checks under pinMu instead. func (s *Store) pinned(hex string) bool { s.pinMu.Lock() defer s.pinMu.Unlock() diff --git a/internal/imagecache/retire.go b/internal/imagecache/retire.go index e630e59a6c..6f84eb77a6 100644 --- a/internal/imagecache/retire.go +++ b/internal/imagecache/retire.go @@ -81,18 +81,15 @@ func isLayerDirName(name string) bool { // and left in place. // // A fresh layer is ruled out before taking pinMu, which every pull takes to -// pin, and the logging happens outside it. A missing layer that a pull has -// pinned is still unpacking, so it is vetoed to keep the record. +// pin, and the logging happens outside it. A missing layer still goes +// through the pin check, since a pull that pinned it may be unpacking it. func (s *Store) retireLayer(hex string, cutoff time.Time) (string, retireStatus, error) { if !isLayerDirName(hex) { return "", retireVetoed, fmt.Errorf("not a layer dir name: %q", hex) } dir := filepath.Join(s.layersDir(), hex) - if st, mtime, err := retirable(dir, cutoff); st != retireRetired { - if st == retireGone && s.pinned(hex) { - return "", retireVetoed, nil - } - if st == retireVetoed && err == nil { + if st, mtime, err := retirable(dir, cutoff); st == retireVetoed { + if err == nil { slog.Info(logMsgLayerRetireVetoed, slog.String("diffid", hex), slog.Time("last_used", mtime)) } return "", st, err From 78dd5b50926b0043d8a8a096c4e658f8e45742f8 Mon Sep 17 00:00:00 2001 From: Luiz Oliveira Date: Tue, 20 Oct 2026 20:22:28 +0000 Subject: [PATCH 15/63] Remove workerpool autoscaling demo (#2916) After mega-worker, this demo is no longer correct. We need more though on how workers autoscaling would work, so let's remove this demo in the meantime. --- README.md | 1 - cmd/ate-setup/commands.md | 1 - cmd/ate-setup/differences.md | 8 - cmd/ate-setup/internal/cmd/delete.go | 2 +- cmd/ate-setup/internal/demos/all/all.go | 1 - .../autoscaledworkerpool.go | 94 ------- cmd/ate-setup/internal/demos/demos.go | 28 +- cmd/ate-setup/internal/demos/render.go | 2 +- cmd/ate-setup/internal/steps/env.go | 3 +- demos/autoscaled-workerpool/README.md | 173 ------------- .../autoscaled-workerpool-template.yaml.tmpl | 49 ---- .../autoscaled-workerpool.yaml.tmpl | 42 --- demos/autoscaled-workerpool/hpa-kind.yaml | 66 ----- .../prometheus-adapter.yaml | 242 ------------------ 14 files changed, 6 insertions(+), 706 deletions(-) delete mode 100644 cmd/ate-setup/internal/demos/autoscaledworkerpool/autoscaledworkerpool.go delete mode 100644 demos/autoscaled-workerpool/README.md delete mode 100644 demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl delete mode 100644 demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl delete mode 100644 demos/autoscaled-workerpool/hpa-kind.yaml delete mode 100644 demos/autoscaled-workerpool/prometheus-adapter.yaml diff --git a/README.md b/README.md index fb7c783492..372285d83c 100644 --- a/README.md +++ b/README.md @@ -244,7 +244,6 @@ We provide several sample applications demonstrating Agent Substrate's capabilit 3. **[Claude Code Multiplex](demos/claude-code-multiplex/README.md)**: Demonstrates oversubscribing physical hardware by multiplexing multiple Claude Code agents onto a limited pool of workers. 4. **[Multi-Template](demos/multi-template/README.md)**: Two `ActorTemplate`s running different binaries share one `WorkerPool`, even though the templates live in different atespaces. 5. **[Request Parking](demos/parking/README.md)**: An oversubscribed pool where the router holds inbound requests until a worker frees up, instead of returning `503`. -6. **[Autoscaled WorkerPool](demos/autoscaled-workerpool/README.md)**: Scales a `WorkerPool` on its assigned-worker count with an HPA fed by prometheus-adapter. ### Documentation & Guides * [Architecture](docs/architecture.md): How the control plane, node supervisor, and networking stack fit together. diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index a32674c2a2..54a80ea7f1 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -205,7 +205,6 @@ See | `deploy demo sandbox` | `--deploy-demo-sandbox` | An on-demand sandbox actor driven by the sandbox client | | `deploy demo multi-template` | `--deploy-demo-multi-template` | Two ActorTemplates sharing one WorkerPool | | `deploy demo parking` | `--deploy-demo-parking` | Actor parking and unparking on a small WorkerPool | -| `deploy demo autoscaled-workerpool` | `--deploy-demo-autoscaled-workerpool` | A WorkerPool scaled by an HPA over custom metrics (Kind only) | | `deploy demo claude-code-multiplex` | `--deploy-demo-claude-code-multiplex` | Several Claude Code agents multiplexed onto one WorkerPool (requires `ANTHROPIC_API_KEY`, `BUCKET_NAME`, `KO_DOCKER_REPO`) | Each has a matching `delete demo NAME` / `--delete-demo-NAME`. Demo flags bind diff --git a/cmd/ate-setup/differences.md b/cmd/ate-setup/differences.md index 20f60de36c..d03d7b2385 100644 --- a/cmd/ate-setup/differences.md +++ b/cmd/ate-setup/differences.md @@ -232,14 +232,6 @@ demo is its own package registering itself from `init`, and `internal/demos/all` is the single list of what gets linked in — with a test asserting no demo package is missing from it. -**Kind-only demos are visible but refuse.** `demo-autoscaled-workerpool` -registered itself only when `ATE_INSTALL_KIND=true`, so on GKE -`--deploy-demo-autoscaled-workerpool` was rejected as an *unknown option*. The -cobra tree is built before flags are parsed, so `--kind` is not known yet; the -subcommand always exists and fails at run time with `demo-autoscaled-workerpool -is only supported for Kind installations; re-run with --kind`. `delete all` -still skips it on a non-Kind install, via `demos.For`. - **Template expansion is not sed.** The demo templates were expanded by a stack of `sed -e` expressions doing two things: substitute `${PLACEHOLDER}`, or delete the whole line (`/${NAME}/d`) when the placeholder is not in play. diff --git a/cmd/ate-setup/internal/cmd/delete.go b/cmd/ate-setup/internal/cmd/delete.go index 7bb9df466e..dab32dd0e6 100644 --- a/cmd/ate-setup/internal/cmd/delete.go +++ b/cmd/ate-setup/internal/cmd/delete.go @@ -48,7 +48,7 @@ var deleteAllCmd = &cobra.Command{ Short: "Delete every demo and then the core system", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { - return env.DeleteAll(cmd.Context(), demos.Deleters(env.Cfg)) + return env.DeleteAll(cmd.Context(), demos.Deleters()) }, } diff --git a/cmd/ate-setup/internal/demos/all/all.go b/cmd/ate-setup/internal/demos/all/all.go index 49fec9da2c..11d7ba220a 100644 --- a/cmd/ate-setup/internal/demos/all/all.go +++ b/cmd/ate-setup/internal/demos/all/all.go @@ -21,7 +21,6 @@ package all import ( - _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/autoscaledworkerpool" _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/claudemultiplex" _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/counter" _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/countermicrovm" diff --git a/cmd/ate-setup/internal/demos/autoscaledworkerpool/autoscaledworkerpool.go b/cmd/ate-setup/internal/demos/autoscaledworkerpool/autoscaledworkerpool.go deleted file mode 100644 index 4b246f6b16..0000000000 --- a/cmd/ate-setup/internal/demos/autoscaledworkerpool/autoscaledworkerpool.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package autoscaledworkerpool installs the counter workload plus the -// custom-metrics stack that drives a HorizontalPodAutoscaler over its -// WorkerPool. -// -// The demo is Kind only: it ships its own prometheus-adapter and a Kind -// specific HPA, so it is not offered at all on a GKE install. -package autoscaledworkerpool - -import ( - "context" - - "go.ate.dev/substrate/cmd/ate-setup/internal/demos" - "go.ate.dev/substrate/cmd/ate-setup/internal/kube" - "go.ate.dev/substrate/cmd/ate-setup/internal/log" - "go.ate.dev/substrate/cmd/ate-setup/internal/steps" - "go.ate.dev/substrate/internal/resources" -) - -// namespace holds the demo workload, its prometheus-adapter, and the HPA; -// it doubles as the atespace holding the demo's ActorTemplate. -const namespace = "ate-demo-autoscaled-workerpool" - -// Add-ons that only make sense on Kind. -const ( - prometheusAdapterManifest = "demos/autoscaled-workerpool/prometheus-adapter.yaml" - hpaKindManifest = "demos/autoscaled-workerpool/hpa-kind.yaml" -) - -type demo struct { - demos.Substrate -} - -func init() { - demos.Register(&demo{Substrate: demos.Substrate{ - DemoName: "demo-autoscaled-workerpool", - Short: "A WorkerPool scaled by an HPA over custom metrics (Kind only)", - WorkerPoolManifest: "demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl", - Deployments: []steps.TemplateRef{{Atespace: namespace, Name: "counter"}}, - Templates: []demos.SubstrateTemplate{{ - Manifest: "demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl", - Ref: resources.ActorTemplateRef{Atespace: namespace, Name: "counter"}, - }}, - }}) -} - -func (d *demo) KindOnly() bool { return true } - -func (d *demo) Deploy(ctx context.Context, e *steps.Env) error { - if err := e.RequireKind(d.DemoName); err != nil { - return err - } - if err := d.Substrate.Deploy(ctx, e); err != nil { - return err - } - - log.Step("Deploying prometheus-adapter and HPA for kind...") - if err := e.Kube.ApplyPath(ctx, e.Cfg.Path(prometheusAdapterManifest)); err != nil { - return err - } - if err := e.Kube.RolloutStatus(ctx, kube.KindDeployment, namespace, - "prometheus-adapter", e.Cfg.WaitTimeout(steps.BootstrapTimeout)); err != nil { - return err - } - return e.Kube.ApplyPath(ctx, e.Cfg.Path(hpaKindManifest)) -} - -func (d *demo) Delete(ctx context.Context, e *steps.Env) error { - if err := e.RequireKind(d.DemoName); err != nil { - return err - } - // The HPA goes first so it cannot scale the pool back up while the - // workload is being removed. - if err := e.Kube.DeletePath(ctx, e.Cfg.Path(hpaKindManifest)); err != nil { - return err - } - if err := e.Kube.DeletePath(ctx, e.Cfg.Path(prometheusAdapterManifest)); err != nil { - return err - } - return d.Substrate.Delete(ctx, e) -} diff --git a/cmd/ate-setup/internal/demos/demos.go b/cmd/ate-setup/internal/demos/demos.go index ff99a65f92..44a14c23ea 100644 --- a/cmd/ate-setup/internal/demos/demos.go +++ b/cmd/ate-setup/internal/demos/demos.go @@ -54,11 +54,6 @@ type Demo interface { Delete(ctx context.Context, e *steps.Env) error } -// kindOnlyDemo is implemented by demos that cannot run on GKE. -type kindOnlyDemo interface { - KindOnly() bool -} - // registry holds every demo, keyed by name. var registry = map[string]Demo{} @@ -97,30 +92,13 @@ func All() []Demo { return list } -// For returns the demos that apply to cfg, sorted by name. -// -// Kind-only demos are filtered out for non-Kind installs, mirroring how the -// autoscaled-workerpool demo only registered itself with the shell installer -// when ATE_INSTALL_KIND was true. DeleteAll iterates this, so a GKE teardown -// does not try to delete resources that could never have been installed. -func For(cfg *config.Config) []Demo { - var list []Demo - for _, d := range All() { - if ko, ok := d.(kindOnlyDemo); ok && ko.KindOnly() && !cfg.Kind { - continue - } - list = append(list, d) - } - return list -} - -// Deleters returns the demos that apply to cfg as the narrower interface +// Deleters returns every registered demo as the narrower interface // [steps.Env.DeleteAll] takes. // // The conversion lives here because the dependency only runs one way: the demos // import steps for Env, so steps cannot name this package's Demo. -func Deleters(cfg *config.Config) []steps.Deleter { - list := For(cfg) +func Deleters() []steps.Deleter { + list := All() deleters := make([]steps.Deleter, 0, len(list)) for _, d := range list { deleters = append(deleters, d) diff --git a/cmd/ate-setup/internal/demos/render.go b/cmd/ate-setup/internal/demos/render.go index 941d1d3dc1..a755057dfe 100644 --- a/cmd/ate-setup/internal/demos/render.go +++ b/cmd/ate-setup/internal/demos/render.go @@ -24,7 +24,7 @@ import ( const bucketNamePlaceholder = "BUCKET_NAME" // ExternalVolumePlaceholders are the optional external-volume hooks in the -// counter and autoscaled-workerpool templates. Every path except +// counter templates. Every path except // `deploy demo counter --with-external-volume` drops them, which removes the // lines entirely. var ExternalVolumePlaceholders = []string{ diff --git a/cmd/ate-setup/internal/steps/env.go b/cmd/ate-setup/internal/steps/env.go index 63b81920ae..8cfba859a7 100644 --- a/cmd/ate-setup/internal/steps/env.go +++ b/cmd/ate-setup/internal/steps/env.go @@ -278,8 +278,7 @@ func (e *Env) EnsureAteSystemNamespace(ctx context.Context) error { return e.Kube.WaitNamespaceActive(ctx, ns, NamespaceTimeout) } -// RequireKind fails a step that only makes sense on a local Kind cluster. The -// Kind-only demos, which live outside this package, use it too. +// RequireKind fails a step that only makes sense on a local Kind cluster. func (e *Env) RequireKind(what string) error { if !e.Cfg.Kind { return fmt.Errorf("%s is only supported for Kind installations; re-run with --kind", what) diff --git a/demos/autoscaled-workerpool/README.md b/demos/autoscaled-workerpool/README.md deleted file mode 100644 index 5382ba8c37..0000000000 --- a/demos/autoscaled-workerpool/README.md +++ /dev/null @@ -1,173 +0,0 @@ - - -# Autoscaling WorkerPools with an HPA - -This describes how to autoscale a `WorkerPool` on how many of its workers are -currently full, using a Kubernetes HorizontalPodAutoscaler (HPA) fed by the -`ate_workerpool_workers` metric through prometheus-adapter. - -## Prerequisites - -- A local kind cluster with Agent Substrate installed (`./hack/install-ate-kind.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}'`). - Note: this demo is currently only supported on kind. -- `ko` installed for building images. -- A GCS bucket for storing snapshots (configured via `BUCKET_NAME` env var). - -## Architecture (kind) - -``` -ate-api-server :9090/metrics (ate_workerpool_workers, OTel -> Prometheus gauge) - │ scrape (prometheus.io/scrape annotation, 15s) - ▼ -Prometheus manifests/ate-install/monitoring/prometheus.yaml - │ PromQL - ▼ -prometheus-adapter demos/autoscaled-workerpool/prometheus-adapter.yaml - │ external.metrics.k8s.io - ▼ -HorizontalPodAutoscaler - │ /scale (writes WorkerPool.spec.replicas) - ▼ -atecontroller ──► Deployment.spec.replicas ──► worker pods -``` -## HPA Configuration (External + AverageValue) - -The example HPA uses an **External** metric with target type **AverageValue**: - -``` -desiredReplicas = ceil( metricValue / target.averageValue ) -``` - -where `metricValue = max(ate_workerpool_workers{namespace=, name=, state=at_capacity})`: -the pool's count of full workers. WorkerPool names are only unique within a -namespace, so the selector must pin both. - -This is the pool's utilization only while each worker holds one actor, which -is true today. Once a worker holds more, a count of workers cannot say how full -the pool is, and this signal needs to change. - -`averageValue` is the target **full-workers-per-replica**: - -| `averageValue` | Meaning | Example (`at_capacity=7`) | -| -------------- | ---------------------------------------- | ------------------------- | -| `"0.7"` (700m) | ~70% full / 30% idle headroom (default) | `ceil(7/0.7) = 10` | -| `"1"` | pack to 100%, no idle headroom | `ceil(7/1) = 7` | -| `"0.5"` (500m) | lots of headroom, ~2× replicas | `ceil(7/0.5) = 14` | - -Lower `averageValue` → more idle headroom → more replicas. - -### Scale-up/scale-down behavior - -The example HPAs set an aggressive `scaleUp` (no stabilization window, -`selectPolicy: Max`, `Percent 100` / `Pods 10` steps) so a burst is served in a -**batch** rather than creeping up one worker at a time, and a slow `scaleDown` -(300s stabilization) to avoid flapping. `behavior` only sets the _rate of change_; -the HPA still scales to what the metric dictates. - -## How to Run on Agent Substrate - -### 1. Build and Deploy - -```sh -# Local dev (kind) -./hack/install-ate-kind.sh --deploy-demo-autoscaled-workerpool -``` - -This command will: - -- Create the `ate-demo-autoscaled-workerpool` namespace and one `WorkerPool` - (`counter`, starting with 5 replicas). -- Create the `ate-demo-autoscaled-workerpool` atespace and the `counter` actor - template in it (`autoscaled-workerpool-template.yaml.tmpl`, applied with - `kubectl ate create actor-template`), waiting until the pool is rolled out - and the template's golden snapshot is built. -- Deploy `prometheus-adapter` into `ate-demo-autoscaled-workerpool` to serve `ate_workerpool_workers` on `external.metrics.k8s.io`, and one `HorizontalPodAutoscaler` (`counter`). - -### 2. Verify Monitoring Stack & External Metric - -Confirm that `prometheus-adapter` is serving the external metric: - -```sh -# 1. Adapter is serving the External Metrics API -kubectl get apiservice v1beta1.external.metrics.k8s.io # Available=True - -# 2. The external metric resolves for the counter pool -kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1/namespaces/ate-demo-autoscaled-workerpool/ate_workerpool_workers?labelSelector=ate_worker_state%3Dat_capacity,ate_workerpool_namespace%3Date-demo-autoscaled-workerpool,ate_workerpool_name%3Dcounter" -``` - -## How to Use - -We can trigger autoscaling by spawning multiple actors and sending traffic to assign workers in the pool. The actors go in the demo's atespace (`ate-demo-autoscaled-workerpool`) — `--template` names the template, resolved in the actor's atespace: - -### 1. Spawn load actors - -```sh -# Install the CLI as a kubectl plugin if not already installed -go install ./cmd/kubectl-ate - -# Create 15 actors to generate load -for i in {001..015}; do - kubectl ate create actor c$i -a ate-demo-autoscaled-workerpool --template counter -done -``` - -### 2. Port-forward the atenet router and send traffic - -```sh -kubectl port-forward -n ate-system svc/atenet-router 8000:80 & -``` - -In a separate terminal, send requests in a retry loop to activate the actors -and keep them active while the pool scales up. Each request sets the Actor name -for that loop iteration and the demo's Atespace in the routing header: - -```sh -for attempt in {1..10}; do - for i in {001..015}; do - curl -s -H "ate-target-actor: ate-demo-autoscaled-workerpool/c$i" http://localhost:8000 >/dev/null - done - sleep 2 -done -``` - -### 3. Watch the HPA scale up - -As the count of full workers increases, watch the HPA scale up the pool's replicas: - -```sh -kubectl -n ate-demo-autoscaled-workerpool get hpa counter -w -kubectl -n ate-demo-autoscaled-workerpool get workerpool counter -w -``` - -### 4. Trigger scale-down - -Suspend the actors to drop the count of full workers. After the 300s stabilization window, the HPA will scale down the pool: - -```sh -for i in {001..015}; do - kubectl ate suspend actor c$i -a ate-demo-autoscaled-workerpool -done -``` - -## How to Uninstall - -Remove the demo — this deletes the actors, the template, the atespace, and -then the pool and its namespace: - -```sh -./hack/install-ate-kind.sh --delete-demo-autoscaled-workerpool -``` diff --git a/demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl b/demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl deleted file mode 100644 index 81558161b3..0000000000 --- a/demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# ActorTemplate (protojson-shaped ateapipb.ActorTemplate) for the -# autoscaled-workerpool demo. Applied with -# `kubectl ate create actor-template -f -` after `ko resolve` replaces the -# ko:// image reference; the ate-demo-autoscaled-workerpool atespace must -# exist. -metadata: - atespace: ate-demo-autoscaled-workerpool - name: counter -workerSelector: - matchLabels: - workload: counter-autoscaled -containers: -- name: counter - image: ko://go.ate.dev/substrate/demos/counter - command: - - /ko-app/counter - wakeupProbe: - httpGet: - path: /readyz - port: 80 - volumeMounts: - - name: data - mountPath: /home/counter -# Sandbox size. The pool sets no worker limits, so the worker offers the -# node's allocatable and these only need to fit the node. -resources: - limits: - cpuMillicores: 1000 - memoryBytes: 536870912 # 512Mi -snapshotConfig: - preferredFidelity: SNAPSHOT_FIDELITY_VOLUMES - storageLocation: gs://${BUCKET_NAME}/ate-demo-autoscaled-workerpool/ -volumes: -- name: data - durableDir: {} diff --git a/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl b/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl deleted file mode 100644 index b60beb231f..0000000000 --- a/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This demo demonstrates autoscaling an Agent Substrate WorkerPool using a -# HorizontalPodAutoscaler (HPA) and an External Metric (ate_workerpool_workers). -# The pool runs capacity for the "counter" workload; the HPA scales the pool -# dynamically based on the number of full workers. The ActorTemplate -# lives in autoscaled-workerpool-template.yaml.tmpl and is created through -# the ate API with `kubectl ate create actor-template`. -# -# See hpa-workerpool-autoscaling.md for the design and math details. - -apiVersion: v1 -kind: Namespace -metadata: - name: ate-demo-autoscaled-workerpool - ---- - -apiVersion: ate.dev/v1alpha1 -kind: WorkerPool -metadata: - name: counter - namespace: ate-demo-autoscaled-workerpool - labels: - workload: counter-autoscaled -spec: - replicas: 5 - workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor - sandboxClasses: - - name: gvisor diff --git a/demos/autoscaled-workerpool/hpa-kind.yaml b/demos/autoscaled-workerpool/hpa-kind.yaml deleted file mode 100644 index 0385e0a4ff..0000000000 --- a/demos/autoscaled-workerpool/hpa-kind.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Autoscale the counter WorkerPool based on how many of its workers are full. -# The metric ate_workerpool_workers (emitted by ate-api-server) is served to the -# External Metrics API by prometheus-adapter (demos/autoscaled-workerpool). -# External AverageValue math: desiredReplicas = ceil(at_capacity / averageValue). -# With averageValue=0.7 the pool targets ~70% full workers / 30% idle headroom. -# This is utilization only while each worker holds one actor. With more slots -# per worker, a worker count cannot say how full the pool is. -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: counter - namespace: ate-demo-autoscaled-workerpool -spec: - scaleTargetRef: - apiVersion: ate.dev/v1alpha1 - kind: WorkerPool - name: counter - minReplicas: 2 # warm floor: keep at least 2 workers ready - maxReplicas: 25 - metrics: - - type: External - external: - metric: - name: ate_workerpool_workers - selector: - matchLabels: - # The namespace is required: pool names are unique per namespace, and - # the counter demo ships a WorkerPool named "counter" too. - ate_workerpool_namespace: ate-demo-autoscaled-workerpool - ate_workerpool_name: counter - ate_worker_state: at_capacity - target: - type: AverageValue - averageValue: "0.7" # 70% utilization - behavior: - scaleUp: - stabilizationWindowSeconds: 0 - selectPolicy: Max - policies: - - type: Percent - value: 100 - periodSeconds: 15 - - type: Pods - value: 10 - periodSeconds: 15 - scaleDown: - stabilizationWindowSeconds: 300 - selectPolicy: Max - policies: - - type: Pods - value: 2 - periodSeconds: 60 diff --git a/demos/autoscaled-workerpool/prometheus-adapter.yaml b/demos/autoscaled-workerpool/prometheus-adapter.yaml deleted file mode 100644 index df5c0da493..0000000000 --- a/demos/autoscaled-workerpool/prometheus-adapter.yaml +++ /dev/null @@ -1,242 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# prometheus-adapter exposes the self-hosted Prometheus (prometheus.yaml) to the -# Kubernetes External Metrics API (external.metrics.k8s.io) so an HPA can scale a -# WorkerPool on ate_workerpool_workers. -# This is a standard install (github.com/kubernetes-sigs/prometheus-adapter) -# trimmed to the external-metrics path and pointed at the in-cluster Prometheus. -#Runs in ate-demo-autoscaled-workerpool. -# -# Flow: ate-api-server:9090/metrics --scrape--> Prometheus --query--> adapter -# --external.metrics.k8s.io--> HPA --/scale--> WorkerPool -> Deployment. - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: prometheus-adapter - namespace: ate-demo-autoscaled-workerpool ---- -# The external rule maps the Prometheus series ate_workerpool_workers to an -# external metric of the same name. An HPA's metric.selector.matchLabels are -# substituted into <<.LabelMatchers>>, so a selector of -# {ate_workerpool_namespace: demo-ns, ate_workerpool_name: shared-pool, ate_worker_state: at_capacity} -# yields max(ate_workerpool_workers{ate_workerpool_namespace="demo-ns",ate_workerpool_name="shared-pool",ate_worker_state="at_capacity"}), -# the pool's count of full workers. Selecting on the name alone would match -# every same-named pool in the cluster, so the namespace belongs in the selector. - -# We use max() rather than sum() because ate-api-server is sharded; -# every replica reports the global worker tally. -apiVersion: v1 -kind: ConfigMap -metadata: - name: prometheus-adapter-config - namespace: ate-demo-autoscaled-workerpool -data: - config.yaml: | - externalRules: - - seriesQuery: '{__name__="ate_workerpool_workers"}' - resources: - namespaced: false - name: - matches: "^(.*)$" - as: "$1" - # max() across ate-api-server replicas since each replica reports the - # global count. - metricsQuery: 'max(<<.Series>>{<<.LabelMatchers>>}) by (ate_workerpool_namespace, ate_workerpool_name, ate_worker_state)' ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: prometheus-adapter - namespace: ate-demo-autoscaled-workerpool - labels: - app: prometheus-adapter -spec: - replicas: 1 - selector: - matchLabels: - app: prometheus-adapter - template: - metadata: - labels: - app: prometheus-adapter - spec: - serviceAccountName: prometheus-adapter - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - containers: - - name: prometheus-adapter - image: registry.k8s.io/prometheus-adapter/prometheus-adapter:v0.12.0 - args: - - --secure-port=6443 - - --cert-dir=/tmp/cert - - --prometheus-url=http://prometheus.otel-system.svc:9090/ - - --metrics-relist-interval=30s - - --config=/etc/adapter/config.yaml - - --v=4 - ports: - - name: https - containerPort: 6443 - readinessProbe: - httpGet: - path: /readyz - port: https - scheme: HTTPS - initialDelaySeconds: 10 - periodSeconds: 5 - failureThreshold: 5 - livenessProbe: - httpGet: - path: /livez - port: https - scheme: HTTPS - initialDelaySeconds: 10 - periodSeconds: 15 - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - memory: 256Mi - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: - - ALL - volumeMounts: - - name: config - mountPath: /etc/adapter/ - readOnly: true - - name: tmp - mountPath: /tmp - volumes: - - name: config - configMap: - name: prometheus-adapter-config - - name: tmp - emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: prometheus-adapter - namespace: ate-demo-autoscaled-workerpool - labels: - app: prometheus-adapter -spec: - type: ClusterIP - selector: - app: prometheus-adapter - ports: - - name: https - port: 443 - targetPort: https ---- -# Register the adapter as the backend for external.metrics.k8s.io. insecureSkipTLSVerify -# is fine for dev (the adapter self-signs its serving cert via --cert-dir); for prod, -# supply a CA bundle instead. NOTE: only one adapter can own this APIService per -# cluster — the GMP custom-metrics-stackdriver-adapter registers the same group, so -# the two non-GMP / GMP variants must run in separate clusters. -apiVersion: apiregistration.k8s.io/v1 -kind: APIService -metadata: - name: v1beta1.external.metrics.k8s.io -spec: - service: - name: prometheus-adapter - namespace: ate-demo-autoscaled-workerpool - port: 443 - group: external.metrics.k8s.io - version: v1beta1 - insecureSkipTLSVerify: true - groupPriorityMinimum: 100 - versionPriority: 100 ---- -# Let the adapter authenticate/authorize incoming aggregated-API requests. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: prometheus-adapter-auth-delegator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: -- kind: ServiceAccount - name: prometheus-adapter - namespace: ate-demo-autoscaled-workerpool ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: prometheus-adapter-auth-reader - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader -subjects: -- kind: ServiceAccount - name: prometheus-adapter - namespace: ate-demo-autoscaled-workerpool ---- -# The adapter discovers namespaces/pods/services to resolve metric associations. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: prometheus-adapter-resource-reader -rules: -- apiGroups: [""] - resources: ["namespaces", "pods", "services", "configmaps"] - verbs: ["get", "list", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: prometheus-adapter-resource-reader -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: prometheus-adapter-resource-reader -subjects: -- kind: ServiceAccount - name: prometheus-adapter - namespace: ate-demo-autoscaled-workerpool ---- -# Allow the built-in HPA controller to read external metrics from the aggregated API. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: prometheus-adapter-external-metrics-reader -rules: -- apiGroups: ["external.metrics.k8s.io"] - resources: ["*"] - verbs: ["get", "list", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: hpa-controller-external-metrics -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: prometheus-adapter-external-metrics-reader -subjects: -- kind: ServiceAccount - name: horizontal-pod-autoscaler - namespace: kube-system From 161aa67852256bb9865faa0e9582c247d483079c Mon Sep 17 00:00:00 2001 From: ILLUM1N0X Date: Tue, 20 Oct 2026 20:24:52 +0000 Subject: [PATCH 16/63] Snapshot transfers on GCS run on a credential scoped to the actor (#2924) Part of #777 Every snapshot fetch and upload on GCS now runs on a token limited to the actor's own snapshot prefixes, read-write on its own and read-only on the tag it was created from, instead of the node's project-wide credential. atelet mints a worker-for-actor JWT per actor through `MintWorkerActorJwt`, caches it until five minutes before expiry, and sends it on every transfer. The node plugin on GCS trades it at the exchange the control sidecar serves (#2762) for a downscoped token, caches the token per actor until twenty minutes before expiry, and runs each transfer on a client built with it, closed when the transfer ends. A request GCS refuses is retried once with a token the exchange re-derives, which is what carries the first upload after a repoint, and a second refusal is `PERMISSION_DENIED`. S3 ignores the JWT and behaves as before. The atelet manifest gives the node sidecar the pod namespace and the service DNS trust bundle it dials the exchange with. Two e2e tests, GCS only, prove it against the bucket: the token reads the golden and cannot write under it, writes only under the actor's own prefix, reads its own snapshot and not a sibling's, and the exchange refuses the actor's own token kind, another audience, a stale UID, and a node the actor has left. With the `PerTemplateSnapshotLocation` gate, a repoint to another location is followed through the first suspend. Neither runs in PR CI, whose lane is kind over S3. Verified on a GKE dev cluster, installed from this branch with the gate on: the full e2e run passes, every suspend and resume in the demo and identity suites going through the new path, and both new tests pass. The six egress HTTPS tests were skipped, since they fail on main there too for lack of a GKE equivalent of the kind TLS setup script. Memory on the node sidecar was sampled through two demo suite runs: it peaks during transfers on the upload buffers, returns to its idle level between runs, and a thousand idle cached clients measure at 0.1 MiB locally. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- cmd/atelet/lifecycle_test.go | 4 +- cmd/atelet/main.go | 30 +- cmd/atelet/main_test.go | 4 +- cmd/atelet/snapshot_transfer.go | 36 +- cmd/atelet/snapshot_transfer_test.go | 84 ++- cmd/atelet/snapshotcache.go | 18 +- cmd/atelet/snapshotcache_linux_test.go | 4 +- cmd/atelet/snapshotcache_test.go | 24 +- cmd/atelet/worker_actor_jwt.go | 105 ++++ cmd/atelet/worker_actor_jwt_test.go | 173 ++++++ cmd/objectstore-plugin/README.md | 19 + cmd/objectstore-plugin/main.go | 22 +- internal/e2e/README.md | 9 + internal/e2e/backend.go | 54 ++ internal/e2e/backend_test.go | 53 ++ .../snapshotcredentials_test.go | 512 ++++++++++++++++++ .../snapshotcredentials/testmain_test.go | 24 + .../credential/credential.go | 6 + internal/objectstoreplugin/credentials.go | 143 +++++ .../objectstoreplugin/credentials_test.go | 256 +++++++++ .../objectstoreplugin/exchange/exchange.go | 4 + .../exchangeclient/exchangeclient.go | 167 ++++++ .../exchangeclient/exchangeclient_test.go | 254 +++++++++ .../objectstoreplugin/gcsissuer/issuer.go | 5 +- internal/objectstoreplugin/node.go | 85 ++- internal/objectstoreplugin/status.go | 5 + manifests/ate-install/atelet.yaml | 12 + pkg/objectstorage/transferclient_test.go | 33 ++ pkg/objectstorage/upload.go | 7 + pkg/proto/objectstorepb/v1/objectstore.pb.go | 8 +- pkg/proto/objectstorepb/v1/objectstore.proto | 8 +- 31 files changed, 2079 insertions(+), 89 deletions(-) create mode 100644 cmd/atelet/worker_actor_jwt.go create mode 100644 cmd/atelet/worker_actor_jwt_test.go create mode 100644 internal/e2e/backend.go create mode 100644 internal/e2e/backend_test.go create mode 100644 internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go create mode 100644 internal/e2e/suites/snapshotcredentials/testmain_test.go create mode 100644 internal/objectstoreplugin/credentials.go create mode 100644 internal/objectstoreplugin/credentials_test.go create mode 100644 internal/objectstoreplugin/exchangeclient/exchangeclient.go create mode 100644 internal/objectstoreplugin/exchangeclient/exchangeclient_test.go create mode 100644 pkg/objectstorage/transferclient_test.go diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index fbb0451460..322bf16051 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -806,7 +806,7 @@ func TestRestorePrefersLocalAndFallsBackToObjectStorage(t *testing.T) { } remoteSrcDir := t.TempDir() writeLocalSnapshot(t, remoteSrcDir, manifestRec, map[string]string{"checkpoint.img": "object-storage-memory"}) - if err := s.uploadSnapshot(t.Context(), uri, remoteSrcDir, &manifestRec); err != nil { + if err := s.uploadSnapshot(t.Context(), testActor, uri, remoteSrcDir, &manifestRec); err != nil { t.Fatalf("uploadSnapshot: %v", err) } @@ -944,7 +944,7 @@ func TestRestoreFailureRemovesStagedSnapshotFiles(t *testing.T) { } srcDir := t.TempDir() writeLocalSnapshot(t, srcDir, manifestRec, map[string]string{"checkpoint.img": "shared-memory"}) - if err := s.uploadSnapshot(t.Context(), uri, srcDir, &manifestRec); err != nil { + if err := s.uploadSnapshot(t.Context(), testActor, uri, srcDir, &manifestRec); err != nil { t.Fatalf("uploadSnapshot: %v", err) } advertiseRuntime(t, s, sandboxRec) diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 5e69327244..1ce756b21b 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -421,6 +421,7 @@ func main() { csiDriverConfigGetter, systemInfoVolumes, snapshots, + workerService, ) store.s = wmService snapshots.Start() @@ -717,6 +718,8 @@ type Atelet struct { snapshots *snapshottracker.Tracker // actorLocks serializes lifecycle RPCs per actor. actorLocks actorlock.Locks + // workerActorJWTs mints the JWT every snapshot transfer carries. + workerActorJWTs *workerActorJWTs } var _ ateletpb.AteletServer = (*Atelet)(nil) @@ -747,6 +750,7 @@ func NewService( csiDriverConfigGetter csi.CSIDriverConfigGetter, systemInfoVolumes *systemInfoVolumeRefresher, snapshots *snapshottracker.Tracker, + workers ateapipb.WorkerServiceClient, ) *Atelet { wms := &Atelet{ workerDialer: workerDialer, @@ -764,6 +768,7 @@ func NewService( csiDriverConfigGetter: csiDriverConfigGetter, systemInfoVolumes: systemInfoVolumes, snapshots: snapshots, + workerActorJWTs: newWorkerActorJWTs(workers), } return wms } @@ -1218,7 +1223,8 @@ func (s *Atelet) uploadExternalCheckpoint(ctx context.Context, req *ateletpb.Che if err != nil { return err } - return s.uploadSnapshot(ctx, uri, checkpointDir, rec, grpc.WaitForReady(true)) + actor := actorRef{atespace: req.GetAtespace(), name: req.GetActorName(), uid: req.GetActorUid()} + return s.uploadSnapshot(ctx, actor, uri, checkpointDir, rec, grpc.WaitForReady(true)) } // uploadSnapshot uploads rec's snapshot files from srcDir to uri through the @@ -1228,8 +1234,8 @@ func (s *Atelet) uploadExternalCheckpoint(ctx context.Context, req *ateletpb.Che // files, never a manifest pointing at files that never landed; retries // overwrite the deterministic object names. opts are passed to every plugin // call, and plugin errors are returned as uploadSnapshotFiles returns them. -func (s *Atelet) uploadSnapshot(ctx context.Context, uri resources.SnapshotURI, srcDir string, rec *sandboxAssetsRecord, opts ...grpc.CallOption) error { - if err := s.uploadSnapshotFiles(ctx, uri.String(), srcDir, rec.SnapshotFiles, opts...); err != nil { +func (s *Atelet) uploadSnapshot(ctx context.Context, actor actorRef, uri resources.SnapshotURI, srcDir string, rec *sandboxAssetsRecord, opts ...grpc.CallOption) error { + if err := s.uploadSnapshotFiles(ctx, actor, uri.String(), srcDir, rec.SnapshotFiles, opts...); err != nil { return fmt.Errorf("while uploading snapshot files: %w", err) } @@ -1237,7 +1243,7 @@ func (s *Atelet) uploadSnapshot(ctx context.Context, uri resources.SnapshotURI, if err != nil { return fmt.Errorf("while marshaling snapshot manifest: %w", err) } - if err := s.uploadManifest(ctx, uri.String(), manifest, opts...); err != nil { + if err := s.uploadManifest(ctx, actor, uri.String(), manifest, opts...); err != nil { return fmt.Errorf("while uploading snapshot manifest: %w", err) } return nil @@ -1313,6 +1319,7 @@ func (s *Atelet) PromoteSnapshot(ctx context.Context, req *ateletpb.PromoteSnaps // manifest was not read), and whether it uploaded: it does not when the local // copy is gone but already uploaded. Parameterized by localDir for tests. func (s *Atelet) uploadLocalCheckpointDir(ctx context.Context, req *ateletpb.PromoteSnapshotRequest, localDir string, uri resources.SnapshotURI) (sandboxClass string, uploaded bool, err error) { + actor := actorRef{atespace: req.GetAtespace(), name: req.GetActorName(), uid: req.GetActorUid()} localSnapshotName := req.GetSnapshot().GetSnapshotUuid() manifest, err := readSnapshotManifest(localDir) if errors.Is(err, os.ErrNotExist) { @@ -1320,7 +1327,7 @@ func (s *Atelet) uploadLocalCheckpointDir(ctx context.Context, req *ateletpb.Pro // it: the remote manifest is uploaded last, so its presence means the // whole snapshot is committed. Absent on both sides, the local // snapshot is unrecoverable. - _, fetchErr := s.fetchManifest(ctx, uri.String()) + _, fetchErr := s.fetchManifest(ctx, actor, uri.String()) if fetchErr == nil { slog.InfoContext(ctx, "Local snapshot already uploaded and pruned; nothing to do", slog.String("snapshot_uri", uri.String())) return "", false, nil @@ -1354,7 +1361,7 @@ func (s *Atelet) uploadLocalCheckpointDir(ctx context.Context, req *ateletpb.Pro // The local snapshot stays until the upload succeeds, so a retry can // upload it again. - if err := pluginsocket.CallError(s.uploadSnapshot(ctx, uri, localDir, rec)); err != nil { + if err := pluginsocket.CallError(s.uploadSnapshot(ctx, actor, uri, localDir, rec)); err != nil { return rec.SandboxClass, false, err } return rec.SandboxClass, true, nil @@ -1418,6 +1425,7 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res } actorUID := req.GetActorUid() + actor := actorRef{atespace: req.GetAtespace(), name: req.GetActorName(), uid: actorUID} actorRef := resources.ActorRef{Atespace: req.GetAtespace(), Name: req.GetActorName()} // The sandbox (binaries + pause image) that runs the restored workload is @@ -1540,7 +1548,7 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res slog.WarnContext(ctx, "Local snapshot is unusable; restoring from object storage", slog.String("local_dir", localDir), slog.Any("err", localErr)) } - manifest, err := s.fetchManifestCached(ctx, durable.GetSnapshotUri(), snapshotCacheMode(durable, cacheModeLink)) + manifest, err := s.fetchManifestCached(ctx, actor, durable.GetSnapshotUri(), snapshotCacheMode(durable, cacheModeLink)) if status.Code(err) == codes.NotFound { return nil, apierror.NotFound("snapshot %q has no usable local copy and no uploaded copy: %w", req.GetSnapshot().GetSnapshotUuid(), localErr) @@ -1597,7 +1605,7 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res // consumes it, the one the request resolved, which is what runs the // restored workload. mode := snapshotCacheMode(durable, cacheModeFor(runtimeRec.SandboxClass)) - return s.downloadExternalCheckpoint(gctx, durable.GetSnapshotUri(), checkpointDir, sandboxRec.SnapshotFiles, mode) + return s.downloadExternalCheckpoint(gctx, actor, durable.GetSnapshotUri(), checkpointDir, sandboxRec.SnapshotFiles, mode) }) g.Go(func() (err error) { defer func() { prepErr = err }() @@ -1903,12 +1911,12 @@ func copyRootFile(root *os.Root, src, dst string) (int64, error) { // is served from the cache (see downloadSnapshotFilesCached): the caller // asserts the snapshot is immutable at its URI. Otherwise the files are // downloaded into the restore directory directly. -func (s *Atelet) downloadExternalCheckpoint(ctx context.Context, snapshotURI string, dstDir string, files []string, mode cacheMode) error { +func (s *Atelet) downloadExternalCheckpoint(ctx context.Context, actor actorRef, snapshotURI string, dstDir string, files []string, mode cacheMode) error { var err error if mode != cacheModeOff && s.snapshotCache != nil { - err = s.downloadSnapshotFilesCached(ctx, snapshotURI, dstDir, files, mode) + err = s.downloadSnapshotFilesCached(ctx, actor, snapshotURI, dstDir, files, mode) } else { - err = s.fetchSnapshotFiles(ctx, snapshotURI, dstDir, files) + err = s.fetchSnapshotFiles(ctx, actor, snapshotURI, dstDir, files) } if err != nil { return fmt.Errorf("while downloading snapshot files: %w", err) diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index a1918db774..cae10e8d8b 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -508,7 +508,7 @@ func TestUploadSnapshotRejectsSymlinkOutsideRoot(t *testing.T) { t.Fatal(err) } store := &recordingObjectStorage{} - err = newPluginHerder(t, store).uploadSnapshot(context.Background(), uri, checkpointDir, + err = newPluginHerder(t, store).uploadSnapshot(context.Background(), testActor, uri, checkpointDir, &sandboxAssetsRecord{SnapshotFiles: []string{"checkpoint.img"}}) if err == nil { t.Fatal("uploadSnapshot() followed a symlink outside the checkpoint directory") @@ -541,7 +541,7 @@ func TestDownloadExternalCheckpointRejectsSymlinkOutsideRoot(t *testing.T) { t.Fatal(err) } err := newPluginHerder(t, store).downloadExternalCheckpoint( - context.Background(), testSnapshotURI, restoreDir, []string{"checkpoint.img"}, cacheModeOff) + context.Background(), testActor, testSnapshotURI, restoreDir, []string{"checkpoint.img"}, cacheModeOff) if err == nil { t.Fatal("downloadExternalCheckpoint() followed a symlink outside the restore directory") } diff --git a/cmd/atelet/snapshot_transfer.go b/cmd/atelet/snapshot_transfer.go index 83fdfa5b35..fedac43674 100644 --- a/cmd/atelet/snapshot_transfer.go +++ b/cmd/atelet/snapshot_transfer.go @@ -48,45 +48,57 @@ func pluginCallError(op, snapshotURI string, err error) error { return status.Errorf(code, "object-store plugin %s of %s failed (%s)", op, redactURLString(snapshotURI), code) } -// fetchSnapshotFiles downloads the named snapshot files into dstDir. -func (s *Atelet) fetchSnapshotFiles(ctx context.Context, snapshotURI, dstDir string, files []string) error { +// fetchSnapshotFiles downloads the named snapshot files into dstDir, with the +// JWT the plugin authorizes the transfer for actor with. +func (s *Atelet) fetchSnapshotFiles(ctx context.Context, actor actorRef, snapshotURI, dstDir string, files []string) error { if len(files) == 0 { return nil } - _, err := s.objectStorePlugin.FetchSnapshot(ctx, &objectstorev1.FetchSnapshotRequest{ + // A mint failure is shaped like a plugin failure and mapped the same way. + jwt, err := s.workerActorJWT(ctx, actor) + if err != nil { + return pluginsocket.CallError(err) + } + _, err = s.objectStorePlugin.FetchSnapshot(ctx, &objectstorev1.FetchSnapshotRequest{ SnapshotUri: snapshotURI, WritePath: dstDir, Files: files, + ActorJwt: jwt, }) return pluginsocket.CallError(pluginCallError("fetch", snapshotURI, err)) } // uploadSnapshotFiles uploads the named files in srcDir to the snapshot, // passing opts to the plugin call. It returns the plugin's code without its -// message, and leaves the Unavailable mapping to the caller: an upload that can -// be retried reports it through pluginsocket.CallError, one that cannot -// does not. -func (s *Atelet) uploadSnapshotFiles(ctx context.Context, snapshotURI, srcDir string, files []string, opts ...grpc.CallOption) error { +// message, or the mint's, and leaves the Unavailable mapping to the caller: +// an upload that can be retried reports it through pluginsocket.CallError, +// one that cannot does not. +func (s *Atelet) uploadSnapshotFiles(ctx context.Context, actor actorRef, snapshotURI, srcDir string, files []string, opts ...grpc.CallOption) error { if len(files) == 0 { return nil } - _, err := s.objectStorePlugin.UploadSnapshot(ctx, &objectstorev1.UploadSnapshotRequest{ + jwt, err := s.workerActorJWT(ctx, actor) + if err != nil { + return err + } + _, err = s.objectStorePlugin.UploadSnapshot(ctx, &objectstorev1.UploadSnapshotRequest{ SnapshotUri: snapshotURI, LocalPath: srcDir, Files: files, + ActorJwt: jwt, }, opts...) return pluginCallError("upload", snapshotURI, err) } // fetchManifest returns a snapshot's manifest. The returned error is the // plugin's gRPC status, so a missing manifest reports codes.NotFound. -func (s *Atelet) fetchManifest(ctx context.Context, snapshotURI string) ([]byte, error) { +func (s *Atelet) fetchManifest(ctx context.Context, actor actorRef, snapshotURI string) ([]byte, error) { dir, err := s.snapshotScratch() if err != nil { return nil, err } defer func() { _ = os.RemoveAll(dir) }() - if err := s.fetchSnapshotFiles(ctx, snapshotURI, dir, []string{sandboxManifestName}); err != nil { + if err := s.fetchSnapshotFiles(ctx, actor, snapshotURI, dir, []string{sandboxManifestName}); err != nil { return nil, err } return os.ReadFile(filepath.Join(dir, sandboxManifestName)) @@ -94,7 +106,7 @@ func (s *Atelet) fetchManifest(ctx context.Context, snapshotURI string) ([]byte, // uploadManifest uploads manifest as the snapshot's manifest, as // uploadSnapshotFiles does. -func (s *Atelet) uploadManifest(ctx context.Context, snapshotURI string, manifest []byte, opts ...grpc.CallOption) error { +func (s *Atelet) uploadManifest(ctx context.Context, actor actorRef, snapshotURI string, manifest []byte, opts ...grpc.CallOption) error { dir, err := s.snapshotScratch() if err != nil { return err @@ -103,7 +115,7 @@ func (s *Atelet) uploadManifest(ctx context.Context, snapshotURI string, manifes if err := os.WriteFile(filepath.Join(dir, sandboxManifestName), manifest, 0o600); err != nil { return fmt.Errorf("while staging snapshot manifest: %w", err) } - return s.uploadSnapshotFiles(ctx, snapshotURI, dir, []string{sandboxManifestName}, opts...) + return s.uploadSnapshotFiles(ctx, actor, snapshotURI, dir, []string{sandboxManifestName}, opts...) } // snapshotScratch creates a fresh directory for one manifest transfer. The diff --git a/cmd/atelet/snapshot_transfer_test.go b/cmd/atelet/snapshot_transfer_test.go index 62f048919c..3847a66a50 100644 --- a/cmd/atelet/snapshot_transfer_test.go +++ b/cmd/atelet/snapshot_transfer_test.go @@ -57,6 +57,7 @@ func newPluginHerder(t *testing.T, store objectstorage.ObjectStorage) *Atelet { } return &Atelet{ objectStorePlugin: objectstoreplugintest.NodeClient(plugin), + workerActorJWTs: testWorkerActorJWTs(), snapshotScratchDir: t.TempDir(), snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), sandboxAssetCache: newTestAssetCache(t), @@ -111,7 +112,7 @@ func TestUploadSnapshotManifestLast(t *testing.T) { } } - if err := s.uploadSnapshot(context.Background(), uri, dir, &rec); err != nil { + if err := s.uploadSnapshot(context.Background(), testActor, uri, dir, &rec); err != nil { t.Fatalf("uploadSnapshot: %v", err) } if len(store.puts) != 4 { @@ -125,7 +126,7 @@ func TestUploadSnapshotManifestLast(t *testing.T) { } // The manifest round-trips, and scratch space is cleaned up. - got, err := s.fetchManifest(context.Background(), uri.String()) + got, err := s.fetchManifest(context.Background(), testActor, uri.String()) if err != nil { t.Fatalf("fetchManifest: %v", err) } @@ -144,10 +145,10 @@ func TestUploadSnapshotManifestLast(t *testing.T) { func TestSnapshotTransferSkipsEmptyFileLists(t *testing.T) { store := &orderedObjectStorage{} s := newPluginHerder(t, store) - if err := s.downloadExternalCheckpoint(context.Background(), pausedSnapshotURI, t.TempDir(), nil, cacheModeOff); err != nil { + if err := s.downloadExternalCheckpoint(context.Background(), testActor, pausedSnapshotURI, t.TempDir(), nil, cacheModeOff); err != nil { t.Errorf("downloadExternalCheckpoint(no files) = %v, want nil", err) } - if err := s.uploadSnapshotFiles(context.Background(), pausedSnapshotURI, t.TempDir(), nil); err != nil { + if err := s.uploadSnapshotFiles(context.Background(), testActor, pausedSnapshotURI, t.TempDir(), nil); err != nil { t.Errorf("uploadSnapshotFiles(no files) = %v, want nil", err) } if len(store.puts) != 0 { @@ -190,13 +191,13 @@ func TestObjectStorePluginErrorCodeThroughAtelet(t *testing.T) { run func(ctx context.Context, s *Atelet) error }{ {"download", true, func(ctx context.Context, s *Atelet) error { - return s.downloadExternalCheckpoint(ctx, pausedSnapshotURI, t.TempDir(), []string{"a"}, cacheModeOff) + return s.downloadExternalCheckpoint(ctx, testActor, pausedSnapshotURI, t.TempDir(), []string{"a"}, cacheModeOff) }}, {"cached download, link", true, func(ctx context.Context, s *Atelet) error { - return s.downloadExternalCheckpoint(ctx, pausedSnapshotURI, t.TempDir(), []string{"a"}, cacheModeLink) + return s.downloadExternalCheckpoint(ctx, testActor, pausedSnapshotURI, t.TempDir(), []string{"a"}, cacheModeLink) }}, {"cached download, copy", true, func(ctx context.Context, s *Atelet) error { - return s.downloadExternalCheckpoint(ctx, pausedSnapshotURI, t.TempDir(), []string{"a"}, cacheModeCopy) + return s.downloadExternalCheckpoint(ctx, testActor, pausedSnapshotURI, t.TempDir(), []string{"a"}, cacheModeCopy) }}, {"paused upload", true, func(ctx context.Context, s *Atelet) error { dir := t.TempDir() @@ -221,6 +222,7 @@ func TestObjectStorePluginErrorCodeThroughAtelet(t *testing.T) { } s := &Atelet{ objectStorePlugin: failingNodePlugin{err: status.Error(pluginCode, "plugin failed")}, + workerActorJWTs: testWorkerActorJWTs(), snapshotScratchDir: t.TempDir(), snapshotCache: newTestAssetCache(t), } @@ -235,6 +237,59 @@ func TestObjectStorePluginErrorCodeThroughAtelet(t *testing.T) { } } +// A mint failure reaches ate-api-server the way a plugin failure does: +// Unavailable where the transfer can be retried, Internal where it cannot, +// and never NotFound, which callers read as a missing snapshot object. +func TestMintErrorCodeThroughAtelet(t *testing.T) { + uri, err := resources.ParseSnapshotURI(pausedSnapshotURI) + if err != nil { + t.Fatal(err) + } + rec := sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: testPauseImage, SnapshotFiles: []string{"a"}, Fidelity: ateattr.SnapshotFidelityMemory} + transfers := []struct { + name string + retryable bool + run func(ctx context.Context, s *Atelet) error + }{ + {"download", true, func(ctx context.Context, s *Atelet) error { + return s.downloadExternalCheckpoint(ctx, testActor, pausedSnapshotURI, t.TempDir(), []string{"a"}, cacheModeOff) + }}, + {"paused upload", true, func(ctx context.Context, s *Atelet) error { + dir := t.TempDir() + writeLocalSnapshot(t, dir, rec, map[string]string{"a": "a"}) + _, _, err := s.uploadLocalCheckpointDir(ctx, validPromoteSnapshotRequest(), dir, uri) + return err + }}, + {"checkpoint upload", false, func(ctx context.Context, s *Atelet) error { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "a"), []byte("a"), 0o600); err != nil { + return err + } + return s.uploadExternalCheckpoint(ctx, validCheckpointRequest(), dir, &rec) + }}, + } + for _, mint := range []codes.Code{codes.Unavailable, codes.NotFound, codes.Aborted, codes.Internal} { + for _, tr := range transfers { + t.Run(mint.String()+"/"+tr.name, func(t *testing.T) { + want := codes.Internal + if mint == codes.Unavailable && tr.retryable { + want = codes.Unavailable + } + s := &Atelet{ + objectStorePlugin: failingNodePlugin{err: status.Error(codes.Internal, "the plugin must not be reached")}, + workerActorJWTs: newWorkerActorJWTs(&fakeJWTMinter{err: status.Error(mint, "mint failed")}), + snapshotScratchDir: t.TempDir(), + } + _, err := ateinterceptors.InternalServerUnaryInterceptor(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "/test"}, + func(ctx context.Context, _ any) (any, error) { return nil, tr.run(ctx, s) }) + if got := status.Code(err); got != want { + t.Errorf("code = %s (%v), want %s", got, err, want) + } + }) + } + } +} + // An object-store plugin's status text can quote a signed URL or other credential. // Neither it nor the snapshot URI's query reaches atelet's RPC log or the // error ate-api-server receives: only the plugin's code and the snapshot URI @@ -256,21 +311,21 @@ func TestObjectStorePluginErrorKeepsCredentialsOut(t *testing.T) { run func(ctx context.Context, s *Atelet) error }{ {"download", signedURI, func(ctx context.Context, s *Atelet) error { - return s.downloadExternalCheckpoint(ctx, signedURI, t.TempDir(), []string{"a"}, cacheModeOff) + return s.downloadExternalCheckpoint(ctx, testActor, signedURI, t.TempDir(), []string{"a"}, cacheModeOff) }}, // The cached modes also build a cache key from the URI, which the // cache quotes in its errors and logs. {"cached download, link", signedURI, func(ctx context.Context, s *Atelet) error { - return s.downloadExternalCheckpoint(ctx, signedURI, t.TempDir(), []string{"a"}, cacheModeLink) + return s.downloadExternalCheckpoint(ctx, testActor, signedURI, t.TempDir(), []string{"a"}, cacheModeLink) }}, {"cached download, copy", signedURI, func(ctx context.Context, s *Atelet) error { - return s.downloadExternalCheckpoint(ctx, signedURI, t.TempDir(), []string{"a"}, cacheModeCopy) + return s.downloadExternalCheckpoint(ctx, testActor, signedURI, t.TempDir(), []string{"a"}, cacheModeCopy) }}, {"upload", signedURI, func(ctx context.Context, s *Atelet) error { - return s.uploadSnapshotFiles(ctx, signedURI, t.TempDir(), []string{"a"}) + return s.uploadSnapshotFiles(ctx, testActor, signedURI, t.TempDir(), []string{"a"}) }}, {"manifest fetch", signedURI, func(ctx context.Context, s *Atelet) error { - _, err := s.fetchManifest(ctx, signedURI) + _, err := s.fetchManifest(ctx, testActor, signedURI) return err }}, {"paused upload", "", func(ctx context.Context, s *Atelet) error { @@ -286,6 +341,7 @@ func TestObjectStorePluginErrorKeepsCredentialsOut(t *testing.T) { logs := captureLogs(t) s := &Atelet{ objectStorePlugin: failingNodePlugin{err: status.Error(pluginCode, "get "+signedURI+": AccountKey="+secret)}, + workerActorJWTs: testWorkerActorJWTs(), snapshotScratchDir: t.TempDir(), snapshotCache: newTestAssetCache(t), } @@ -401,6 +457,7 @@ func TestCheckpointUploadWaitsOutPluginOutage(t *testing.T) { } s := &Atelet{ objectStorePlugin: objectstorev1.NewNodeProviderClient(conn), + workerActorJWTs: testWorkerActorJWTs(), snapshotScratchDir: t.TempDir(), } rec := sandboxAssetsRecord{ @@ -462,9 +519,10 @@ func TestCheckpointUploadWaitsOutPluginOutage(t *testing.T) { func TestFetchManifestKeepsNotFound(t *testing.T) { s := &Atelet{ objectStorePlugin: failingNodePlugin{err: status.Error(codes.NotFound, "no manifest")}, + workerActorJWTs: testWorkerActorJWTs(), snapshotScratchDir: t.TempDir(), } - _, err := s.fetchManifest(context.Background(), pausedSnapshotURI) + _, err := s.fetchManifest(context.Background(), testActor, pausedSnapshotURI) if status.Code(err) != codes.NotFound { t.Errorf("fetchManifest = %v, want %s", err, codes.NotFound) } diff --git a/cmd/atelet/snapshotcache.go b/cmd/atelet/snapshotcache.go index 9b07457d68..6bf5dc4b71 100644 --- a/cmd/atelet/snapshotcache.go +++ b/cmd/atelet/snapshotcache.go @@ -179,7 +179,7 @@ func snapshotCacheMode(object *ateletpb.ObjectStorage, classMode cacheMode) cach // in dstDir or in its one checkpoint subdirectory, as the plugin's own // download allows; a destination that already exists, a symlink included, // fails rather than being followed. -func (s *Atelet) downloadSnapshotFilesCached(ctx context.Context, snapshotURI, dstDir string, files []string, mode cacheMode) error { +func (s *Atelet) downloadSnapshotFilesCached(ctx context.Context, actor actorRef, snapshotURI, dstDir string, files []string, mode cacheMode) error { if err := resources.ValidateSnapshotFileNames(files); err != nil { return err } @@ -199,7 +199,7 @@ func (s *Atelet) downloadSnapshotFilesCached(ctx context.Context, snapshotURI, d g, gCtx := errgroup.WithContext(ctx) for _, fileName := range files { g.Go(func() error { - if err := s.fetchSnapshotFile(gCtx, snapshotURI, fileName, filepath.Join(dstDir, fileName), mode); err != nil { + if err := s.fetchSnapshotFile(gCtx, actor, snapshotURI, fileName, filepath.Join(dstDir, fileName), mode); err != nil { return fmt.Errorf("while downloading %s: %w", fileName, err) } return nil @@ -230,9 +230,9 @@ func mkdirNoFollow(dir string) error { // the node makes no request to object storage at all. atelet reads the // manifest itself, so a link serves whatever class the files will take; // it is read from a scratch dir and the link dropped again. -func (s *Atelet) fetchManifestCached(ctx context.Context, snapshotURI string, mode cacheMode) ([]byte, error) { +func (s *Atelet) fetchManifestCached(ctx context.Context, actor actorRef, snapshotURI string, mode cacheMode) ([]byte, error) { if mode == cacheModeOff || s.snapshotCache == nil { - return s.fetchManifest(ctx, snapshotURI) + return s.fetchManifest(ctx, actor, snapshotURI) } dir, err := s.snapshotScratch() if err != nil { @@ -240,7 +240,7 @@ func (s *Atelet) fetchManifestCached(ctx context.Context, snapshotURI string, mo } defer func() { _ = os.RemoveAll(dir) }() local := filepath.Join(dir, sandboxManifestName) - if err := s.fetchSnapshotFile(ctx, snapshotURI, sandboxManifestName, local, cacheModeLink); err != nil { + if err := s.fetchSnapshotFile(ctx, actor, snapshotURI, sandboxManifestName, local, cacheModeLink); err != nil { return nil, err } return os.ReadFile(local) @@ -251,7 +251,7 @@ func (s *Atelet) fetchManifestCached(ctx context.Context, snapshotURI string, mo // link, so local stays valid regardless of later eviction; a cacheModeCopy // hit is a private copy. Fetch errors pass through the cache wrapped, so // the plugin's code classification survives. -func (s *Atelet) fetchSnapshotFile(ctx context.Context, snapshotURI, fileName, local string, mode cacheMode) error { +func (s *Atelet) fetchSnapshotFile(ctx context.Context, actor actorRef, snapshotURI, fileName, local string, mode cacheMode) error { // Keyed by the object's address without its query: a signed URL's // query is a credential, and the key is logged and written to meta.json. u, err := url.Parse(snapshotURI) @@ -264,7 +264,7 @@ func (s *Atelet) fetchSnapshotFile(ctx context.Context, snapshotURI, fileName, l } key := filecache.URIKey(objectURI) fetch := func(ctx context.Context, dst string) error { - return s.fetchSnapshotFileTo(ctx, snapshotURI, fileName, dst) + return s.fetchSnapshotFileTo(ctx, actor, snapshotURI, fileName, dst) } if mode == cacheModeCopy { return s.snapshotCache.GetFileCopyTo(ctx, key, local, fetch) @@ -292,13 +292,13 @@ func (s *Atelet) fetchSnapshotFile(ctx context.Context, snapshotURI, fileName, l // a scratch dir beside dst, on the cache's filesystem, and moves the file // into place. The plugin accepts a write path under the node state dir, // which is where the cache lives. -func (s *Atelet) fetchSnapshotFileTo(ctx context.Context, snapshotURI, fileName, dst string) error { +func (s *Atelet) fetchSnapshotFileTo(ctx context.Context, actor actorRef, snapshotURI, fileName, dst string) error { scratch, err := os.MkdirTemp(filepath.Dir(dst), ".fetch-*") if err != nil { return fmt.Errorf("while creating a snapshot fetch dir: %w", err) } defer func() { _ = os.RemoveAll(scratch) }() - if err := s.fetchSnapshotFiles(ctx, snapshotURI, scratch, []string{fileName}); err != nil { + if err := s.fetchSnapshotFiles(ctx, actor, snapshotURI, scratch, []string{fileName}); err != nil { return err } return os.Rename(filepath.Join(scratch, fileName), dst) diff --git a/cmd/atelet/snapshotcache_linux_test.go b/cmd/atelet/snapshotcache_linux_test.go index 59dfb30442..75ed7af2ff 100644 --- a/cmd/atelet/snapshotcache_linux_test.go +++ b/cmd/atelet/snapshotcache_linux_test.go @@ -54,7 +54,7 @@ func TestDownloadExternalCheckpointLinkFallsBackToCopyAcrossMounts(t *testing.T) if err := os.Mkdir(first, 0o700); err != nil { t.Fatal(err) } - if err := s.downloadExternalCheckpoint(ctx, testSnapshotURI, first, files, cacheModeLink); err != nil { + if err := s.downloadExternalCheckpoint(ctx, testActor, testSnapshotURI, first, files, cacheModeLink); err != nil { t.Fatalf("cross-mount link-mode restore: %v", err) } if got, err := os.ReadFile(filepath.Join(first, "memory-ranges")); err != nil || string(got) != "golden memory" { @@ -70,7 +70,7 @@ func TestDownloadExternalCheckpointLinkFallsBackToCopyAcrossMounts(t *testing.T) if err := os.Mkdir(second, 0o700); err != nil { t.Fatal(err) } - if err := s.downloadExternalCheckpoint(ctx, testSnapshotURI, second, files, cacheModeLink); err != nil { + if err := s.downloadExternalCheckpoint(ctx, testActor, testSnapshotURI, second, files, cacheModeLink); err != nil { t.Fatal(err) } if n := store.gets.Load(); n != 1 { diff --git a/cmd/atelet/snapshotcache_test.go b/cmd/atelet/snapshotcache_test.go index f1d947d041..109315c78d 100644 --- a/cmd/atelet/snapshotcache_test.go +++ b/cmd/atelet/snapshotcache_test.go @@ -246,7 +246,7 @@ func TestFetchManifestCached(t *testing.T) { } for i := range 2 { - got, err := s.fetchManifestCached(ctx, testSnapshotURI, cacheModeLink) + got, err := s.fetchManifestCached(ctx, testActor, testSnapshotURI, cacheModeLink) if err != nil { t.Fatalf("fetchManifestCached #%d: %v", i+1, err) } @@ -262,7 +262,7 @@ func TestFetchManifestCached(t *testing.T) { } // Not shared: read from object storage, as before the cache existed. - if _, err := s.fetchManifestCached(ctx, testSnapshotURI, cacheModeOff); err != nil { + if _, err := s.fetchManifestCached(ctx, testActor, testSnapshotURI, cacheModeOff); err != nil { t.Fatal(err) } if n := store.gets.Load(); n != 2 { @@ -278,7 +278,7 @@ func TestDownloadExternalCheckpointServesFromCache(t *testing.T) { // Two cached restores: one download, both destinations share the inode. first, second := restoreDir(t, base, "first"), restoreDir(t, base, "second") for _, dir := range []string{first, second} { - if err := s.downloadExternalCheckpoint(ctx, testSnapshotURI, dir, files, cacheModeLink); err != nil { + if err := s.downloadExternalCheckpoint(ctx, testActor, testSnapshotURI, dir, files, cacheModeLink); err != nil { t.Fatalf("downloadExternalCheckpoint into %s: %v", dir, err) } if got, err := os.ReadFile(filepath.Join(dir, "memory-ranges")); err != nil || string(got) != "golden memory" { @@ -299,7 +299,7 @@ func TestDownloadExternalCheckpointServesFromCache(t *testing.T) { // A non-cacheable restore downloads fresh and shares nothing. direct := restoreDir(t, base, "direct") - if err := s.downloadExternalCheckpoint(ctx, testSnapshotURI, direct, files, cacheModeOff); err != nil { + if err := s.downloadExternalCheckpoint(ctx, testActor, testSnapshotURI, direct, files, cacheModeOff); err != nil { t.Fatalf("direct downloadExternalCheckpoint: %v", err) } if n := store.gets.Load(); n != 2 { @@ -320,7 +320,7 @@ func TestDownloadExternalCheckpointCopyMode(t *testing.T) { first, second := restoreDir(t, base, "first"), restoreDir(t, base, "second") for _, dir := range []string{first, second} { - if err := s.downloadExternalCheckpoint(ctx, testSnapshotURI, dir, files, cacheModeCopy); err != nil { + if err := s.downloadExternalCheckpoint(ctx, testActor, testSnapshotURI, dir, files, cacheModeCopy); err != nil { t.Fatalf("downloadExternalCheckpoint into %s: %v", dir, err) } } @@ -342,7 +342,7 @@ func TestDownloadExternalCheckpointCopyMode(t *testing.T) { t.Fatalf("consumer write to its copy: %v", err) } third := restoreDir(t, base, "third") - if err := s.downloadExternalCheckpoint(ctx, testSnapshotURI, third, files, cacheModeCopy); err != nil { + if err := s.downloadExternalCheckpoint(ctx, testActor, testSnapshotURI, third, files, cacheModeCopy); err != nil { t.Fatal(err) } if got, err := os.ReadFile(filepath.Join(third, "memory-ranges")); err != nil || string(got) != "golden memory" { @@ -358,7 +358,7 @@ func TestDownloadExternalCheckpointWithoutCacheDownloads(t *testing.T) { s.snapshotCache = nil // caching disabled (--snapshot-cache-dir="") dir := restoreDir(t, base, "out") - if err := s.downloadExternalCheckpoint(context.Background(), testSnapshotURI, dir, []string{"memory-ranges"}, cacheModeLink); err != nil { + if err := s.downloadExternalCheckpoint(context.Background(), testActor, testSnapshotURI, dir, []string{"memory-ranges"}, cacheModeLink); err != nil { t.Fatalf("downloadExternalCheckpoint with nil cache: %v", err) } if got, err := os.ReadFile(filepath.Join(dir, "memory-ranges")); err != nil || string(got) != "golden memory" { @@ -372,7 +372,7 @@ func TestDownloadExternalCheckpointWithoutCacheDownloads(t *testing.T) { func TestDownloadExternalCheckpointCachedFetchErrorReachesCaller(t *testing.T) { s, _, base := newSnapshotCacheHerder(t, map[string]string{"memory-ranges": "golden memory"}) dir := restoreDir(t, base, "out") - err := s.downloadExternalCheckpoint(context.Background(), testSnapshotURI, dir, []string{"missing"}, cacheModeLink) + err := s.downloadExternalCheckpoint(context.Background(), testActor, testSnapshotURI, dir, []string{"missing"}, cacheModeLink) if err == nil { t.Fatal("downloadExternalCheckpoint succeeded for a missing file") } @@ -394,7 +394,7 @@ func TestDownloadExternalCheckpointSharesOneDownload(t *testing.T) { for i := range restores { dir := restoreDir(t, base, fmt.Sprintf("actor-%d", i)) wg.Go(func() { - errs[i] = s.downloadExternalCheckpoint(context.Background(), testSnapshotURI, dir, files, cacheModeLink) + errs[i] = s.downloadExternalCheckpoint(context.Background(), testActor, testSnapshotURI, dir, files, cacheModeLink) }) } wg.Wait() @@ -424,7 +424,7 @@ func TestDownloadExternalCheckpointCachedSubdirAndSymlinks(t *testing.T) { // Two files share the subdirectory and are fetched in parallel: the // directory is made once, before either, not raced into by both. dir := restoreDir(t, base, "subdir") - if err := s.downloadExternalCheckpoint(ctx, testSnapshotURI, dir, []string{"memory-ranges", "fs/data.tar", "fs/rootfs-upper-app.tar"}, cacheModeLink); err != nil { + if err := s.downloadExternalCheckpoint(ctx, testActor, testSnapshotURI, dir, []string{"memory-ranges", "fs/data.tar", "fs/rootfs-upper-app.tar"}, cacheModeLink); err != nil { t.Fatalf("downloadExternalCheckpoint with subdirectory files: %v", err) } for name, want := range map[string]string{"fs/data.tar": "fs delta", "fs/rootfs-upper-app.tar": "rootfs delta"} { @@ -445,14 +445,14 @@ func TestDownloadExternalCheckpointCachedSubdirAndSymlinks(t *testing.T) { if err := os.Symlink(keep, filepath.Join(fileLink, "memory-ranges")); err != nil { t.Fatal(err) } - if err := s.downloadExternalCheckpoint(ctx, testSnapshotURI, fileLink, []string{"memory-ranges"}, cacheModeLink); err == nil { + if err := s.downloadExternalCheckpoint(ctx, testActor, testSnapshotURI, fileLink, []string{"memory-ranges"}, cacheModeLink); err == nil { t.Error("a symlink in place of the restore file was followed") } dirLink := restoreDir(t, base, "dir-link") if err := os.Symlink(outside, filepath.Join(dirLink, "fs")); err != nil { t.Fatal(err) } - if err := s.downloadExternalCheckpoint(ctx, testSnapshotURI, dirLink, []string{"fs/data.tar"}, cacheModeCopy); err == nil { + if err := s.downloadExternalCheckpoint(ctx, testActor, testSnapshotURI, dirLink, []string{"fs/data.tar"}, cacheModeCopy); err == nil { t.Error("a symlink in place of the checkpoint subdirectory was followed") } if got, err := os.ReadFile(keep); err != nil || string(got) != "keep me" { diff --git a/cmd/atelet/worker_actor_jwt.go b/cmd/atelet/worker_actor_jwt.go new file mode 100644 index 0000000000..00b89c49c9 --- /dev/null +++ b/cmd/atelet/worker_actor_jwt.go @@ -0,0 +1,105 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/utils/lru" + + "go.ate.dev/substrate/internal/objectstoreplugin/exchange" + "go.ate.dev/substrate/pkg/proto/ateapipb" +) + +// actorRef identifies the actor a transfer is for, as atelet's RPCs carry it. +type actorRef struct { + atespace, name, uid string +} + +const ( + // workerActorJWTRefreshMargin is the least remaining life a cached JWT is used + // with. The JWT only has to be valid when the node plugin exchanges it, + // not for the duration of a transfer. + workerActorJWTRefreshMargin = 5 * time.Minute + // workerActorJWTCacheSize bounds the per-actor cache. A miss costs one mint, + // never correctness. + workerActorJWTCacheSize = 10_000 +) + +type cachedWorkerActorJWT struct { + jwt string + expiry time.Time +} + +// workerActorJWTs mints worker-for-actor JWTs through ate-api-server and caches +// them per actor UID. The node plugin trades one for storage credentials +// scoped to the actor, so atelet sends one with every snapshot transfer. +type workerActorJWTs struct { + workers ateapipb.WorkerServiceClient + now func() time.Time + cache *lru.Cache +} + +func newWorkerActorJWTs(workers ateapipb.WorkerServiceClient) *workerActorJWTs { + return &workerActorJWTs{workers: workers, now: time.Now, cache: lru.New(workerActorJWTCacheSize)} +} + +// get returns a JWT for the actor. The cache serves it while more than +// workerActorJWTRefreshMargin remains, otherwise ate-api-server mints a new +// one. A mint failure comes back as a gRPC status with the code a plugin +// failure would carry, so the transfer helpers handle both the same way: +// - Unavailable: ate-api-server could not be reached. Kept, so the callers +// that retry a transfer do. +// - FailedPrecondition: the actor is not this node's to transfer for, which +// the mint reports as NotFound or Aborted. Renamed because callers read +// NotFound from a transfer as a missing snapshot object. +// - Internal: anything else, including an empty JWT in the response. +func (j *workerActorJWTs) get(ctx context.Context, actor actorRef) (string, error) { + if v, ok := j.cache.Get(actor.uid); ok { + if c := v.(cachedWorkerActorJWT); c.expiry.Sub(j.now()) > workerActorJWTRefreshMargin { + return c.jwt, nil + } + } + resp, err := j.workers.MintWorkerActorJwt(ctx, &ateapipb.MintWorkerActorJwtRequest{ + Actor: &ateapipb.ObjectRef{Atespace: actor.atespace, Name: actor.name}, + ActorUid: actor.uid, + Audiences: []string{exchange.ActorJWTAudience}, + }) + switch status.Code(err) { + case codes.OK: + case codes.Unavailable: + return "", status.Errorf(codes.Unavailable, "while minting the actor JWT for %s/%s: %s", actor.atespace, actor.name, status.Convert(err).Message()) + case codes.NotFound, codes.Aborted: + return "", status.Errorf(codes.FailedPrecondition, "actor %s/%s cannot be transferred from this node: %s", actor.atespace, actor.name, status.Convert(err).Message()) + default: + return "", status.Errorf(codes.Internal, "while minting the actor JWT for %s/%s: %s", actor.atespace, actor.name, status.Convert(err).Message()) + } + if resp.GetActorJwt() == "" { + return "", status.Errorf(codes.Internal, "ate-api-server minted an empty actor JWT for %s/%s", actor.atespace, actor.name) + } + j.cache.Add(actor.uid, cachedWorkerActorJWT{jwt: resp.GetActorJwt(), expiry: resp.GetExpireTime().AsTime()}) + return resp.GetActorJwt(), nil +} + +// workerActorJWT is the JWT a transfer for actor carries. +func (s *Atelet) workerActorJWT(ctx context.Context, actor actorRef) (string, error) { + if s.workerActorJWTs == nil { + return "", status.Error(codes.Internal, "actor JWT source is not configured") + } + return s.workerActorJWTs.get(ctx, actor) +} diff --git a/cmd/atelet/worker_actor_jwt_test.go b/cmd/atelet/worker_actor_jwt_test.go new file mode 100644 index 0000000000..07eae46875 --- /dev/null +++ b/cmd/atelet/worker_actor_jwt_test.go @@ -0,0 +1,173 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" + + "go.ate.dev/substrate/internal/objectstoreplugin/exchange" + "go.ate.dev/substrate/pkg/proto/ateapipb" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" +) + +// testActor is the actor the transfer tests act for. +var testActor = actorRef{atespace: "ate-demo", name: "bot", uid: "9f1e0b2a-9e4f-4c3d-8a6b-5f2e1d0c9b8a"} + +// fakeJWTMinter answers MintWorkerActorJwt with a fixed token and records +// what it was asked for. +type fakeJWTMinter struct { + ateapipb.WorkerServiceClient + jwt string + expiry time.Time + err error + got []*ateapipb.MintWorkerActorJwtRequest +} + +func (m *fakeJWTMinter) MintWorkerActorJwt(_ context.Context, in *ateapipb.MintWorkerActorJwtRequest, _ ...grpc.CallOption) (*ateapipb.MintWorkerActorJwtResponse, error) { + m.got = append(m.got, in) + if m.err != nil { + return nil, m.err + } + return &ateapipb.MintWorkerActorJwtResponse{ActorJwt: m.jwt, ExpireTime: timestamppb.New(m.expiry)}, nil +} + +// testWorkerActorJWTs is the JWT source transfer tests give an Atelet. +func testWorkerActorJWTs() *workerActorJWTs { + return newWorkerActorJWTs(&fakeJWTMinter{jwt: "test-actor-jwt", expiry: time.Now().Add(time.Hour)}) +} + +func TestWorkerActorJWTs_MintsOncePerActorUntilNearExpiry(t *testing.T) { + now := time.Date(2026, 10, 16, 10, 0, 0, 0, time.UTC) + clock := now + minter := &fakeJWTMinter{jwt: "jwt-1", expiry: now.Add(time.Hour)} + j := newWorkerActorJWTs(minter) + j.now = func() time.Time { return clock } + + for range 2 { + jwt, err := j.get(t.Context(), testActor) + if err != nil || jwt != "jwt-1" { + t.Fatalf("get = %q, %v", jwt, err) + } + } + if len(minter.got) != 1 { + t.Fatalf("minted %d times, want 1", len(minter.got)) + } + req := minter.got[0] + if req.GetActor().GetAtespace() != testActor.atespace || req.GetActor().GetName() != testActor.name || req.GetActorUid() != testActor.uid { + t.Errorf("mint request names %v uid=%s", req.GetActor(), req.GetActorUid()) + } + if len(req.GetAudiences()) != 1 || req.GetAudiences()[0] != exchange.ActorJWTAudience { + t.Errorf("mint audiences = %v, want the exchange's", req.GetAudiences()) + } + + other := testActor + other.uid = "another-uid" + if _, err := j.get(t.Context(), other); err != nil { + t.Fatal(err) + } + if len(minter.got) != 2 { + t.Fatalf("a different actor should mint, minted %d times", len(minter.got)) + } + + // Under the refresh margin the cached JWT is not used. + clock = now.Add(time.Hour - workerActorJWTRefreshMargin + time.Second) + minter.jwt = "jwt-2" + jwt, err := j.get(t.Context(), testActor) + if err != nil || jwt != "jwt-2" { + t.Fatalf("get near expiry = %q, %v, want a fresh mint", jwt, err) + } +} + +func TestWorkerActorJWTs_Errors(t *testing.T) { + for _, tc := range []struct { + name string + minter *fakeJWTMinter + want codes.Code + }{ + // NotFound and Aborted must not reach callers as NotFound, which they + // read from a transfer as a missing snapshot object. + {"not this node's actor", &fakeJWTMinter{err: status.Error(codes.NotFound, "actor not found")}, codes.FailedPrecondition}, + {"recreated actor", &fakeJWTMinter{err: status.Error(codes.Aborted, "uid changed")}, codes.FailedPrecondition}, + {"ate-api-server down", &fakeJWTMinter{err: status.Error(codes.Unavailable, "connection refused")}, codes.Unavailable}, + {"ate-api-server refuses atelet", &fakeJWTMinter{err: status.Error(codes.Unauthenticated, "bad cert")}, codes.Internal}, + {"empty token", &fakeJWTMinter{jwt: "", expiry: time.Now().Add(time.Hour)}, codes.Internal}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := newWorkerActorJWTs(tc.minter).get(t.Context(), testActor) + if status.Code(err) != tc.want { + t.Fatalf("code = %v (%v), want %v", status.Code(err), err, tc.want) + } + }) + } + t.Run("no source configured", func(t *testing.T) { + _, err := (&Atelet{}).workerActorJWT(t.Context(), testActor) + if status.Code(err) != codes.Internal { + t.Fatalf("code = %v, want Internal", status.Code(err)) + } + }) +} + +// recordingNodePlugin captures the transfer requests atelet sends. +type recordingNodePlugin struct { + objectstorev1.NodeProviderClient + fetches []*objectstorev1.FetchSnapshotRequest + uploads []*objectstorev1.UploadSnapshotRequest +} + +func (p *recordingNodePlugin) FetchSnapshot(_ context.Context, in *objectstorev1.FetchSnapshotRequest, _ ...grpc.CallOption) (*objectstorev1.FetchSnapshotResponse, error) { + p.fetches = append(p.fetches, in) + return &objectstorev1.FetchSnapshotResponse{}, nil +} + +func (p *recordingNodePlugin) UploadSnapshot(_ context.Context, in *objectstorev1.UploadSnapshotRequest, _ ...grpc.CallOption) (*objectstorev1.UploadSnapshotResponse, error) { + p.uploads = append(p.uploads, in) + return &objectstorev1.UploadSnapshotResponse{}, nil +} + +func TestTransfersCarryTheActorJWT(t *testing.T) { + plugin := &recordingNodePlugin{} + s := &Atelet{objectStorePlugin: plugin, workerActorJWTs: testWorkerActorJWTs()} + if err := s.fetchSnapshotFiles(t.Context(), testActor, pausedSnapshotURI, t.TempDir(), []string{"a"}); err != nil { + t.Fatal(err) + } + if err := s.uploadSnapshotFiles(t.Context(), testActor, pausedSnapshotURI, t.TempDir(), []string{"a"}); err != nil { + t.Fatal(err) + } + if len(plugin.fetches) != 1 || plugin.fetches[0].GetActorJwt() != "test-actor-jwt" { + t.Errorf("fetch carried actor_jwt %q", plugin.fetches[0].GetActorJwt()) + } + if len(plugin.uploads) != 1 || plugin.uploads[0].GetActorJwt() != "test-actor-jwt" { + t.Errorf("upload carried actor_jwt %q", plugin.uploads[0].GetActorJwt()) + } +} + +func TestTransfersStopWhenTheMintFails(t *testing.T) { + plugin := &recordingNodePlugin{} + s := &Atelet{objectStorePlugin: plugin, workerActorJWTs: newWorkerActorJWTs(&fakeJWTMinter{err: status.Error(codes.NotFound, "actor not found")})} + err := s.fetchSnapshotFiles(t.Context(), testActor, pausedSnapshotURI, t.TempDir(), []string{"a"}) + if status.Code(err) != codes.FailedPrecondition { + t.Fatalf("err = %v, want FailedPrecondition for an actor this node cannot transfer for", err) + } + if len(plugin.fetches) != 0 { + t.Fatal("the plugin was called without a JWT") + } +} diff --git a/cmd/objectstore-plugin/README.md b/cmd/objectstore-plugin/README.md index ce02d4c5c1..a1460802e4 100644 --- a/cmd/objectstore-plugin/README.md +++ b/cmd/objectstore-plugin/README.md @@ -15,6 +15,19 @@ snapshot prefixes, read-write on its own and read-only on the tag it was created from. The exchange is private to this plugin, defined in `internal/proto/objectstoreexchangepb`. +On the node side, `node` mode on GCS dials that exchange, at +`--exchange-address` through the ate-api-server Service, verified with +`--exchange-ca-file` and `--exchange-server-name`. Each `FetchSnapshot` or +`UploadSnapshot` runs on a storage client built with the actor's token. The +token and the client are cached under the JWT they were obtained for, never +under a claim read out of it, while at least twenty minutes remain, so the +transfers that follow one another for an actor share connections. A client +that leaves the cache is not closed, its connections close once idle. A request +GCS refuses with a cached token is retried once with a token the exchange +re-derives, and a second refusal is `PERMISSION_DENIED`. A transfer without +`actor_jwt` is `INVALID_ARGUMENT` on GCS. S3 ignores the field and transfers +with the pod's own credentials. + The JWT is the one ate-api-server mints for atelet through `WorkerService.MintWorkerActorJwt`, with the audience named on the `actor_jwt` proto field. Its shape is defined in `internal/actoridjwt`, and @@ -35,6 +48,9 @@ Reusable by every provider: This binary's, GCS and S3: +- `internal/objectstoreplugin`: the node, asset and control plugins behind the + socket, and the `Credentials` seam a transfer gets its storage client from, + the pod's own on S3. - `internal/proto/objectstoreexchangepb`: the private wire format between the node and control sidecars. - `internal/objectstoreplugin/exchange`: the handler, and the `Issuer` interface @@ -43,6 +59,9 @@ This binary's, GCS and S3: rules. - `internal/objectstoreplugin/gcsissuer`: the GCS `Issuer`, STS exchange and token cache. +- `internal/objectstoreplugin/exchangeclient`: the node side on GCS, the + exchange call and the cache of tokens and storage clients per JWT. The + `Credentials` interface in `internal/objectstoreplugin` is its seam. - `cmd/objectstore-plugin`: the binary and its wiring. S3 has no issuer: its node plugin ignores the JWT and transfers with the diff --git a/cmd/objectstore-plugin/main.go b/cmd/objectstore-plugin/main.go index 5b432ea37d..9e42ba69f7 100644 --- a/cmd/objectstore-plugin/main.go +++ b/cmd/objectstore-plugin/main.go @@ -25,8 +25,10 @@ import ( "context" "fmt" "log/slog" + "net" "os" "os/signal" + "strconv" "syscall" "time" @@ -36,6 +38,8 @@ import ( "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/objectstore" "go.ate.dev/substrate/internal/objectstoreplugin" + "go.ate.dev/substrate/internal/objectstoreplugin/exchange" + "go.ate.dev/substrate/internal/objectstoreplugin/exchangeclient" "go.ate.dev/substrate/internal/pluginsocket" "go.ate.dev/substrate/internal/preview" "go.ate.dev/substrate/internal/serverboot" @@ -63,6 +67,11 @@ func main() { assetStagingDir := flags.String("asset-staging-dir", "/var/lib/objectstore-plugin/asset-staging", "node mode: directory sandbox assets are downloaded into and verified before they are copied below --root; must not be below --root or readable by the caller") assetStagingCapacity := flags.String("asset-staging-capacity", "8Gi", "node mode: the most sandbox asset downloads in --asset-staging-dir may hold at once, as a Kubernetes quantity; downloads wait for space, and an asset larger than this is refused") timeout := flags.Duration("timeout", 5*time.Second, "healthcheck mode: how long to wait for the plugin to report that it is serving") + apiHost := installdefaults.APIServiceHost(installdefaults.NamespaceFromPodEnv()) + var ncfg exchangeclient.Config + flags.StringVar(&ncfg.Address, "exchange-address", net.JoinHostPort(apiHost, strconv.Itoa(exchange.DefaultPort)), "node mode on GCS: address of the credential exchange the control sidecar serves, where each transfer's actor JWT is traded for a token scoped to the actor") + flags.StringVar(&ncfg.CAFile, "exchange-ca-file", "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "node mode on GCS: CA bundle used to verify the exchange") + flags.StringVar(&ncfg.ServerName, "exchange-server-name", apiHost, "node mode on GCS: DNS name expected on the exchange's certificate, the API Service in the pod's namespace") var xcfg exchangeConfig flags.StringVar(&xcfg.listen, "exchange-listen", "", "control mode: TCP address to serve the credential exchange on over TLS, for example :8445. Empty leaves it unserved.") flags.StringVar(&xcfg.credBundle, "exchange-cred-bundle", "/run/servicedns.podcert.ate.dev/credential-bundle.pem", "control mode: PEM credential bundle the exchange serves with, the pod's service DNS certificate") @@ -115,7 +124,18 @@ func main() { if err != nil { serverboot.Fatal(ctx, "Invalid --upload-memory-budget", err) } - plugin, err := objectstoreplugin.NewNodePlugin(objects, *root, uploadCapacity) + // On GCS every transfer runs on a token scoped to its actor, from the + // exchange. S3 has no per-actor credential and keeps the pod's own. + var nodeOpts []objectstoreplugin.NodeOption + if !objectstorage.UsesS3() { + creds, err := exchangeclient.New(ncfg) + if err != nil { + serverboot.Fatal(ctx, "Failed to set up the credential exchange client", err) + } + defer creds.Close() + nodeOpts = append(nodeOpts, objectstoreplugin.WithCredentials(creds)) + } + plugin, err := objectstoreplugin.NewNodePlugin(objects, *root, uploadCapacity, nodeOpts...) if err != nil { serverboot.Fatal(ctx, "Invalid node upload configuration", err) } diff --git a/internal/e2e/README.md b/internal/e2e/README.md index 7b72641665..798adf5b3f 100644 --- a/internal/e2e/README.md +++ b/internal/e2e/README.md @@ -84,6 +84,15 @@ Unset or empty means no gates. CI must pass it, even if only as empty, so a lane that forgets cannot silently skip every preview test; see [preview.go](preview.go). +## GCS-only suites + +`internal/e2e/suites/snapshotcredentials` runs only against an install that +stores snapshots in GCS, which it reads from the objectstore-plugin sidecar on +ate-api-server, and skips on any other backend, such as the kind lane's +S3-compatible service. A lane that is meant to be GCS sets +`E2E_REQUIRE_GCS=true`, which turns that skip into a failure, so a lane +misconfigured as S3 cannot pass by running nothing. + ## After a failure A suite deletes the namespaces it created only when it passed. A failed run diff --git a/internal/e2e/backend.go b/internal/e2e/backend.go new file mode 100644 index 0000000000..cb77758a03 --- /dev/null +++ b/internal/e2e/backend.go @@ -0,0 +1,54 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "os" + "testing" +) + +// GCSRequiredEnv makes a suite that needs GCS fail, rather than skip, on an +// install that stores snapshots elsewhere. A GCS e2e lane sets it, so a lane +// misconfigured as S3 cannot pass by running nothing. +const GCSRequiredEnv = "E2E_REQUIRE_GCS" + +// GCSRequired reports whether the lane declared itself a GCS lane. +func GCSRequired() bool { return os.Getenv(GCSRequiredEnv) == "true" } + +// RequireGCSBackend skips t unless backend, the install's snapshot backend +// as the objectstore-plugin reports it, is GCS. Under GCSRequired it fails +// instead. +func RequireGCSBackend(t testing.TB, backend string) { + t.Helper() + skip, fail := requireGCSDecision(backend, GCSRequired()) + if fail != "" { + t.Fatal(fail) + } + if skip != "" { + t.Skip(skip) + } +} + +// requireGCSDecision is RequireGCSBackend's rule, apart from testing.TB so +// its strict branch is proven without a cluster. +func requireGCSDecision(backend string, required bool) (skip, fail string) { + if backend == "gcs" { + return "", "" + } + if required { + return "", "snapshot backend is " + backend + " and GCS is required (" + GCSRequiredEnv + " is set)" + } + return "snapshot backend is " + backend + ", this suite needs GCS", "" +} diff --git a/internal/e2e/backend_test.go b/internal/e2e/backend_test.go new file mode 100644 index 0000000000..41e33be118 --- /dev/null +++ b/internal/e2e/backend_test.go @@ -0,0 +1,53 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import "testing" + +// recordingTB captures what RequireGCSBackend does to the test. +type recordingTB struct { + testing.TB + skipped, failed string +} + +func (r *recordingTB) Helper() {} +func (r *recordingTB) Skip(args ...any) { r.skipped = args[0].(string) } +func (r *recordingTB) Fatal(args ...any) { r.failed = args[0].(string) } +func (r *recordingTB) Skipped() bool { return r.skipped != "" } +func (r *recordingTB) Failed() bool { return r.failed != "" } + +func TestRequireGCSBackend(t *testing.T) { + for _, tc := range []struct { + name string + backend string + required string + skip bool + fail bool + }{ + {"gcs runs", "gcs", "", false, false}, + {"gcs runs when required", "gcs", "true", false, false}, + {"s3 skips", "s3", "", true, false}, + {"s3 fails when required", "s3", "true", false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(GCSRequiredEnv, tc.required) + rec := &recordingTB{TB: t} + RequireGCSBackend(rec, tc.backend) + if rec.Skipped() != tc.skip || rec.Failed() != tc.fail { + t.Fatalf("skipped=%v (%q) failed=%v (%q), want skip=%v fail=%v", rec.Skipped(), rec.skipped, rec.Failed(), rec.failed, tc.skip, tc.fail) + } + }) + } +} diff --git a/internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go b/internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go new file mode 100644 index 0000000000..bf343c5ff9 --- /dev/null +++ b/internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go @@ -0,0 +1,512 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package snapshotcredentials proves, against real GCS, that the credential +// exchange hands a node a token that reaches exactly the actor's own snapshot +// prefixes. It runs only on a GCS install. The kind lane stores snapshots in +// an S3-compatible service, where the exchange answers UNIMPLEMENTED, so the +// suite skips there, or fails when E2E_REQUIRE_GCS says a GCS lane was meant. +package snapshotcredentials + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "net/http" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "go.ate.dev/substrate/internal/actoridjwt" + "go.ate.dev/substrate/internal/ateclient" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/localjwtauthority" + "go.ate.dev/substrate/internal/objectstoreplugin/exchange" + "go.ate.dev/substrate/internal/portforward" + "go.ate.dev/substrate/internal/preview" + exchangepb "go.ate.dev/substrate/internal/proto/objectstoreexchangepb" + "go.ate.dev/substrate/pkg/proto/ateapipb" +) + +const probeTemplate = "probe" + +// requireGCS skips, or fails under e2e.GCSRequired, unless the install under +// test stores snapshots in GCS. The backend is the objectstore-plugin +// sidecar's ATE_STORAGE_BACKEND on ate-api-server. +func requireGCS(t *testing.T, ctx context.Context, clients *e2e.Clients) { + t.Helper() + deploy, err := clients.K8s.AppsV1().Deployments(e2e.SystemNamespace()).Get(ctx, e2e.ResourceName("ate-api-server"), metav1.GetOptions{}) + if err != nil { + t.Fatalf("getting the ate-api-server Deployment: %v", err) + } + backend := "gcs" + for _, c := range deploy.Spec.Template.Spec.InitContainers { + if c.Name != "objectstore-plugin" { + continue + } + for _, env := range c.Env { + if env.Name == "ATE_STORAGE_BACKEND" && env.Value != "" { + backend = env.Value + } + } + } + e2e.RequireGCSBackend(t, backend) +} + +// signer mints worker-for-actor JWTs with the install's own pool, as +// ate-api-server would for atelet. The harness cannot call +// MintWorkerActorJwt, which needs atelet's identity, so it reads the pool +// Secret with its admin client and signs with the same code and keys. +type signer struct { + pool *localjwtauthority.ConcretePool + issuer string +} + +func newSigner(t *testing.T, ctx context.Context, clients *e2e.Clients) *signer { + t.Helper() + ns := e2e.SystemNamespace() + secret, err := clients.K8s.CoreV1().Secrets(ns).Get(ctx, e2e.ResourceName("actor-id-jwt-pool"), metav1.GetOptions{}) + if err != nil { + t.Fatalf("getting the actor JWT pool Secret: %v", err) + } + pool, err := localjwtauthority.Unmarshal(secret.Data["pool"]) + if err != nil { + t.Fatalf("decoding the actor JWT pool: %v", err) + } + cm, err := clients.K8s.CoreV1().ConfigMaps(ns).Get(ctx, e2e.ResourceName("ate-api-server-envvars"), metav1.GetOptions{}) + if err != nil { + t.Fatalf("getting the ate-api-server ConfigMap: %v", err) + } + issuer := cm.Data["ATE_API_ACTOR_JWT_ISSUER"] + if issuer == "" { + t.Fatal("ATE_API_ACTOR_JWT_ISSUER is empty in the ate-api-server ConfigMap") + } + return &signer{pool: pool, issuer: issuer} +} + +type claims struct { + kind actoridjwt.Kind + audience string + actor *ateapipb.Actor + uid string + node string +} + +// forActor is the token ate-api-server would mint for the node hosting actor. +func forActor(actor *ateapipb.Actor) claims { + return claims{ + kind: actoridjwt.KindWorkerForActor, audience: exchange.ActorJWTAudience, + actor: actor, uid: actor.GetMetadata().GetUid(), node: actor.GetStatus().GetAssignedNode(), + } +} + +func (s *signer) jwt(t *testing.T, c claims) string { + t.Helper() + now := time.Now() + meta := c.actor.GetMetadata() + token, err := s.pool.SignJWT(&actoridjwt.Claims{ + Kind: c.kind, + Issuer: s.issuer, + Subject: c.kind.SubjectPrefix() + "/" + meta.GetAtespace() + "/" + meta.GetName(), + Audiences: []string{c.audience}, + Expiration: now.Add(time.Hour), + NotBefore: now.Add(-time.Minute), + IssuedAt: now, + Substrate: actoridjwt.SubstrateClaims{Atespace: meta.GetAtespace(), ActorName: meta.GetName(), ActorUID: c.uid, Node: c.node}, + }) + if err != nil { + t.Fatalf("signing a %v token: %v", c.kind, err) + } + return token +} + +// dialExchange port-forwards the ate-api-server Service's exchange port and +// verifies it the way the node plugin does, with the service DNS trust bundle +// and the Service's DNS name. +func dialExchange(t *testing.T, ctx context.Context, clients *e2e.Clients) exchangepb.CredentialExchangeClient { + t.Helper() + config, err := ateclient.LoadKubeConfig(e2e.KubeConfig, e2e.KubeContext) + if err != nil { + t.Fatalf("loading kubeconfig: %v", err) + } + ns := e2e.SystemNamespace() + port, stop, err := portforward.ServicePortForward(ctx, config, clients.K8s, ns, e2e.ResourceName("api"), exchange.DefaultPort) + if err != nil { + t.Fatalf("port-forwarding the exchange: %v", err) + } + t.Cleanup(stop) + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM([]byte(e2e.ServiceDNSTrustBundle(t, ctx, clients))) { + t.Fatal("the service DNS trust bundle holds no certificate") + } + creds := credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS13, RootCAs: roots, ServerName: e2e.ResourceName("api") + "." + ns + ".svc"}) + conn, err := grpc.NewClient(fmt.Sprintf("127.0.0.1:%d", port), grpc.WithTransportCredentials(creds)) + if err != nil { + t.Fatalf("dialing the exchange: %v", err) + } + t.Cleanup(func() { conn.Close() }) + return exchangepb.NewCredentialExchangeClient(conn) +} + +// gcs sends one request to GCS's XML API with the downscoped token and +// returns the status code. uri is gs://bucket/object. +func gcs(t *testing.T, ctx context.Context, token, method, uri, body string) int { + t.Helper() + path := strings.TrimPrefix(uri, "gs://") + req, err := http.NewRequestWithContext(ctx, method, "https://storage.googleapis.com/"+path, strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer "+token) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("%s %s: %v", method, uri, err) + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + return resp.StatusCode +} + +func expectGCS(t *testing.T, ctx context.Context, what, token, method, uri, body string, want int) { + t.Helper() + if got := gcs(t, ctx, token, method, uri, body); got != want { + t.Errorf("%s: %s %s = %d, want %d", what, method, uri, got, want) + } +} + +// durableURI is the prefix of the actor's generation with the given owner, +// which must be COMPLETED in object storage. +func durableURI(t *testing.T, actor *ateapipb.Actor, owner ateapipb.SnapshotOwner) string { + t.Helper() + for _, snap := range actor.GetStatus().GetSnapshots() { + if snap.GetOwner() == owner && snap.GetDurableSnapshot().GetStatus() == ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + return snap.GetDurableSnapshot().GetObject().GetSnapshotUri() + } + } + t.Fatalf("actor %s has no completed %v snapshot: %v", actor.GetMetadata().GetName(), owner, actor.GetStatus().GetSnapshots()) + return "" +} + +func getActor(t *testing.T, ctx context.Context, clients *e2e.Clients, atespace, name string) *ateapipb.Actor { + t.Helper() + actor, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{Actor: &ateapipb.ObjectRef{Atespace: atespace, Name: name}}) + if err != nil { + t.Fatalf("GetActor %s: %v", name, err) + } + return actor +} + +func waitForActorState(t *testing.T, ctx context.Context, clients *e2e.Clients, atespace, name string, want ateapipb.ActorState) *ateapipb.Actor { + t.Helper() + deadline := time.Now().Add(2 * time.Minute) + for { + actor := getActor(t, ctx, clients, atespace, name) + if actor.GetStatus().GetState() == want { + return actor + } + if time.Now().After(deadline) { + t.Fatalf("actor %s is %v after two minutes, want %v", name, actor.GetStatus().GetState(), want) + } + time.Sleep(time.Second) + } +} + +// createAndResume creates an actor from the probe template and resumes it +// from the golden snapshot. The record outlives the fixture namespace, so a +// leftover from a failed run is cleared first. +func createAndResume(t *testing.T, ctx context.Context, clients *e2e.Clients, atespace, name string) *ateapipb.Actor { + t.Helper() + ref := &ateapipb.ObjectRef{Atespace: atespace, Name: name} + _, _ = clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: ref}) + _, _ = clients.SubstrateAPI.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: ref}) + if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: name}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: atespace, Name: probeTemplate}, + }}); err != nil { + t.Fatalf("CreateActor %s: %v", name, err) + } + t.Cleanup(func() { + // The test's context is cancelled by the time cleanup runs, and an + // actor left running crashes when the fixture's worker pool goes. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + _, _ = clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: ref}) + if _, err := clients.SubstrateAPI.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: ref}); err != nil { + t.Logf("cleanup: DeleteActor %s failed, actor leaked (remove with: kubectl ate delete actor %s -a %s): %v", name, name, atespace, err) + } + }) + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{Actor: ref}); err != nil { + t.Fatalf("ResumeActor %s: %v", name, err) + } + return waitForActorState(t, ctx, clients, atespace, name, ateapipb.ActorState_ACTOR_STATE_RUNNING) +} + +// suspendSync suspends the actor and waits for its snapshot to be durable. +func suspendSync(t *testing.T, ctx context.Context, clients *e2e.Clients, atespace, name string) *ateapipb.Actor { + t.Helper() + ref := &ateapipb.ObjectRef{Atespace: atespace, Name: name} + if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: ref, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}); err != nil { + t.Fatalf("SuspendActor %s: %v", name, err) + } + return waitForActorState(t, ctx, clients, atespace, name, ateapipb.ActorState_ACTOR_STATE_SUSPENDED) +} + +// TestExchangeScopesTheTokenToTheActor walks the exchange and the boundary +// it grants through one actor's life: fresh from a golden, it may read the +// golden and write only under its own prefix. Once it and a sibling have +// their own snapshots, it reads its own and not the sibling's. Tokens the +// exchange must refuse are checked alongside. +func TestExchangeScopesTheTokenToTheActor(t *testing.T) { + env, err := e2e.CheckEnv("BUCKET_NAME") + if err != nil { + t.Fatalf("CheckEnv failed: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + defer cancel() + clients := e2e.GetClients() + requireGCS(t, ctx, clients) + + atespace, _ := e2e.DeployProbe(t, env["BUCKET_NAME"], "snapcreds") + sign := newSigner(t, ctx, clients) + xchg := dialExchange(t, ctx, clients) + stamp := time.Now().UTC().Format("20060102T150405Z") + + // Both actors start from the golden snapshot, on whichever nodes the + // scheduler picks. + alpha := createAndResume(t, ctx, clients, atespace, "creds-alpha") + beta := createAndResume(t, ctx, clients, atespace, "creds-beta") + goldenURI := durableURI(t, alpha, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG) + + // A token for alpha, as its node would present it. + resp, err := xchg.ExchangeActorJwt(ctx, &exchangepb.ExchangeActorJwtRequest{ActorJwt: sign.jwt(t, forActor(alpha))}) + if err != nil { + t.Fatalf("exchange for a running actor: %v", err) + } + token := resp.GetAccessToken() + if left := time.Until(resp.GetExpireTime().AsTime()); token == "" || left <= 0 { + t.Fatalf("exchange returned token %q expiring in %v", token, left) + } + // The golden is readable and not writable. The actor's own prefix, + // which holds nothing yet, is writable. + expectGCS(t, ctx, "golden manifest read", token, http.MethodGet, goldenURI+"/manifest.json", "", http.StatusOK) + expectGCS(t, ctx, "golden write refused", token, http.MethodPut, goldenURI+"/e2e-"+stamp, "x", http.StatusForbidden) + own := ownPrefix(goldenURI, alpha) + expectGCS(t, ctx, "own prefix write", token, http.MethodPut, own+"/e2e-"+stamp, "hello", http.StatusOK) + expectGCS(t, ctx, "own prefix read back", token, http.MethodGet, own+"/e2e-"+stamp, "", http.StatusOK) + expectGCS(t, ctx, "own prefix delete", token, http.MethodDelete, own+"/e2e-"+stamp, "", http.StatusNoContent) + expectGCS(t, ctx, "sibling actor's prefix refused", token, http.MethodPut, ownPrefix(goldenURI, beta)+"/e2e-"+stamp, "x", http.StatusForbidden) + // GCS answers 403 outside the boundary and 404 for a missing object inside + // it, so the pair below tells a denied read from an absent object. + expectGCS(t, ctx, "sibling actor's prefix read refused", token, http.MethodGet, ownPrefix(goldenURI, beta)+"/e2e-"+stamp, "", http.StatusForbidden) + expectGCS(t, ctx, "own prefix read of a missing object", token, http.MethodGet, own+"/e2e-missing-"+stamp, "", http.StatusNotFound) + + // Tokens the exchange refuses, with the code the node plugin would see. + for _, tc := range []struct { + name string + jwt string + want codes.Code + }{ + {"the actor's own token", sign.jwt(t, claims{kind: actoridjwt.KindActor, audience: exchange.ActorJWTAudience, actor: alpha, uid: alpha.GetMetadata().GetUid()}), codes.PermissionDenied}, + {"another audience", sign.jwt(t, withAudience(forActor(alpha), "other")), codes.Unauthenticated}, + {"a recreated actor's old uid", sign.jwt(t, withUID(forActor(alpha), "00000000-0000-4000-8000-000000000000")), codes.NotFound}, + {"a node the actor is not on", sign.jwt(t, withNode(forActor(alpha), "not-the-node")), codes.PermissionDenied}, + {"an empty token", "", codes.InvalidArgument}, + } { + _, err := xchg.ExchangeActorJwt(ctx, &exchangepb.ExchangeActorJwtRequest{ActorJwt: tc.jwt}) + if status.Code(err) != tc.want { + t.Errorf("exchange of %s = %v (%v), want %v", tc.name, status.Code(err), err, tc.want) + } + } + + // Each actor gets its own snapshot. Alpha comes back, beta stays + // suspended with no node, so no node may act for it. + suspendSync(t, ctx, clients, atespace, "creds-alpha") + beta = suspendSync(t, ctx, clients, atespace, "creds-beta") + if _, err := xchg.ExchangeActorJwt(ctx, &exchangepb.ExchangeActorJwtRequest{ActorJwt: sign.jwt(t, withNode(forActor(beta), "any-node"))}); status.Code(err) != codes.PermissionDenied { + t.Errorf("exchange for a suspended actor held by no node = %v, want PermissionDenied", err) + } + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{Actor: &ateapipb.ObjectRef{Atespace: atespace, Name: "creds-alpha"}}); err != nil { + t.Fatalf("ResumeActor creds-alpha: %v", err) + } + alpha = waitForActorState(t, ctx, clients, atespace, "creds-alpha", ateapipb.ActorState_ACTOR_STATE_RUNNING) + + // The forced exchange re-derives the grants: the golden is gone from + // alpha's status, its own snapshot is in. + resp, err = xchg.ExchangeActorJwt(ctx, &exchangepb.ExchangeActorJwtRequest{ActorJwt: sign.jwt(t, forActor(alpha)), Force: true}) + if err != nil { + t.Fatalf("forced exchange after the first suspend: %v", err) + } + token = resp.GetAccessToken() + expectGCS(t, ctx, "own snapshot manifest read", token, http.MethodGet, durableURI(t, alpha, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR)+"/manifest.json", "", http.StatusOK) + expectGCS(t, ctx, "sibling snapshot manifest refused", token, http.MethodGet, durableURI(t, beta, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR)+"/manifest.json", "", http.StatusForbidden) + expectGCS(t, ctx, "golden read dropped with the tag generation", token, http.MethodGet, goldenURI+"/manifest.json", "", http.StatusForbidden) +} + +// ownPrefix is the actor's own prefix at the location its golden lives in, +// the read-write grant the exchange derives before the actor has a snapshot +// of its own: /atespaces//actors/. +func ownPrefix(goldenURI string, actor *ateapipb.Actor) string { + location, _, _ := strings.Cut(goldenURI, "/atespaces/") + return location + "/atespaces/" + actor.GetMetadata().GetAtespace() + "/actors/" + actor.GetMetadata().GetUid() +} + +// locationOf is the storage location a snapshot URI lives under, the part +// before its owner's atespaces segment. +func locationOf(snapshotURI string) string { + location, _, _ := strings.Cut(snapshotURI, "/atespaces/") + return location +} + +// ownPrefixAt is ownPrefix for a location given directly, for a template +// whose location differs from the golden's. +func ownPrefixAt(location string, actor *ateapipb.Actor) string { + return strings.TrimSuffix(location, "/") + "/atespaces/" + actor.GetMetadata().GetAtespace() + "/actors/" + actor.GetMetadata().GetUid() +} + +// TestRepointMovesTheGrant repoints an actor to a template whose snapshots +// live under another location and checks the exchange follows. Grants come +// from the URIs in the actor's status, so nothing names the new location +// until the first suspend records a pending snapshot there: until then the +// exchange keeps answering for the old location, the suspend's upload is the +// first request the new location sees, and it goes through on the node +// plugin's forced re-exchange. After it, an exchange without force mints for +// the new location and the old token is good for nothing the actor owns. +// Per-template locations are a preview feature, so the test needs the +// install to enable the gate and the run to pass --preview. +func TestRepointMovesTheGrant(t *testing.T) { + e2e.RequirePreview(t, preview.GatePerTemplateSnapshotLocation) + env, err := e2e.CheckEnv("BUCKET_NAME") + if err != nil { + t.Fatalf("CheckEnv failed: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute) + defer cancel() + clients := e2e.GetClients() + requireGCS(t, ctx, clients) + + atespace, probe := e2e.DeployProbe(t, env["BUCKET_NAME"], "snapcreds-repoint") + sign := newSigner(t, ctx, clients) + xchg := dialExchange(t, ctx, clients) + stamp := time.Now().UTC().Format("20060102T150405Z") + + // Template B is the probe template under another location in the same + // bucket. Its golden is not what the actor resumes from, but the actor's + // next snapshot goes under B's location. + const nameB = "probe-repoint" + locationB := "gs://" + env["BUCKET_NAME"] + "/ate-e2e-repoint-" + stamp + "/" + templateB := proto.Clone(probe).(*ateapipb.ActorTemplate) + templateB.Metadata = &ateapipb.ResourceMetadata{Atespace: atespace, Name: nameB} + templateB.Status = nil + templateB.SnapshotConfig = proto.Clone(probe.GetSnapshotConfig()).(*ateapipb.SnapshotConfig) + templateB.SnapshotConfig.StorageLocation = locationB + if _, err := clients.SubstrateAPI.CreateActorTemplate(ctx, &ateapipb.CreateActorTemplateRequest{ActorTemplate: templateB}); err != nil { + t.Fatalf("CreateActorTemplate %s: %v", nameB, err) + } + t.Cleanup(func() { + if _, err := clients.SubstrateAPI.DeleteActorTemplate(context.Background(), &ateapipb.DeleteActorTemplateRequest{ActorTemplate: &ateapipb.ObjectRef{Atespace: atespace, Name: nameB}}); err != nil { + t.Logf("cleanup: DeleteActorTemplate %s failed: %v", nameB, err) + } + }) + e2e.WaitForSubstrateTemplateReady(ctx, t, clients, atespace, nameB) + + // Running from template A, the token writes under A's location only. + const name = "creds-repoint" + actor := createAndResume(t, ctx, clients, atespace, name) + locationA := locationOf(durableURI(t, actor, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG)) + first, err := xchg.ExchangeActorJwt(ctx, &exchangepb.ExchangeActorJwtRequest{ActorJwt: sign.jwt(t, forActor(actor))}) + if err != nil { + t.Fatalf("exchange before the repoint: %v", err) + } + expectGCS(t, ctx, "old location write before the repoint", first.GetAccessToken(), http.MethodPut, ownPrefixAt(locationA, actor)+"/e2e-"+stamp, "a", http.StatusOK) + expectGCS(t, ctx, "old location delete before the repoint", first.GetAccessToken(), http.MethodDelete, ownPrefixAt(locationA, actor)+"/e2e-"+stamp, "", http.StatusNoContent) + expectGCS(t, ctx, "new location write before the repoint", first.GetAccessToken(), http.MethodPut, ownPrefixAt(locationB, actor)+"/e2e-"+stamp, "b", http.StatusForbidden) + + // A repoint needs a suspended actor that owns no snapshot, so the actor + // is reverted to its golden rather than suspended. + ref := &ateapipb.ObjectRef{Atespace: atespace, Name: name} + if _, err := clients.SubstrateAPI.RevertActor(ctx, &ateapipb.RevertActorRequest{Actor: ref}); err != nil { + t.Fatalf("RevertActor %s: %v", name, err) + } + actor = waitForActorState(t, ctx, clients, atespace, name, ateapipb.ActorState_ACTOR_STATE_SUSPENDED) + actor.ActorTemplate = &ateapipb.ObjectRef{Atespace: atespace, Name: nameB} + if _, err := clients.SubstrateAPI.UpdateActor(ctx, &ateapipb.UpdateActorRequest{Actor: actor}); err != nil { + t.Fatalf("UpdateActor %s to template %s: %v", name, nameB, err) + } + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{Actor: ref}); err != nil { + t.Fatalf("ResumeActor %s after the repoint: %v", name, err) + } + actor = waitForActorState(t, ctx, clients, atespace, name, ateapipb.ActorState_ACTOR_STATE_RUNNING) + + // The actor's status still names only A's location, so the exchange + // still grants A and not B, whatever it serves from its cache. + between, err := xchg.ExchangeActorJwt(ctx, &exchangepb.ExchangeActorJwtRequest{ActorJwt: sign.jwt(t, forActor(actor))}) + if err != nil { + t.Fatalf("exchange after the repoint: %v", err) + } + expectGCS(t, ctx, "new location write after the repoint, before any snapshot there", between.GetAccessToken(), http.MethodPut, ownPrefixAt(locationB, actor)+"/e2e-"+stamp, "b", http.StatusForbidden) + expectGCS(t, ctx, "old location write after the repoint", between.GetAccessToken(), http.MethodPut, ownPrefixAt(locationA, actor)+"/e2e-"+stamp, "a", http.StatusOK) + expectGCS(t, ctx, "old location delete after the repoint", between.GetAccessToken(), http.MethodDelete, ownPrefixAt(locationA, actor)+"/e2e-"+stamp, "", http.StatusNoContent) + + // The real path: the suspend records a pending snapshot under B, atelet + // asks the node plugin to upload there, GCS refuses the token the node + // cached for the resume, and one forced re-exchange, now granting B, + // carries the upload. The snapshot lands under B's location. + actor = suspendSync(t, ctx, clients, atespace, name) + snapshot := durableURI(t, actor, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR) + if !strings.HasPrefix(snapshot, ownPrefixAt(locationB, actor)) { + t.Fatalf("snapshot after the repoint is at %s, want it under %s", snapshot, ownPrefixAt(locationB, actor)) + } + + // A suspended actor is on no node, so the exchange answers for it only + // once it runs again. Now the status names B, so an exchange without + // force must not serve the old token, whose grants no longer cover the + // actor. + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{Actor: ref}); err != nil { + t.Fatalf("ResumeActor %s after the suspend: %v", name, err) + } + actor = waitForActorState(t, ctx, clients, atespace, name, ateapipb.ActorState_ACTOR_STATE_RUNNING) + after, err := xchg.ExchangeActorJwt(ctx, &exchangepb.ExchangeActorJwtRequest{ActorJwt: sign.jwt(t, forActor(actor))}) + if err != nil { + t.Fatalf("exchange after the suspend: %v", err) + } + if after.GetAccessToken() == first.GetAccessToken() { + t.Error("the exchange served the old token after the suspend, its grants no longer cover the actor") + } + expectGCS(t, ctx, "snapshot read with the new token", after.GetAccessToken(), http.MethodGet, snapshot+"/manifest.json", "", http.StatusOK) + expectGCS(t, ctx, "snapshot read with the old token", first.GetAccessToken(), http.MethodGet, snapshot+"/manifest.json", "", http.StatusForbidden) + + // The suspend pruned the actor's generation at A, so A is no longer + // granted. Only a forced exchange shows that: without force the exchange + // may serve any cached token that covers B, and the token minted for the + // upload, while both generations were in status, covers A as well. + fresh, err := xchg.ExchangeActorJwt(ctx, &exchangepb.ExchangeActorJwtRequest{ActorJwt: sign.jwt(t, forActor(actor)), Force: true}) + if err != nil { + t.Fatalf("forced exchange after the suspend: %v", err) + } + expectGCS(t, ctx, "old location write after the suspend", fresh.GetAccessToken(), http.MethodPut, ownPrefixAt(locationA, actor)+"/e2e-"+stamp, "a", http.StatusForbidden) +} + +func withAudience(c claims, audience string) claims { c.audience = audience; return c } +func withUID(c claims, uid string) claims { c.uid = uid; return c } +func withNode(c claims, node string) claims { c.node = node; return c } diff --git a/internal/e2e/suites/snapshotcredentials/testmain_test.go b/internal/e2e/suites/snapshotcredentials/testmain_test.go new file mode 100644 index 0000000000..667a2f3d65 --- /dev/null +++ b/internal/e2e/suites/snapshotcredentials/testmain_test.go @@ -0,0 +1,24 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package snapshotcredentials + +import ( + "os" + "testing" + + "go.ate.dev/substrate/internal/e2e" +) + +func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } diff --git a/internal/objectstoreplugin/credential/credential.go b/internal/objectstoreplugin/credential/credential.go index 78022dbe62..3b0686047e 100644 --- a/internal/objectstoreplugin/credential/credential.go +++ b/internal/objectstoreplugin/credential/credential.go @@ -24,3 +24,9 @@ type Token struct { AccessToken string Expiry time.Time } + +// MinServeLifetime is the least remaining lifetime a cached Token is served +// with, on both sides of the exchange, so a transfer never starts on a token +// about to expire. There is no mid-transfer refresh. A token with less left is +// still returned to the caller that asked for it, but not cached. +const MinServeLifetime = 20 * time.Minute diff --git a/internal/objectstoreplugin/credentials.go b/internal/objectstoreplugin/credentials.go new file mode 100644 index 0000000000..a3f3e6cf47 --- /dev/null +++ b/internal/objectstoreplugin/credentials.go @@ -0,0 +1,143 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstoreplugin + +import ( + "context" + "net/http" + "sync" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "go.ate.dev/substrate/pkg/objectstorage" +) + +// Credentials supplies the storage client a snapshot transfer runs with. The +// node plugin asks once per transfer with the actor JWT the request carries, +// and again with force set after storage refused a request, which means the +// credential behind the client no longer covers the actor. The GCS source is +// exchangeclient, which trades the JWT for a token scoped to the actor. The +// S3 source is Ambient, which ignores the JWT. +type Credentials interface { + // Client returns the client for the actor actorJWT names. + Client(ctx context.Context, actorJWT string, force bool) (objectstorage.ObjectStorage, error) +} + +// Ambient returns Credentials that hand out client for every transfer, the +// pod's own credential, whatever JWT the request carries. +func Ambient(client objectstorage.ObjectStorage) Credentials { + return ambient{client: client} +} + +type ambient struct{ client objectstorage.ObjectStorage } + +func (a ambient) Client(context.Context, string, bool) (objectstorage.ObjectStorage, error) { + return a.client, nil +} + +// refused reports whether storage turned a request down for lack of +// authorization. With a per-actor credential that means the credential is +// stale: the actor's prefixes changed, or the token expired mid-transfer. +func refused(err error) bool { + desc, ok := objectstorage.DescribeServiceError(err) + return ok && (desc.HTTPStatus == http.StatusUnauthorized || desc.HTTPStatus == http.StatusForbidden) +} + +// transfer is one FetchSnapshot or UploadSnapshot: the client its object +// requests run with, obtained on first use, and replaced when a request is +// refused. Object requests run concurrently, so a replacement one of them +// forces serves the others. +type transfer struct { + creds Credentials + actorJWT string + budget *objectstorage.Uploader + + mu sync.Mutex + client objectstorage.ObjectStorage + gen int +} + +func (p *NodePlugin) begin(actorJWT string) *transfer { + return &transfer{creds: p.creds, actorJWT: actorJWT, budget: p.uploader} +} + +// obtainLocked replaces the transfer's client with a new one from creds. A +// NotFound or Aborted from the source means the actor is not this node's to +// transfer for, gone, recreated or elsewhere. It is reported as +// FailedPrecondition, since atelet reads NotFound from a transfer as a +// missing snapshot object. +func (t *transfer) obtainLocked(ctx context.Context, force bool) error { + client, err := t.creds.Client(ctx, t.actorJWT, force) + switch status.Code(err) { + case codes.NotFound, codes.Aborted: + return status.Errorf(codes.FailedPrecondition, "the actor cannot be transferred from this node: %v", err) + } + if err != nil { + return err + } + t.client, t.gen = client, t.gen+1 + return nil +} + +// current returns the transfer's client, obtaining it on first use, and the +// generation it belongs to. +func (t *transfer) current(ctx context.Context) (objectstorage.ObjectStorage, int, error) { + t.mu.Lock() + defer t.mu.Unlock() + if t.client == nil { + if err := t.obtainLocked(ctx, false); err != nil { + return nil, 0, err + } + } + return t.client, t.gen, nil +} + +// replace returns a client newer than generation gen: the current one if +// another request already replaced it, otherwise a fresh one forced from +// creds. +func (t *transfer) replace(ctx context.Context, gen int) (objectstorage.ObjectStorage, error) { + t.mu.Lock() + defer t.mu.Unlock() + if t.gen == gen { + if err := t.obtainLocked(ctx, true); err != nil { + return nil, err + } + } + return t.client, nil +} + +// do runs one object request. A request storage refuses is retried once with +// a replaced client. A second refusal is final and means the actor's +// credential does not cover the request, which the caller sees as +// PermissionDenied. +func (t *transfer) do(ctx context.Context, op func(client objectstorage.ObjectStorage, uploader *objectstorage.Uploader) error) error { + client, gen, err := t.current(ctx) + if err != nil { + return err + } + if err := op(client, t.budget.WithClient(client)); !refused(err) { + return err + } + client, err = t.replace(ctx, gen) + if err != nil { + return err + } + if err := op(client, t.budget.WithClient(client)); refused(err) { + return status.Errorf(codes.PermissionDenied, "storage refused the request with the actor's credential: %v", err) + } else { + return err + } +} diff --git a/internal/objectstoreplugin/credentials_test.go b/internal/objectstoreplugin/credentials_test.go new file mode 100644 index 0000000000..4adbd35b16 --- /dev/null +++ b/internal/objectstoreplugin/credentials_test.go @@ -0,0 +1,256 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstoreplugin + +import ( + "context" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + + "google.golang.org/api/googleapi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "go.ate.dev/substrate/pkg/objectstorage" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" +) + +// refusingObjects wraps a backend and answers 403 to every object request +// until allowed is set, the way GCS treats a token that no longer covers the +// actor's prefix. +type refusingObjects struct { + *memObjects + allowed atomic.Bool + refusals atomic.Int32 +} + +func (r *refusingObjects) refuse() error { + r.refusals.Add(1) + return &googleapi.Error{Code: http.StatusForbidden, Message: "forbidden"} +} + +func (r *refusingObjects) GetObject(ctx context.Context, bucket, object string) (io.ReadCloser, error) { + if !r.allowed.Load() { + return nil, r.refuse() + } + return r.memObjects.GetObject(ctx, bucket, object) +} + +func (r *refusingObjects) PutObject(ctx context.Context, bucket, object string, body io.Reader) error { + if !r.allowed.Load() { + // GCS reads the body before it answers, so a refused upload leaves + // the caller's file at its end. + _, _ = io.Copy(io.Discard, body) + return r.refuse() + } + return r.memObjects.PutObject(ctx, bucket, object, body) +} + +// fakeCredentials hands out one client per call and records the calls. The +// forced client is allowed through, the first is not, unless stale is false. +type fakeCredentials struct { + backend *memObjects + stale bool + err error + + mu sync.Mutex + forces []bool +} + +func (f *fakeCredentials) Client(_ context.Context, _ string, force bool) (objectstorage.ObjectStorage, error) { + f.mu.Lock() + f.forces = append(f.forces, force) + f.mu.Unlock() + if f.err != nil { + return nil, f.err + } + client := &refusingObjects{memObjects: f.backend} + client.allowed.Store(force || !f.stale) + return client, nil +} + +func (f *fakeCredentials) calls() []bool { + f.mu.Lock() + defer f.mu.Unlock() + return append([]bool(nil), f.forces...) +} + +func newCredsPlugin(t *testing.T, creds *fakeCredentials, root string) *NodePlugin { + t.Helper() + p, err := NewNodePlugin(creds.backend, root, objectstorage.DefaultUploadMemory, WithCredentials(creds)) + if err != nil { + t.Fatal(err) + } + return p +} + +func writeFiles(t *testing.T, dir string, names ...string) { + t.Helper() + for _, name := range names { + if err := os.WriteFile(filepath.Join(dir, name), []byte("payload "+name), 0o600); err != nil { + t.Fatal(err) + } + } +} + +func TestTransferRetriesOnceWithAReplacedCredential(t *testing.T) { + root := t.TempDir() + src := filepath.Join(root, "src") + if err := os.Mkdir(src, 0o700); err != nil { + t.Fatal(err) + } + writeFiles(t, src, "a.img", "b.img", "c.img", manifestFile) + creds := &fakeCredentials{backend: newMemObjects(), stale: true} + p := newCredsPlugin(t, creds, root) + + _, err := p.UploadSnapshot(t.Context(), &objectstorev1.UploadSnapshotRequest{ + SnapshotUri: testURI, ActorJwt: "jwt", LocalPath: src, Files: []string{"a.img", "b.img", "c.img", manifestFile}, + }) + if err != nil { + t.Fatalf("UploadSnapshot with a stale credential: %v", err) + } + // Three files refused on the first client force one replacement between + // them, not one each, and the manifest then rides the replaced client. + forces := creds.calls() + if want := []bool{false, true}; len(forces) != 2 || forces[0] || !forces[1] { + t.Errorf("credential calls = %v, want %v", forces, want) + } + if got := len(creds.backend.keys()); got != 4 { + t.Errorf("stored %d objects, want 4", got) + } + + dst := filepath.Join(root, "dst") + if err := os.Mkdir(dst, 0o700); err != nil { + t.Fatal(err) + } + creds2 := &fakeCredentials{backend: creds.backend, stale: true} + p2 := newCredsPlugin(t, creds2, root) + if _, err := p2.FetchSnapshot(t.Context(), &objectstorev1.FetchSnapshotRequest{ + SnapshotUri: testURI, ActorJwt: "jwt", WritePath: dst, Files: []string{"a.img", manifestFile}, + }); err != nil { + t.Fatalf("FetchSnapshot with a stale credential: %v", err) + } + if forces := creds2.calls(); len(forces) != 2 || !forces[1] { + t.Errorf("fetch credential calls = %v, want one plain and one forced", forces) + } +} + +func TestTransferRetriesTheManifestFromItsStart(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, manifestFile) + creds := &fakeCredentials{backend: newMemObjects(), stale: true} + p := newCredsPlugin(t, creds, root) + if _, err := p.UploadSnapshot(t.Context(), &objectstorev1.UploadSnapshotRequest{ + SnapshotUri: testURI, ActorJwt: "jwt", LocalPath: root, Files: []string{manifestFile}, + }); err != nil { + t.Fatalf("UploadSnapshot of the manifest with a stale credential: %v", err) + } + got, err := io.ReadAll(mustGet(t, creds.backend, testBucket, testPrefix+"/"+manifestFile)) + if err != nil { + t.Fatal(err) + } + if want := "payload " + manifestFile; string(got) != want { + t.Fatalf("stored manifest = %q, want %q: the retry sent what was left of the file", got, want) + } +} + +func mustGet(t *testing.T, store *memObjects, bucket, object string) io.ReadCloser { + t.Helper() + r, err := store.GetObject(t.Context(), bucket, object) + if err != nil { + t.Fatalf("GetObject %s/%s: %v", bucket, object, err) + } + return r +} + +func TestTransferReportsAnActorItCannotTransferFor(t *testing.T) { + root := t.TempDir() + for _, code := range []codes.Code{codes.NotFound, codes.Aborted} { + creds := &fakeCredentials{backend: newMemObjects(), err: status.Error(code, "actor not found")} + p := newCredsPlugin(t, creds, root) + _, err := p.FetchSnapshot(t.Context(), &objectstorev1.FetchSnapshotRequest{ + SnapshotUri: testURI, ActorJwt: "jwt", WritePath: root, Files: []string{manifestFile}, + }) + if status.Code(err) != codes.FailedPrecondition { + t.Errorf("credential source %v: FetchSnapshot = %v, want FailedPrecondition, NotFound is a missing object", code, err) + } + } +} + +func TestTransferFailsClosedOnASecondRefusal(t *testing.T) { + root := t.TempDir() + creds := &fakeCredentials{backend: newMemObjects(), stale: true} + p := newCredsPlugin(t, creds, root) + // A forced client that still refuses: every client this source hands out + // refuses, which is what a request outside the actor's prefixes sees. + refuseAll := &fakeCredentials{backend: creds.backend, stale: true} + p.creds = credentialsFunc(func(ctx context.Context, jwt string, force bool) (objectstorage.ObjectStorage, error) { + client, err := refuseAll.Client(ctx, jwt, force) + if err == nil { + client.(*refusingObjects).allowed.Store(false) + } + return client, err + }) + _, err := p.FetchSnapshot(t.Context(), &objectstorev1.FetchSnapshotRequest{ + SnapshotUri: testURI, ActorJwt: "jwt", WritePath: root, Files: []string{manifestFile}, + }) + if status.Code(err) != codes.PermissionDenied { + t.Fatalf("FetchSnapshot refused twice = %v, want PermissionDenied", err) + } + if forces := refuseAll.calls(); len(forces) != 2 { + t.Errorf("credential calls = %v, want two", forces) + } +} + +type credentialsFunc func(context.Context, string, bool) (objectstorage.ObjectStorage, error) + +func (f credentialsFunc) Client(ctx context.Context, jwt string, force bool) (objectstorage.ObjectStorage, error) { + return f(ctx, jwt, force) +} + +func TestTransferReturnsTheCredentialSourcesStatus(t *testing.T) { + root := t.TempDir() + creds := &fakeCredentials{backend: newMemObjects(), err: status.Error(codes.PermissionDenied, "actor is no longer on the token's node")} + p := newCredsPlugin(t, creds, root) + _, err := p.FetchSnapshot(t.Context(), &objectstorev1.FetchSnapshotRequest{ + SnapshotUri: testURI, ActorJwt: "jwt", WritePath: root, Files: []string{manifestFile}, + }) + if status.Code(err) != codes.PermissionDenied || !strings.Contains(status.Convert(err).Message(), "actor is no longer on the token's node") { + t.Fatalf("err = %v, want the source's code and message", err) + } +} + +func TestAmbientCredentialsIgnoreTheJWT(t *testing.T) { + root := t.TempDir() + backend := newMemObjects() + p, err := NewNodePlugin(backend, root, objectstorage.DefaultUploadMemory) + if err != nil { + t.Fatal(err) + } + writeFiles(t, root, manifestFile) + for _, jwt := range []string{"", "anything"} { + if _, err := p.UploadSnapshot(t.Context(), &objectstorev1.UploadSnapshotRequest{ + SnapshotUri: testURI, ActorJwt: jwt, LocalPath: root, Files: []string{manifestFile}, + }); err != nil { + t.Fatalf("UploadSnapshot(actor_jwt=%q) with ambient credentials: %v", jwt, err) + } + } +} diff --git a/internal/objectstoreplugin/exchange/exchange.go b/internal/objectstoreplugin/exchange/exchange.go index d3b3e5cda2..dc6b1c60f0 100644 --- a/internal/objectstoreplugin/exchange/exchange.go +++ b/internal/objectstoreplugin/exchange/exchange.go @@ -42,6 +42,10 @@ import ( "go.ate.dev/substrate/pkg/proto/ateapipb" ) +// DefaultPort is the port the control sidecar serves the exchange on and the +// node plugin dials by default, through the ate-api-server Service. +const DefaultPort = 8445 + // ActorJWTAudience is the audience of the objectstore plugin API. atelet // requests it on the worker-for-actor JWT it hands to whichever node plugin is // installed, and every exchange serving this API requires it. A token minted diff --git a/internal/objectstoreplugin/exchangeclient/exchangeclient.go b/internal/objectstoreplugin/exchangeclient/exchangeclient.go new file mode 100644 index 0000000000..40496993e7 --- /dev/null +++ b/internal/objectstoreplugin/exchangeclient/exchangeclient.go @@ -0,0 +1,167 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package exchangeclient is the GCS node plugin's side of the credential +// exchange. For each transfer it trades the actor JWT atelet handed it for a +// GCS access token scoped to that actor's snapshot prefixes, caches the token +// and the storage client built with it under that JWT, and hands the client +// to the transfer. It is this binary's: a provider's own plugin brings its +// own. +package exchangeclient + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "time" + + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "golang.org/x/oauth2" + "google.golang.org/api/option" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" + "k8s.io/utils/lru" + + "go.ate.dev/substrate/internal/credbundle" + "go.ate.dev/substrate/internal/objectstoreplugin/credential" + exchangepb "go.ate.dev/substrate/internal/proto/objectstoreexchangepb" + "go.ate.dev/substrate/pkg/objectstorage" +) + +// defaultCacheSize bounds the cache, one entry per JWT atelet holds, which +// is one per actor the node transferred for within a JWT lifetime. Eviction +// only costs the next transfer an exchange and a new client. +const defaultCacheSize = 1_000 + +// Config is how the exchange is reached. +type Config struct { + // Address is the exchange's host and port, the ate-api-server Service. + Address string + // CAFile verifies the exchange's serving certificate. + CAFile string + // ServerName is the DNS name expected on that certificate. + ServerName string +} + +// Source implements objectstoreplugin.Credentials for GCS. +type Source struct { + exchange exchangepb.CredentialExchangeClient + conn io.Closer + // newClient builds the storage client a token is used through. + newClient func(ctx context.Context, tok credential.Token) (objectstorage.ObjectStorage, error) + now func() time.Time + // cache maps an actor JWT to its entry: the token the exchange returned + // for that JWT and the storage client built with the token. Nothing + // closes a client that leaves the cache. A transfer still using it + // finishes on it, and its connections close on their own once idle. + cache *lru.Cache +} + +// entry is a token the exchange returned for one JWT and the storage client +// built with it. +type entry struct { + token credential.Token + client objectstorage.ObjectStorage +} + +// New dials the exchange over TLS. Nothing is sent until the first transfer. +func New(cfg Config) (*Source, error) { + if cfg.Address == "" { + return nil, errors.New("exchangeclient: Address is required") + } + pool, err := credbundle.ParsePool(cfg.CAFile) + if err != nil { + return nil, fmt.Errorf("exchangeclient: while loading the exchange CA bundle: %w", err) + } + creds := credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS13, RootCAs: pool, ServerName: cfg.ServerName}) + conn, err := grpc.NewClient(cfg.Address, grpc.WithTransportCredentials(creds), grpc.WithStatsHandler(otelgrpc.NewClientHandler())) + if err != nil { + return nil, fmt.Errorf("exchangeclient: while creating the exchange client: %w", err) + } + s := newSource(exchangepb.NewCredentialExchangeClient(conn), newGCSClient) + s.conn = conn + return s, nil +} + +func newSource(exchange exchangepb.CredentialExchangeClient, newClient func(context.Context, credential.Token) (objectstorage.ObjectStorage, error)) *Source { + return &Source{exchange: exchange, newClient: newClient, now: time.Now, cache: lru.New(defaultCacheSize)} +} + +// newGCSClient builds a client that authenticates with tok alone. It outlives +// the request that built it, so it must not carry that request's +// cancellation. +func newGCSClient(ctx context.Context, tok credential.Token) (objectstorage.ObjectStorage, error) { + source := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: tok.AccessToken, Expiry: tok.Expiry}) + return objectstorage.NewGCSClient(context.WithoutCancel(ctx), option.WithTokenSource(source)) +} + +// Close closes the connection to the exchange. +func (s *Source) Close() error { + if s.conn == nil { + return nil + } + return s.conn.Close() +} + +// Client returns a storage client scoped to the actor actorJWT names: the +// cached one for that JWT while its token has at least +// credential.MinServeLifetime left, otherwise one built on a fresh exchange, +// which replaces the cached one. force skips the cache, after GCS refused a +// request made with the cached token, and asks the exchange to skip its own. +// A refused or nearly expired entry is dropped before the exchange, so it is +// not served again if the exchange fails or returns a token too short to +// keep. An exchange error reaches the caller unchanged. +func (s *Source) Client(ctx context.Context, actorJWT string, force bool) (objectstorage.ObjectStorage, error) { + if actorJWT == "" { + return nil, status.Error(codes.InvalidArgument, "actor_jwt is required on GCS") + } + if !force { + if v, ok := s.cache.Get(actorJWT); ok { + if e := v.(*entry); e.token.Expiry.Sub(s.now()) >= credential.MinServeLifetime { + return e.client, nil + } + } + } + s.cache.Remove(actorJWT) + resp, err := s.exchange.ExchangeActorJwt(ctx, &exchangepb.ExchangeActorJwtRequest{ActorJwt: actorJWT, Force: force}) + if err != nil { + return nil, err + } + tok := credential.Token{AccessToken: resp.GetAccessToken(), Expiry: expiry(resp.GetExpireTime())} + if tok.AccessToken == "" { + return nil, status.Error(codes.Internal, "the exchange returned no access token") + } + client, err := s.newClient(ctx, tok) + if err != nil { + return nil, status.Errorf(codes.Internal, "while building the actor's storage client: %v", err) + } + // A token minted inside the source credential's end-of-hour window is + // shorter than the floor. It serves this transfer and is not kept. + if tok.Expiry.Sub(s.now()) >= credential.MinServeLifetime { + s.cache.Add(actorJWT, &entry{token: tok, client: client}) + } + return client, nil +} + +func expiry(ts *timestamppb.Timestamp) time.Time { + if ts == nil { + return time.Time{} + } + return ts.AsTime() +} diff --git a/internal/objectstoreplugin/exchangeclient/exchangeclient_test.go b/internal/objectstoreplugin/exchangeclient/exchangeclient_test.go new file mode 100644 index 0000000000..e882bdbfdb --- /dev/null +++ b/internal/objectstoreplugin/exchangeclient/exchangeclient_test.go @@ -0,0 +1,254 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package exchangeclient + +import ( + "context" + "errors" + "io" + "path/filepath" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" + "k8s.io/utils/lru" + + "go.ate.dev/substrate/internal/objectstoreplugin/credential" + exchangepb "go.ate.dev/substrate/internal/proto/objectstoreexchangepb" + "go.ate.dev/substrate/pkg/objectstorage" +) + +// fakeExchange answers ExchangeActorJwt with one token and records the calls. +type fakeExchange struct { + exchangepb.CredentialExchangeClient + token string + expiry time.Time + err error + calls []*exchangepb.ExchangeActorJwtRequest +} + +func (f *fakeExchange) ExchangeActorJwt(_ context.Context, in *exchangepb.ExchangeActorJwtRequest, _ ...grpc.CallOption) (*exchangepb.ExchangeActorJwtResponse, error) { + f.calls = append(f.calls, in) + if f.err != nil { + return nil, f.err + } + return &exchangepb.ExchangeActorJwtResponse{AccessToken: f.token, ExpireTime: timestamppb.New(f.expiry)}, nil +} + +// fakeClient is the storage client built for a token. +type fakeClient struct { + objectstorage.ObjectStorage + token string +} + +func newTestSource(exchange *fakeExchange, now time.Time) (*Source, *int) { + built := 0 + s := newSource(exchange, func(_ context.Context, tok credential.Token) (objectstorage.ObjectStorage, error) { + built++ + return &fakeClient{token: tok.AccessToken}, nil + }) + s.now = func() time.Time { return now } + return s, &built +} + +func tokenOf(client objectstorage.ObjectStorage) string { return client.(*fakeClient).token } + +func TestClient_CachesTheTokenAndClientPerJWTUntilNearExpiry(t *testing.T) { + now := time.Date(2026, 10, 20, 10, 0, 0, 0, time.UTC) + exchange := &fakeExchange{token: "tok-1", expiry: now.Add(time.Hour)} + s, built := newTestSource(exchange, now) + + first, err := s.Client(t.Context(), "jwt-a", false) + if err != nil { + t.Fatal(err) + } + again, err := s.Client(t.Context(), "jwt-a", false) + if err != nil { + t.Fatal(err) + } + if again != first || tokenOf(first) != "tok-1" { + t.Fatalf("second transfer got %v, want the same client on tok-1", again) + } + if len(exchange.calls) != 1 || exchange.calls[0].GetForce() || exchange.calls[0].GetActorJwt() != "jwt-a" { + t.Fatalf("exchange calls = %d (%v), want one unforced with the JWT", len(exchange.calls), exchange.calls) + } + if *built != 1 { + t.Fatalf("built %d clients, want 1 shared by both transfers", *built) + } + + // Another JWT is a miss, even one naming the same actor: the key is the + // JWT the exchange verified, never a claim read out of it. + if _, err := s.Client(t.Context(), "jwt-b-same-actor", false); err != nil { + t.Fatal(err) + } + if len(exchange.calls) != 2 { + t.Fatalf("a JWT the exchange has not seen must exchange, calls = %d", len(exchange.calls)) + } + + // Under the floor the cache is not served and the entry is replaced. + s.now = func() time.Time { return now.Add(time.Hour - credential.MinServeLifetime + time.Second) } + exchange.token, exchange.expiry = "tok-2", now.Add(2*time.Hour) + client, err := s.Client(t.Context(), "jwt-a", false) + if err != nil || tokenOf(client) != "tok-2" { + t.Fatalf("near expiry client = %v, %v, want tok-2 from a fresh exchange", client, err) + } + if client, err := s.Client(t.Context(), "jwt-a", false); err != nil || tokenOf(client) != "tok-2" || len(exchange.calls) != 3 { + t.Fatalf("after the refresh: client %v, err %v, calls %d, want tok-2 from the cache", client, err, len(exchange.calls)) + } +} + +func TestClient_ForceSkipsBothCaches(t *testing.T) { + now := time.Date(2026, 10, 20, 10, 0, 0, 0, time.UTC) + exchange := &fakeExchange{token: "tok-1", expiry: now.Add(time.Hour)} + s, built := newTestSource(exchange, now) + if _, err := s.Client(t.Context(), "jwt", false); err != nil { + t.Fatal(err) + } + exchange.token = "tok-2" + client, err := s.Client(t.Context(), "jwt", true) + if err != nil || tokenOf(client) != "tok-2" { + t.Fatalf("forced client = %v, %v, want one on tok-2", client, err) + } + if len(exchange.calls) != 2 || !exchange.calls[1].GetForce() { + t.Fatalf("exchange calls = %v, want the second forced", exchange.calls) + } + // The forced token replaces the cached one. + client, err = s.Client(t.Context(), "jwt", false) + if err != nil || tokenOf(client) != "tok-2" || len(exchange.calls) != 2 { + t.Fatalf("after force: client %v, err %v, calls %d, want tok-2 from the cache", client, err, len(exchange.calls)) + } + if *built != 2 { + t.Fatalf("built %d clients, want one per token", *built) + } +} + +func TestClient_DropsTheRefusedEntryEvenWhenTheForcedExchangeFails(t *testing.T) { + now := time.Date(2026, 10, 20, 10, 0, 0, 0, time.UTC) + exchange := &fakeExchange{token: "tok-1", expiry: now.Add(time.Hour)} + s, _ := newTestSource(exchange, now) + if _, err := s.Client(t.Context(), "jwt", false); err != nil { + t.Fatal(err) + } + // GCS refused tok-1, and the actor has since left the node. + exchange.err = status.Error(codes.NotFound, "actor not found") + if _, err := s.Client(t.Context(), "jwt", true); status.Code(err) != codes.NotFound { + t.Fatalf("forced exchange error = %v, want NotFound", err) + } + // The refused token must not come back from the cache. + if _, err := s.Client(t.Context(), "jwt", false); status.Code(err) != codes.NotFound { + t.Fatalf("after a failed force: err = %v, want NotFound from a new exchange", err) + } + if len(exchange.calls) != 3 { + t.Fatalf("exchange calls = %d, want three", len(exchange.calls)) + } +} + +func TestClient_DropsTheRefusedEntryWhenTheForcedTokenIsShort(t *testing.T) { + now := time.Date(2026, 10, 20, 10, 0, 0, 0, time.UTC) + exchange := &fakeExchange{token: "tok-1", expiry: now.Add(time.Hour)} + s, _ := newTestSource(exchange, now) + if _, err := s.Client(t.Context(), "jwt", false); err != nil { + t.Fatal(err) + } + exchange.token, exchange.expiry = "short", now.Add(credential.MinServeLifetime-time.Minute) + if client, err := s.Client(t.Context(), "jwt", true); err != nil || tokenOf(client) != "short" { + t.Fatalf("forced client = %v, %v, want one on the short token", client, err) + } + // Neither the refused tok-1 nor the short token is cached. + if client, err := s.Client(t.Context(), "jwt", false); err != nil || tokenOf(client) != "short" || len(exchange.calls) != 3 { + t.Fatalf("after force: client %v, err %v, calls %d, want a third exchange", client, err, len(exchange.calls)) + } +} + +func TestClient_EvictsTheOldestJWTWhenFull(t *testing.T) { + now := time.Date(2026, 10, 20, 10, 0, 0, 0, time.UTC) + exchange := &fakeExchange{token: "tok", expiry: now.Add(time.Hour)} + s, _ := newTestSource(exchange, now) + s.cache = lru.New(1) + for _, jwt := range []string{"jwt-a", "jwt-b", "jwt-a"} { + if _, err := s.Client(t.Context(), jwt, false); err != nil { + t.Fatal(err) + } + } + if len(exchange.calls) != 3 { + t.Fatalf("exchange calls = %d, want 3: jwt-a was evicted by jwt-b and exchanged again", len(exchange.calls)) + } +} + +func TestClient_UsesButDoesNotCacheAShortToken(t *testing.T) { + now := time.Date(2026, 10, 20, 10, 0, 0, 0, time.UTC) + exchange := &fakeExchange{token: "short", expiry: now.Add(credential.MinServeLifetime - time.Minute)} + s, _ := newTestSource(exchange, now) + for range 2 { + client, err := s.Client(t.Context(), "jwt", false) + if err != nil || tokenOf(client) != "short" { + t.Fatalf("client = %v, %v", client, err) + } + } + if len(exchange.calls) != 2 { + t.Fatalf("a short token must not be cached, calls = %d", len(exchange.calls)) + } +} + +func TestClient_Errors(t *testing.T) { + now := time.Date(2026, 10, 20, 10, 0, 0, 0, time.UTC) + for _, tc := range []struct { + name string + jwt string + exchange *fakeExchange + want codes.Code + }{ + {"empty jwt", "", &fakeExchange{token: "t", expiry: now.Add(time.Hour)}, codes.InvalidArgument}, + {"exchange refuses", "jwt", &fakeExchange{err: status.Error(codes.PermissionDenied, "actor is no longer on the token's node")}, codes.PermissionDenied}, + {"exchange unavailable", "jwt", &fakeExchange{err: status.Error(codes.Unavailable, "down")}, codes.Unavailable}, + {"exchange returns no token", "jwt", &fakeExchange{token: "", expiry: now.Add(time.Hour)}, codes.Internal}, + } { + t.Run(tc.name, func(t *testing.T) { + s, _ := newTestSource(tc.exchange, now) + _, err := s.Client(t.Context(), tc.jwt, false) + if status.Code(err) != tc.want { + t.Fatalf("code = %v (%v), want %v", status.Code(err), err, tc.want) + } + if tc.jwt == "" && len(tc.exchange.calls) != 0 { + t.Errorf("the exchange was called %d times for an empty token", len(tc.exchange.calls)) + } + }) + } +} + +func TestClient_ReportsAClientBuildFailure(t *testing.T) { + now := time.Date(2026, 10, 20, 10, 0, 0, 0, time.UTC) + s := newSource(&fakeExchange{token: "t", expiry: now.Add(time.Hour)}, func(context.Context, credential.Token) (objectstorage.ObjectStorage, error) { + return nil, errors.New("no transport") + }) + _, err := s.Client(t.Context(), "jwt", false) + if status.Code(err) != codes.Internal { + t.Fatalf("code = %v (%v), want Internal", status.Code(err), err) + } +} + +func TestNew_ChecksItsConfig(t *testing.T) { + if _, err := New(Config{CAFile: "x"}); err == nil { + t.Error("New without an address succeeded") + } + if _, err := New(Config{Address: "api:8445", CAFile: filepath.Join(t.TempDir(), "absent.pem")}); err == nil { + t.Error("New with a missing CA bundle succeeded") + } +} + +var _ io.Closer = (*Source)(nil) diff --git a/internal/objectstoreplugin/gcsissuer/issuer.go b/internal/objectstoreplugin/gcsissuer/issuer.go index 22b71db891..1cdb9dfb9e 100644 --- a/internal/objectstoreplugin/gcsissuer/issuer.go +++ b/internal/objectstoreplugin/gcsissuer/issuer.go @@ -50,9 +50,8 @@ const ( cloudPlatformScope = "https://www.googleapis.com/auth/cloud-platform" // minServeLifetime is the least remaining lifetime a token is cached and - // served with, so a transfer never starts on a token about to expire. There - // is no mid-transfer refresh. - minServeLifetime = 20 * time.Minute + // served with, the same on the node plugin's side. + minServeLifetime = credential.MinServeLifetime // defaultCacheSize bounds the per-actor cache. A miss costs an exchange, // never correctness. defaultCacheSize = 100_000 diff --git a/internal/objectstoreplugin/node.go b/internal/objectstoreplugin/node.go index 712d3110e0..8533d83897 100644 --- a/internal/objectstoreplugin/node.go +++ b/internal/objectstoreplugin/node.go @@ -25,6 +25,7 @@ import ( "context" "errors" "fmt" + "io" "os" "path/filepath" "slices" @@ -46,24 +47,40 @@ const manifestFile = "manifest.json" type NodePlugin struct { objectstorev1.UnimplementedNodeProviderServer - gcsClient objectstorage.ObjectStorage - uploader *objectstorage.Uploader + // creds supplies the client each transfer runs with. + creds Credentials + uploader *objectstorage.Uploader // root confines every local path a caller names. root string } // NewNodePlugin returns a NodePlugin that reads and writes local files only // below root. All upload requests share uploadMemory bytes of reservations for -// compression and backend buffers. -func NewNodePlugin(gcsClient objectstorage.ObjectStorage, root string, uploadMemory int64) (*NodePlugin, error) { +// compression and backend buffers. client is the pod's own storage client, +// GCS or S3 as the environment selected. It sizes that budget and serves +// every transfer unless WithCredentials supplies a per-actor one. +func NewNodePlugin(client objectstorage.ObjectStorage, root string, uploadMemory int64, opts ...NodeOption) (*NodePlugin, error) { if !filepath.IsAbs(root) { return nil, fmt.Errorf("root %q is not an absolute path", root) } - uploader, err := objectstorage.NewUploader(gcsClient, uploadMemory) + uploader, err := objectstorage.NewUploader(client, uploadMemory) if err != nil { return nil, err } - return &NodePlugin{gcsClient: gcsClient, uploader: uploader, root: filepath.Clean(root)}, nil + p := &NodePlugin{creds: Ambient(client), uploader: uploader, root: filepath.Clean(root)} + for _, opt := range opts { + opt(p) + } + return p, nil +} + +// NodeOption configures a NodePlugin. +type NodeOption func(*NodePlugin) + +// WithCredentials makes transfers run with the client creds supplies for the +// actor JWT each request carries, instead of the ambient client. +func WithCredentials(creds Credentials) NodeOption { + return func(p *NodePlugin) { p.creds = creds } } // FetchSnapshot downloads the requested snapshot files into write_path. @@ -77,13 +94,14 @@ func (p *NodePlugin) FetchSnapshot(ctx context.Context, req *objectstorev1.Fetch return nil, toStatus(fmt.Errorf("while opening restore directory: %w", err)) } defer root.Close() + tr := p.begin(req.GetActorJwt()) g, gCtx := errgroup.WithContext(ctx) for _, fileName := range req.GetFiles() { fileName := fileName g.Go(func() error { if fileName == manifestFile { - return p.fetchManifest(gCtx, uri, root) + return p.fetchManifest(gCtx, tr, uri, root) } objectURI, err := uri.ObjectURI(fileName + ".zstd") if err != nil { @@ -98,7 +116,12 @@ func (p *NodePlugin) FetchSnapshot(ctx context.Context, req *objectstorev1.Fetch if err != nil { return fmt.Errorf("while opening %s in restore directory: %w", fileName, err) } - fetchErr := objectstorage.FetchFileFromGCSWithZstd(gCtx, p.gcsClient, objectURI, local) + fetchErr := tr.do(gCtx, func(client objectstorage.ObjectStorage, _ *objectstorage.Uploader) error { + if err := truncate(local); err != nil { + return err + } + return objectstorage.FetchFileFromGCSWithZstd(gCtx, client, objectURI, local) + }) closeErr := local.Close() if err := errors.Join(fetchErr, closeErr); err != nil { return fmt.Errorf("while downloading %s from GCS: %w", fileName, err) @@ -124,6 +147,7 @@ func (p *NodePlugin) UploadSnapshot(ctx context.Context, req *objectstorev1.Uplo return nil, toStatus(fmt.Errorf("while opening snapshot directory: %w", err)) } defer root.Close() + tr := p.begin(req.GetActorJwt()) upload := func(ctx context.Context, fileName string) error { local, err := root.Open(fileName) @@ -139,13 +163,19 @@ func (p *NodePlugin) UploadSnapshot(ctx context.Context, req *objectstorev1.Uplo return fmt.Errorf("snapshot file %s is not a regular file", fileName) } if fileName == manifestFile { - return p.uploadManifest(ctx, uri, local) + return p.uploadManifest(ctx, tr, uri, local) } objectURI, err := uri.ObjectURI(fileName + ".zstd") if err != nil { return fmt.Errorf("while addressing %s in GCS: %w", fileName, err) } - if err := p.uploader.SendFile(ctx, objectURI, local); err != nil { + err = tr.do(ctx, func(_ objectstorage.ObjectStorage, uploader *objectstorage.Uploader) error { + if err := rewind(local); err != nil { + return err + } + return uploader.SendFile(ctx, objectURI, local) + }) + if err != nil { return fmt.Errorf("while uploading %s to GCS: %w", fileName, err) } return nil @@ -170,12 +200,16 @@ func (p *NodePlugin) UploadSnapshot(ctx context.Context, req *objectstorev1.Uplo } // fetchManifest downloads the uncompressed manifest into root. -func (p *NodePlugin) fetchManifest(ctx context.Context, uri resources.SnapshotURI, root *os.Root) error { +func (p *NodePlugin) fetchManifest(ctx context.Context, tr *transfer, uri resources.SnapshotURI, root *os.Root) error { manifestURI, err := uri.ObjectURI(manifestFile) if err != nil { return err } - manifest, err := objectstorage.FetchFromGCS(ctx, p.gcsClient, manifestURI) + var manifest []byte + err = tr.do(ctx, func(client objectstorage.ObjectStorage, _ *objectstorage.Uploader) (err error) { + manifest, err = objectstorage.FetchFromGCS(ctx, client, manifestURI) + return err + }) if err != nil { return fmt.Errorf("while fetching snapshot manifest: %w", err) } @@ -186,17 +220,40 @@ func (p *NodePlugin) fetchManifest(ctx context.Context, uri resources.SnapshotUR } // uploadManifest uploads the manifest from local uncompressed. -func (p *NodePlugin) uploadManifest(ctx context.Context, uri resources.SnapshotURI, local *os.File) error { +func (p *NodePlugin) uploadManifest(ctx context.Context, tr *transfer, uri resources.SnapshotURI, local *os.File) error { manifestURI, err := uri.ObjectURI(manifestFile) if err != nil { return fmt.Errorf("while addressing snapshot manifest in GCS: %w", err) } - if err := p.uploader.Send(ctx, manifestURI, local); err != nil { + err = tr.do(ctx, func(_ objectstorage.ObjectStorage, uploader *objectstorage.Uploader) error { + if err := rewind(local); err != nil { + return err + } + return uploader.Send(ctx, manifestURI, local) + }) + if err != nil { return fmt.Errorf("while uploading snapshot manifest: %w", err) } return nil } +// rewind puts an upload's source back at its start. A request storage +// refuses may already have read the file, and the retry must send all of it +// rather than what is left. +func rewind(f *os.File) error { + _, err := f.Seek(0, io.SeekStart) + return err +} + +// truncate empties a download's destination, so a retry does not append to +// whatever a refused attempt wrote. +func truncate(f *os.File) error { + if err := f.Truncate(0); err != nil { + return err + } + return rewind(f) +} + // validate checks a request's snapshot URI, local directory and file names, // returning the parsed URI and the local directory relative to p.root. func (p *NodePlugin) validate(snapshotURI, dir string, files []string) (resources.SnapshotURI, string, error) { diff --git a/internal/objectstoreplugin/status.go b/internal/objectstoreplugin/status.go index 60ba1382ac..cf3178b8ea 100644 --- a/internal/objectstoreplugin/status.go +++ b/internal/objectstoreplugin/status.go @@ -29,6 +29,11 @@ import ( // toStatus maps a plugin error to a gRPC status, keeping NotFound // distinguishable so callers can probe for a snapshot. func toStatus(err error) error { + // A status from the credential source, such as the exchange's refusal, or + // a refusal this plugin already classified, reaches atelet as it is. + if st, ok := status.FromError(err); ok { + return st.Err() + } switch { case errors.Is(err, objectstorage.ErrObjectNotFound): return status.Error(codes.NotFound, err.Error()) diff --git a/manifests/ate-install/atelet.yaml b/manifests/ate-install/atelet.yaml index 4350367758..f41d6d8f8f 100644 --- a/manifests/ate-install/atelet.yaml +++ b/manifests/ate-install/atelet.yaml @@ -202,6 +202,15 @@ spec: # on it. - name: ATE_STORAGE_BACKEND value: "gcs" + # On GCS each transfer runs on a token scoped to its actor, from the + # credential exchange the control sidecar serves on the ate-api-server + # Service. The plugin derives the exchange's address and certificate + # name from the pod's namespace, and verifies the certificate with the + # service DNS trust bundle mounted below. S3 never calls it. + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace envFrom: - configMapRef: name: ate-otel-config @@ -210,6 +219,9 @@ spec: mountPath: /var/lib/ate - name: objectstore-plugin mountPath: /run/objectstore-plugin + - name: servicedns-ca + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true # Sandbox assets are downloaded here and verified against their sha256 # before the plugin copies them below /var/lib/ate. Only this container # mounts it, so atelet never sees bytes it did not pin. diff --git a/pkg/objectstorage/transferclient_test.go b/pkg/objectstorage/transferclient_test.go new file mode 100644 index 0000000000..b2ffe2fc5e --- /dev/null +++ b/pkg/objectstorage/transferclient_test.go @@ -0,0 +1,33 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstorage + +import "testing" + +func TestUploaderWithClientSharesTheBudget(t *testing.T) { + store := &gcsClient{} + u, err := NewUploader(store, DefaultUploadMemory) + if err != nil { + t.Fatal(err) + } + other := &gcsClient{} + w := u.WithClient(other) + if w.client != other { + t.Error("WithClient did not switch the client") + } + if w.budget != u.budget { + t.Error("WithClient did not share the budget") + } +} diff --git a/pkg/objectstorage/upload.go b/pkg/objectstorage/upload.go index 247d1e1572..14f4ddee21 100644 --- a/pkg/objectstorage/upload.go +++ b/pkg/objectstorage/upload.go @@ -76,6 +76,13 @@ func NewUploader(client ObjectStorage, capacity int64) (*Uploader, error) { } // uploadBufferMemory includes buffers retained while the compressor is running. +// WithClient returns an Uploader that sends through client and shares u's +// memory budget, for a transfer that runs with its own credentials. client +// must be the same kind of backend as u's, since the budget was sized for it. +func (u *Uploader) WithClient(client ObjectStorage) *Uploader { + return &Uploader{client: client, budget: u.budget} +} + func uploadBufferMemory(client ObjectStorage) int64 { switch client.(type) { case *s3Client: diff --git a/pkg/proto/objectstorepb/v1/objectstore.pb.go b/pkg/proto/objectstorepb/v1/objectstore.pb.go index aa34678e55..e422197c89 100644 --- a/pkg/proto/objectstorepb/v1/objectstore.pb.go +++ b/pkg/proto/objectstorepb/v1/objectstore.pb.go @@ -45,8 +45,8 @@ type FetchSnapshotRequest struct { // minted for the node hosting the actor, with the audience // "objectstore-plugin.ate.dev". That audience belongs to this API, so every // plugin serving it verifies the same value. The token names the node it - // was minted for and is refused once the actor has left it. Substrate does - // not set the field yet, so it is currently always empty. + // was minted for and is refused once the actor has left it. atelet sets it + // on every transfer. A plugin without per-actor authorization ignores it. ActorJwt string `protobuf:"bytes,2,opt,name=actor_jwt,json=actorJwt,proto3" json:"actor_jwt,omitempty"` // Full local path of the directory the files are written into. It must // exist. @@ -161,8 +161,8 @@ type UploadSnapshotRequest struct { // minted for the node hosting the actor, with the audience // "objectstore-plugin.ate.dev". That audience belongs to this API, so every // plugin serving it verifies the same value. The token names the node it - // was minted for and is refused once the actor has left it. Substrate does - // not set the field yet, so it is currently always empty. + // was minted for and is refused once the actor has left it. atelet sets it + // on every transfer. A plugin without per-actor authorization ignores it. ActorJwt string `protobuf:"bytes,2,opt,name=actor_jwt,json=actorJwt,proto3" json:"actor_jwt,omitempty"` // Full local path of the directory the files are read from. LocalPath string `protobuf:"bytes,3,opt,name=local_path,json=localPath,proto3" json:"local_path,omitempty"` diff --git a/pkg/proto/objectstorepb/v1/objectstore.proto b/pkg/proto/objectstorepb/v1/objectstore.proto index e35fb9fae1..698eab75f3 100644 --- a/pkg/proto/objectstorepb/v1/objectstore.proto +++ b/pkg/proto/objectstorepb/v1/objectstore.proto @@ -58,8 +58,8 @@ message FetchSnapshotRequest { // minted for the node hosting the actor, with the audience // "objectstore-plugin.ate.dev". That audience belongs to this API, so every // plugin serving it verifies the same value. The token names the node it - // was minted for and is refused once the actor has left it. Substrate does - // not set the field yet, so it is currently always empty. + // was minted for and is refused once the actor has left it. atelet sets it + // on every transfer. A plugin without per-actor authorization ignores it. string actor_jwt = 2 [ debug_redact = true ]; // Full local path of the directory the files are written into. It must @@ -82,8 +82,8 @@ message UploadSnapshotRequest { // minted for the node hosting the actor, with the audience // "objectstore-plugin.ate.dev". That audience belongs to this API, so every // plugin serving it verifies the same value. The token names the node it - // was minted for and is refused once the actor has left it. Substrate does - // not set the field yet, so it is currently always empty. + // was minted for and is refused once the actor has left it. atelet sets it + // on every transfer. A plugin without per-actor authorization ignores it. string actor_jwt = 2 [ debug_redact = true ]; // Full local path of the directory the files are read from. From 155f574d4a9ac8624f18aea7ac800234bce9f5f7 Mon Sep 17 00:00:00 2001 From: Shruti Nair Date: Tue, 20 Oct 2026 21:11:47 +0000 Subject: [PATCH 17/63] ateapi: always enforce authorization (#2942) Fixes: https://github.com/agent-substrate/substrate/issues/2659 ate-setup already enforces authorization by default, and there's no compatibility promise before v1.0, so the off switch isn't worth keeping. This removes `--experimental-enable-authz` from ate-api-server and `ateapi.authz.enabled` (`ATE_API_EXPERIMENTAL_ENABLE_AUTHZ`) from ate-setup. AccessPolicy RPCs keep the same checks. #### Breaking change ate-api-server won't start without `--authz-bootstrap-owners`. ate-setup always sets it, so only installs that apply `manifests/ate-install` directly need to add it. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- AGENTS.md | 2 +- cmd/ate-setup/internal/config/config.go | 3 - cmd/ate-setup/internal/config/config_test.go | 40 +++--- cmd/ate-setup/internal/config/load.go | 16 +-- cmd/ate-setup/internal/config/setting.go | 7 +- cmd/ate-setup/internal/steps/apienv.go | 34 ++--- cmd/ate-setup/internal/steps/apienv_test.go | 70 ++++------ cmd/ateapi/internal/authz/interceptor.go | 10 +- cmd/ateapi/internal/authz/interceptor_test.go | 124 ++---------------- cmd/ateapi/internal/authz/registry.go | 26 ++-- cmd/ateapi/internal/authz/registry_test.go | 3 - .../internal/controlapi/access_policy_test.go | 4 +- .../internal/controlapi/atespace_test.go | 2 +- cmd/ateapi/main.go | 15 +-- docs/api-guide.md | 2 +- docs/architecture.md | 3 +- docs/authorization.md | 25 +--- docs/operator-install.md | 1 - 18 files changed, 103 insertions(+), 284 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1784a630ed..850c5accf6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -154,5 +154,5 @@ Agent Substrate is built for zero-trust, multi-tenant execution of untrusted act - **Workload & Storage Isolation**: Actors run strictly inside `gVisor` (`cmd/ateworker-gvisor`) or `Cloud Hypervisor` microVM (`cmd/ateworker-microvm`) sandboxes; plain Linux containers are not supported. `atelet` scrubs actor directories across lifecycle transitions (`resetActorDirs`), and snapshots are stamped with `SandboxRuntime` metadata and transferred via `cmd/objectstore-plugin`. - **Component & Actor Identity (mTLS, JWTs, and JWKs)**: Inter-component communication uses mTLS backed by `cmd/podcertcontroller` (`podid`, `servicedns`, and `postgres` signers). Actor JWT signing (`internal/localjwtauthority`), verification (`internal/oidcjwt`), and key discovery (served via `cmd/ate-idp-server`) are built on the standard library's crypto packages. Extend those packages rather than adding `go-jose` or another JOSE library, so the code that decides whether a token is valid stays small and supports only the algorithms Substrate uses. -- **Control-Plane Authentication & Authorization**: `ate-api-server` checks every Control RPC against an embedded OpenFGA model (`cmd/ateapi/internal/authz/model.fga`), on by default. Every served unary RPC needs an entry in `defaultRPCPermissions` (`registry.go`): with enforcement on, an RPC without one is denied, and a test fails the build. Access is granted per global scope and atespace through AccessPolicy (`kubectl ate access-policy`); the install's bootstrap owner is the shared `ate-client` ServiceAccount that `kubectl ate` uses. See `docs/authorization.md` for roles and known gaps. Callers are authenticated via `SPIFFE` mTLS or configured OIDC `JWTProvider`s (`cmd/ateapi/internal/apiauthn`). +- **Control-Plane Authentication & Authorization**: `ate-api-server` checks every Control RPC against an embedded OpenFGA model (`cmd/ateapi/internal/authz/model.fga`), always enforced. Every served unary RPC needs an entry in `defaultRPCPermissions` (`registry.go`): an RPC without one is denied, and a test fails the build. Access is granted per global scope and atespace through AccessPolicy (`kubectl ate access-policy`); the install's bootstrap owner is the shared `ate-client` ServiceAccount that `kubectl ate` uses. See `docs/authorization.md` for roles and known gaps. Callers are authenticated via `SPIFFE` mTLS or configured OIDC `JWTProvider`s (`cmd/ateapi/internal/apiauthn`). - **Network Isolation, Ingress, & Egress**: `manifests/ate-install/ate-system-networkpolicies.yaml` enforces default-deny ingress across `ate-system`, and `cmd/atecontroller` reconciles per-`WorkerPool` `NetworkPolicy` objects. Actors run in private `/30` point-to-point veth network namespaces (`internal/ateworker/actornet`) with link-local blocking, policy-enforced DNS filtering (`internal/atunnel/dns`), and zero ambient credentials. `atenet-router --ingress-auth-mode=static-mtls` requires clients to present a `SPIFFE` certificate signed by `--ingress-client-ca-file` whose ID is in `--ingress-allowed-spiffe-ids` (`manifests/ate-install/atenet-router.yaml`). Outbound traffic traverses `atunnel` to `atenet-egress`, which enforces per-actor `EgressPolicy` and injects credentials out-of-band via `cmd/credential-provider/kubernetes-secrets` (`k8s-credential-provider`) or actor JWT minting. diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index e497c6ebbf..1ee448d8f5 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -122,9 +122,6 @@ type Config struct { // follows neither form. ExpectedJWTIssuer string - // APIExperimentalEnableAuthz turns on ate-api-server's runtime - // authorization (ATE_API_EXPERIMENTAL_ENABLE_AUTHZ). - APIExperimentalEnableAuthz bool // APIAuthzBootstrapOwners is the comma-separated principal list granted // owner on global:root (ATE_API_AUTHZ_BOOTSTRAP_OWNERS). APIAuthzBootstrapOwners string diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index 1e887b4f45..fcfb93ccf4 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -48,7 +48,6 @@ func loadEnv(t *testing.T) { "ANTHROPIC_API_KEY", "ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE", "ATE_API_AUTHZ_BOOTSTRAP_OWNERS", - "ATE_API_EXPERIMENTAL_ENABLE_AUTHZ", "ATE_API_LOG_LEVEL", "ATE_API_POSTGRES_CLOUDSQL_GSA", "ATE_API_POSTGRES_CLOUDSQL_IAM_AUTH", @@ -680,36 +679,27 @@ func TestLoadAPIAuthz(t *testing.T) { owners = "alice@example.com,bob@example.com" ) for _, tc := range []struct { - name string - env map[string]string - wantEnabled bool - wantOwners string - wantErr bool + name string + env map[string]string + wantOwners string + wantErr bool }{ { - name: "on by default, owned by the client ServiceAccount", - wantEnabled: true, - wantOwners: clientOwner, - }, - { - name: "default owner follows the namespace", - env: map[string]string{"ATE_NAMESPACE": "custom"}, - wantEnabled: true, - wantOwners: "system:serviceaccount:custom:ate-client", + name: "owned by the client ServiceAccount by default", + wantOwners: clientOwner, }, { - name: "explicit owners replace the default", - env: map[string]string{"ATE_API_AUTHZ_BOOTSTRAP_OWNERS": owners}, - wantEnabled: true, - wantOwners: owners, + name: "default owner follows the namespace", + env: map[string]string{"ATE_NAMESPACE": "custom"}, + wantOwners: "system:serviceaccount:custom:ate-client", }, { - name: "turned off", - env: map[string]string{"ATE_API_EXPERIMENTAL_ENABLE_AUTHZ": "false"}, - wantOwners: clientOwner, + name: "explicit owners replace the default", + env: map[string]string{"ATE_API_AUTHZ_BOOTSTRAP_OWNERS": owners}, + wantOwners: owners, }, { - name: "enabled with only separators", + name: "only separators", env: map[string]string{"ATE_API_AUTHZ_BOOTSTRAP_OWNERS": " , "}, wantErr: true, }, @@ -729,8 +719,8 @@ func TestLoadAPIAuthz(t *testing.T) { if err != nil { t.Fatalf("Load() error = %v", err) } - if cfg.APIExperimentalEnableAuthz != tc.wantEnabled || cfg.APIAuthzBootstrapOwners != tc.wantOwners { - t.Errorf("authz = %v, %q, want %v, %q", cfg.APIExperimentalEnableAuthz, cfg.APIAuthzBootstrapOwners, tc.wantEnabled, tc.wantOwners) + if cfg.APIAuthzBootstrapOwners != tc.wantOwners { + t.Errorf("APIAuthzBootstrapOwners = %q, want %q", cfg.APIAuthzBootstrapOwners, tc.wantOwners) } }) } diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index 0861321817..03f2c7c8c4 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -212,8 +212,8 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro } // kubectl-ate, ate-setup's demo steps and the e2e clients fall back to // tokens minted from the client ServiceAccount, so making it the default - // bootstrap owner keeps them working with authorization on. The deploy - // warns whenever it is a bootstrap owner. + // bootstrap owner keeps them working. The deploy warns whenever it is a + // bootstrap owner. bootstrapOwners := r.String("ateapi.authz.bootstrapOwners") if bootstrapOwners == "" { bootstrapOwners = installdefaults.ClientPrincipal(r.String("namespace")) @@ -230,8 +230,7 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro ClusterLocation: r.String("gcp.clusterLocation"), ExpectedJWTIssuer: r.String("ateapi.expectedJWTIssuer"), - APIExperimentalEnableAuthz: r.Bool("ateapi.authz.enabled"), - APIAuthzBootstrapOwners: bootstrapOwners, + APIAuthzBootstrapOwners: bootstrapOwners, BucketName: r.String("storage.bucketName"), SnapshotLocation: r.String("storage.snapshotLocation"), @@ -478,12 +477,11 @@ func validateResolved(cfg *Config, r *Resolved) error { return fmt.Errorf("%w (from %s)", err, provider.From.Describe(provider.Setting)) } } - // ate-api-server exits on this combination, so catch it before deploying. - if cfg.APIExperimentalEnableAuthz && strings.Trim(cfg.APIAuthzBootstrapOwners, " ,") == "" { - enabled, _ := r.Value("ateapi.authz.enabled") + // ate-api-server exits without a bootstrap owner, so catch it before + // deploying. + if strings.Trim(cfg.APIAuthzBootstrapOwners, " ,") == "" { owners, _ := r.Value("ateapi.authz.bootstrapOwners") - return &ConflictError{A: enabled, B: owners, - Why: "enforced authorization needs at least one bootstrap owner"} + return &InvalidError{Value: owners, Want: "a list of at least one bootstrap owner"} } return nil } diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index f7d2cfd1ce..d94a95a44f 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -177,15 +177,10 @@ var Registry = []Setting{ Kind: KindString, Usage: "Service account token issuer ate-api-server trusts, overriding derivation and discovery", }, - { - Key: "ateapi.authz.enabled", Env: "ATE_API_EXPERIMENTAL_ENABLE_AUTHZ", - Flag: "ateapi-experimental-enable-authz", Kind: KindBool, Default: "true", - Usage: "Enforce ate-api-server's runtime authorization", - }, { Key: "ateapi.authz.bootstrapOwners", Env: "ATE_API_AUTHZ_BOOTSTRAP_OWNERS", Flag: "ateapi-authz-bootstrap-owners", Kind: KindString, - Usage: "Comma-separated principals that always hold owner on global:root when authorization is enforced. " + + Usage: "Comma-separated principals that always hold owner on global:root. " + "Unset means kubectl-ate's ServiceAccount, system:serviceaccount::ate-client", }, { diff --git a/cmd/ate-setup/internal/steps/apienv.go b/cmd/ate-setup/internal/steps/apienv.go index 2675bdc954..c9fc76e08d 100644 --- a/cmd/ate-setup/internal/steps/apienv.go +++ b/cmd/ate-setup/internal/steps/apienv.go @@ -158,15 +158,12 @@ func (e *Env) UpdateAPIServerEnvVars(ctx context.Context) error { return e.annotateAPIServerEnvHash(ctx) } -// sharedBootstrapOwnerWarning returns a warning when enforced authorization -// makes the shared client ServiceAccount a bootstrap owner, and "" otherwise. +// sharedBootstrapOwnerWarning returns a warning when the shared client +// ServiceAccount is a bootstrap owner, and "" otherwise. // It is the default so demos, benchmarks and e2e work, but everyone who can // mint its token, including every kubectl-ate user without --token-file, is // then a global owner. -func sharedBootstrapOwnerWarning(enabled bool, bootstrapOwners, namespace string) string { - if !enabled { - return "" - } +func sharedBootstrapOwnerWarning(bootstrapOwners, namespace string) string { shared := installdefaults.ClientPrincipal(namespace) for owner := range strings.SplitSeq(bootstrapOwners, ",") { if strings.TrimSpace(owner) == shared { @@ -349,26 +346,22 @@ func (e *Env) applyWithAPIServerEnvHash(ctx context.Context, manifest []byte) er if err := stampAPIServerEnvHash(objs, hash); err != nil { return err } - if err := setAPIServerAuthzArgs(objs, e.Cfg.APIExperimentalEnableAuthz, e.Cfg.APIAuthzBootstrapOwners); err != nil { + if err := setAPIServerAuthzArgs(objs, e.Cfg.APIAuthzBootstrapOwners); err != nil { return err } - if msg := sharedBootstrapOwnerWarning(e.Cfg.APIExperimentalEnableAuthz, e.Cfg.APIAuthzBootstrapOwners, e.Namespace()); msg != "" { + if msg := sharedBootstrapOwnerWarning(e.Cfg.APIAuthzBootstrapOwners, e.Namespace()); msg != "" { log.Warnf("%s", msg) } return e.Kube.Apply(ctx, objs) } -// apiServerAuthzFlags are the ate-api-server flags setAPIServerAuthzArgs owns. -var apiServerAuthzFlags = []string{"--experimental-enable-authz", "--authz-bootstrap-owners"} +// apiServerAuthzFlag is the ate-api-server flag setAPIServerAuthzArgs owns. +const apiServerAuthzFlag = "--authz-bootstrap-owners" -// setAPIServerAuthzArgs sets ate-api-server's authorization flags on its +// setAPIServerAuthzArgs sets ate-api-server's bootstrap owners flag on its // container in objs, replacing any the manifest already carries. A changed // flag changes the pod template, so it rolls the Deployment. -// -// The bootstrap owners are passed even with enforcement off: access policy -// RPCs are checked either way, so without them nobody could set up policies -// before turning enforcement on. -func setAPIServerAuthzArgs(objs []*unstructured.Unstructured, enabled bool, bootstrapOwners string) error { +func setAPIServerAuthzArgs(objs []*unstructured.Unstructured, bootstrapOwners string) error { var dep *unstructured.Unstructured for _, obj := range objs { if obj.GetKind() == "Deployment" && obj.GetName() == "ate-api-server" { @@ -393,15 +386,10 @@ func setAPIServerAuthzArgs(objs []*unstructured.Unstructured, enabled bool, boot return fmt.Errorf("%s: %w", kube.Describe(dep), err) } args = slices.DeleteFunc(args, func(arg string) bool { - return slices.ContainsFunc(apiServerAuthzFlags, func(flag string) bool { - return arg == flag || strings.HasPrefix(arg, flag+"=") - }) + return arg == apiServerAuthzFlag || strings.HasPrefix(arg, apiServerAuthzFlag+"=") }) - if enabled { - args = append(args, "--experimental-enable-authz=true") - } if strings.Trim(bootstrapOwners, " ,") != "" { - args = append(args, "--authz-bootstrap-owners="+bootstrapOwners) + args = append(args, apiServerAuthzFlag+"="+bootstrapOwners) } if err := unstructured.SetNestedStringSlice(container, args, "args"); err != nil { return fmt.Errorf("%s: %w", kube.Describe(dep), err) diff --git a/cmd/ate-setup/internal/steps/apienv_test.go b/cmd/ate-setup/internal/steps/apienv_test.go index a78d3f9378..81e22f22bd 100644 --- a/cmd/ate-setup/internal/steps/apienv_test.go +++ b/cmd/ate-setup/internal/steps/apienv_test.go @@ -618,8 +618,8 @@ spec: } } -// Authorization reaches ate-api-server as flags on its container, so a change -// rolls the Deployment, and the envvars ConfigMap carries none of it. +// The bootstrap owners reach ate-api-server as a flag on its container, so a +// change rolls the Deployment, and the envvars ConfigMap carries none of it. func TestSetAPIServerAuthzArgs(t *testing.T) { const manifest = `apiVersion: apps/v1 kind: Deployment @@ -636,35 +636,22 @@ spec: args: - --config-file=/run/apiconfig.ate.dev/apiconfig.textproto - --authz-bootstrap-owners=stale - - --experimental-enable-authz ` for _, tc := range []struct { - name string - enabled bool - owners string - want []string + name string + owners string + want []string }{ { - name: "enforced", - enabled: true, - owners: "alice,bob", + name: "replaces the owners", + owners: "alice,bob", want: []string{ "--config-file=/run/apiconfig.ate.dev/apiconfig.textproto", - "--experimental-enable-authz=true", "--authz-bootstrap-owners=alice,bob", }, }, { - // Owners stay, so access policies can be set up before enforcement is on. - name: "off keeps the owners", - owners: "alice", - want: []string{ - "--config-file=/run/apiconfig.ate.dev/apiconfig.textproto", - "--authz-bootstrap-owners=alice", - }, - }, - { - name: "off with no owners", + name: "no owners drops the flag", want: []string{"--config-file=/run/apiconfig.ate.dev/apiconfig.textproto"}, }, } { @@ -673,7 +660,7 @@ spec: if err != nil { t.Fatalf("DecodeManifestBytes() error = %v", err) } - if err := setAPIServerAuthzArgs(objs, tc.enabled, tc.owners); err != nil { + if err := setAPIServerAuthzArgs(objs, tc.owners); err != nil { t.Fatalf("setAPIServerAuthzArgs() error = %v", err) } containers, _, _ := unstructured.NestedSlice(objs[0].Object, "spec", "template", "spec", "containers") @@ -694,18 +681,17 @@ spec: if err != nil { t.Fatalf("DecodeManifestBytes() error = %v", err) } - if err := setAPIServerAuthzArgs(objs, true, "alice"); err == nil { + if err := setAPIServerAuthzArgs(objs, "alice"); err == nil { t.Error("setAPIServerAuthzArgs() without deployment/ate-api-server error = nil, want an error") } } func TestCreateAPIServerEnvVarsOmitsAuthz(t *testing.T) { e := &Env{Cfg: &config.Config{ - PostgresReadWriteRole: config.DefaultPostgresReadWriteRole, - PostgresOwnerRole: config.DefaultPostgresOwnerRole, - APIExperimentalEnableAuthz: true, - APIAuthzBootstrapOwners: "system:serviceaccount:ate-system:ate-client", - SnapshotLocation: "gs://ate-snapshots/", + PostgresReadWriteRole: config.DefaultPostgresReadWriteRole, + PostgresOwnerRole: config.DefaultPostgresOwnerRole, + APIAuthzBootstrapOwners: "system:serviceaccount:ate-system:ate-client", + SnapshotLocation: "gs://ate-snapshots/", }, Kube: fakeKube(t, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: NamespaceAteSystem}}, apiServerEnvVarsConfigMap(map[string]string{}), @@ -718,10 +704,8 @@ func TestCreateAPIServerEnvVarsOmitsAuthz(t *testing.T) { if err != nil || cm == nil { t.Fatalf("GetConfigMap() = %v, %v", cm, err) } - for _, key := range []string{"ATE_API_EXPERIMENTAL_ENABLE_AUTHZ", "ATE_API_AUTHZ_BOOTSTRAP_OWNERS"} { - if v, ok := cm.Data[key]; ok { - t.Errorf("ConfigMap carries %s=%q; authorization is passed as flags", key, v) - } + if v, ok := cm.Data["ATE_API_AUTHZ_BOOTSTRAP_OWNERS"]; ok { + t.Errorf("ConfigMap carries ATE_API_AUTHZ_BOOTSTRAP_OWNERS=%q; the bootstrap owners are passed as a flag", v) } } @@ -730,22 +714,20 @@ func TestCreateAPIServerEnvVarsOmitsAuthz(t *testing.T) { func TestSharedBootstrapOwnerWarning(t *testing.T) { const shared = "system:serviceaccount:ate-system:ate-client" for _, tc := range []struct { - name string - enabled bool - owners string - warn bool + name string + owners string + warn bool }{ - {name: "default owner", enabled: true, owners: shared, warn: true}, - {name: "listed among others", enabled: true, owners: "alice, " + shared + " ,bob", warn: true}, - {name: "own principals only", enabled: true, owners: "alice,bob", warn: false}, - {name: "another namespace's client", enabled: true, owners: "system:serviceaccount:other:ate-client", warn: false}, - {name: "prefix is not a match", enabled: true, owners: shared + "-x", warn: false}, - {name: "authorization off", enabled: false, owners: shared, warn: false}, + {name: "default owner", owners: shared, warn: true}, + {name: "listed among others", owners: "alice, " + shared + " ,bob", warn: true}, + {name: "own principals only", owners: "alice,bob", warn: false}, + {name: "another namespace's client", owners: "system:serviceaccount:other:ate-client", warn: false}, + {name: "prefix is not a match", owners: shared + "-x", warn: false}, } { t.Run(tc.name, func(t *testing.T) { - got := sharedBootstrapOwnerWarning(tc.enabled, tc.owners, "ate-system") + got := sharedBootstrapOwnerWarning(tc.owners, "ate-system") if (got != "") != tc.warn { - t.Fatalf("sharedBootstrapOwnerWarning(%v, %q) = %q, want warning: %v", tc.enabled, tc.owners, got, tc.warn) + t.Fatalf("sharedBootstrapOwnerWarning(%q) = %q, want warning: %v", tc.owners, got, tc.warn) } if tc.warn { for _, want := range []string{shared, "--ateapi-authz-bootstrap-owners", "ATE_API_AUTHZ_BOOTSTRAP_OWNERS"} { diff --git a/cmd/ateapi/internal/authz/interceptor.go b/cmd/ateapi/internal/authz/interceptor.go index ee20395535..586c16f35e 100644 --- a/cmd/ateapi/internal/authz/interceptor.go +++ b/cmd/ateapi/internal/authz/interceptor.go @@ -23,10 +23,9 @@ import ( ) // UnaryServerInterceptor returns a gRPC unary interceptor that enforces per-RPC -// permissions registered in defaultRPCPermissions using authorizer. When -// enforce is false, only rules marked alwaysEnforce are checked. When enforce -// is true, an RPC with no rule is denied. -func UnaryServerInterceptor(authorizer *Authorizer, enforce bool) grpc.UnaryServerInterceptor { +// permissions registered in defaultRPCPermissions using authorizer. An RPC +// with no rule is denied. +func UnaryServerInterceptor(authorizer *Authorizer) grpc.UnaryServerInterceptor { return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { if IsBypassed(ctx) { return handler(ctx, req) @@ -35,9 +34,6 @@ func UnaryServerInterceptor(authorizer *Authorizer, enforce bool) grpc.UnaryServ if registered && rule.authorizedInHandler != "" { return handler(ctx, req) } - if !enforce && !rule.alwaysEnforce { - return handler(ctx, req) - } if !registered { return nil, apierror.PermissionDenied("permission denied: no authorization rule for %s", info.FullMethod) } diff --git a/cmd/ateapi/internal/authz/interceptor_test.go b/cmd/ateapi/internal/authz/interceptor_test.go index 52df3035c7..adc5ab63b1 100644 --- a/cmd/ateapi/internal/authz/interceptor_test.go +++ b/cmd/ateapi/internal/authz/interceptor_test.go @@ -16,7 +16,6 @@ package authz import ( "context" - "strings" "testing" "go.ate.dev/substrate/internal/apierror" @@ -57,7 +56,7 @@ func TestUnaryServerInterceptor_QuickRejectionAndDispatch(t *testing.T) { Kind: principal.KindJWT, }) - interceptor := UnaryServerInterceptor(authorizer, true) + interceptor := UnaryServerInterceptor(authorizer) tests := []struct { name string @@ -152,7 +151,7 @@ func TestUnaryServerInterceptor_QuickRejectionAndDispatch(t *testing.T) { func TestUnaryServerInterceptor_MalformedRequestRequiresPrincipalThenFailsClosed(t *testing.T) { authorizer := setupTestAuthorizer(t) - interceptor := UnaryServerInterceptor(authorizer, true) + interceptor := UnaryServerInterceptor(authorizer) // 1. Unauthenticated caller with empty atespace name -> Unauthenticated _, err := interceptor(context.Background(), &ateapipb.GetAtespaceRequest{}, &grpc.UnaryServerInfo{ @@ -196,74 +195,8 @@ func TestUnaryServerInterceptor_MalformedRequestRequiresPrincipalThenFailsClosed } -func TestUnaryServerInterceptor_EnforcementDisabled(t *testing.T) { - authorizer := setupTestAuthorizer(t) - interceptor := UnaryServerInterceptor(authorizer, false) - bobCtx := principal.InjectContext(context.Background(), principal.PrincipalInfo{ - ID: "bob@example.com", - Kind: principal.KindJWT, - }) - aliceCtx := principal.InjectContext(context.Background(), principal.PrincipalInfo{ - ID: "alice@example.com", - Kind: principal.KindJWT, - }) - - tests := []struct { - name string - ctx context.Context - fullMethod string - req any - wantCode codes.Code - }{ - { - name: "regular RPC skips the check", - ctx: bobCtx, - fullMethod: ateapipb.Control_CreateAtespace_FullMethodName, - req: &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: "team1"}}}, - wantCode: codes.OK, - }, - { - name: "governance RPC is still denied", - ctx: bobCtx, - fullMethod: ateapipb.Control_UpdateGlobalAccessPolicy_FullMethodName, - req: &ateapipb.UpdateGlobalAccessPolicyRequest{}, - wantCode: codes.PermissionDenied, - }, - { - name: "governance RPC is still allowed for an owner", - ctx: aliceCtx, - fullMethod: ateapipb.Control_UpdateGlobalAccessPolicy_FullMethodName, - req: &ateapipb.UpdateGlobalAccessPolicyRequest{}, - wantCode: codes.OK, - }, - { - name: "governance RPC still requires a principal", - ctx: context.Background(), - fullMethod: ateapipb.Control_GetAtespaceAccessPolicy_FullMethodName, - req: &ateapipb.GetAtespaceAccessPolicyRequest{}, - wantCode: codes.Unauthenticated, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - handlerCalled := false - _, err := interceptor(tc.ctx, tc.req, &grpc.UnaryServerInfo{FullMethod: tc.fullMethod}, func(ctx context.Context, req any) (any, error) { - handlerCalled = true - return "ok", nil - }) - if apierror.Code(err) != tc.wantCode { - t.Fatalf("apierror.Code(err) = %v, want %v (err: %v)", apierror.Code(err), tc.wantCode, err) - } - if wantHandler := tc.wantCode == codes.OK; handlerCalled != wantHandler { - t.Fatalf("handlerCalled = %v, want %v", handlerCalled, wantHandler) - } - }) - } -} - -// An RPC with no rule is denied when enforcement is on and passes through when -// it is off; an RPC its handler authorizes passes through either way. Neither -// reaches OpenFGA, so the authorizer is nil. +// An RPC with no rule is denied, and an RPC its handler authorizes passes +// through. Neither reaches OpenFGA, so the authorizer is nil. func TestUnaryServerInterceptor_UnregisteredAndHandlerAuthorizedRPCs(t *testing.T) { const unregistered = "/ateapi.Control/NotARealMethod" aliceCtx := principal.InjectContext(context.Background(), principal.PrincipalInfo{ @@ -273,23 +206,20 @@ func TestUnaryServerInterceptor_UnregisteredAndHandlerAuthorizedRPCs(t *testing. tests := []struct { name string - enforce bool ctx context.Context fullMethod string wantCode codes.Code }{ - {"unregistered RPC is denied with enforcement on", true, aliceCtx, unregistered, codes.PermissionDenied}, - {"unregistered RPC is denied without a principal", true, context.Background(), unregistered, codes.PermissionDenied}, - {"unregistered RPC passes with enforcement off", false, aliceCtx, unregistered, codes.OK}, - {"RegisterWorker passes with enforcement on", true, context.Background(), ateapipb.WorkerService_RegisterWorker_FullMethodName, codes.OK}, - {"RequestActorSuspend passes with enforcement on", true, context.Background(), ateapipb.WorkerService_RequestActorSuspend_FullMethodName, codes.OK}, - {"MintWorkerActorCertificate passes with enforcement on", true, context.Background(), ateapipb.WorkerService_MintWorkerActorCertificate_FullMethodName, codes.OK}, - {"MintWorkerActorJwt passes with enforcement on", true, context.Background(), ateapipb.WorkerService_MintWorkerActorJwt_FullMethodName, codes.OK}, - {"RegisterWorker passes with enforcement off", false, context.Background(), ateapipb.WorkerService_RegisterWorker_FullMethodName, codes.OK}, + {"unregistered RPC is denied", aliceCtx, unregistered, codes.PermissionDenied}, + {"unregistered RPC is denied without a principal", context.Background(), unregistered, codes.PermissionDenied}, + {"RegisterWorker passes", context.Background(), ateapipb.WorkerService_RegisterWorker_FullMethodName, codes.OK}, + {"RequestActorSuspend passes", context.Background(), ateapipb.WorkerService_RequestActorSuspend_FullMethodName, codes.OK}, + {"MintWorkerActorCertificate passes", context.Background(), ateapipb.WorkerService_MintWorkerActorCertificate_FullMethodName, codes.OK}, + {"MintWorkerActorJwt passes", context.Background(), ateapipb.WorkerService_MintWorkerActorJwt_FullMethodName, codes.OK}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - interceptor := UnaryServerInterceptor(nil, tc.enforce) + interceptor := UnaryServerInterceptor(nil) handlerCalled := false _, err := interceptor(tc.ctx, struct{}{}, &grpc.UnaryServerInfo{FullMethod: tc.fullMethod}, func(context.Context, any) (any, error) { handlerCalled = true @@ -305,32 +235,6 @@ func TestUnaryServerInterceptor_UnregisteredAndHandlerAuthorizedRPCs(t *testing. } } -// TestAccessPolicyRPCsAlwaysEnforced guards against an AccessPolicy RPC being -// added without a governance rule, which would leave it unchecked while -// enforcement is disabled. -func TestAccessPolicyRPCsAlwaysEnforced(t *testing.T) { - desc := ateapipb.Control_ServiceDesc - found := 0 - for _, m := range desc.Methods { - if !strings.Contains(m.MethodName, "AccessPolicy") { - continue - } - found++ - fullMethod := "/" + desc.ServiceName + "/" + m.MethodName - rule, ok := defaultRPCPermissions[fullMethod] - if !ok { - t.Errorf("%s has no permission rule", fullMethod) - continue - } - if !rule.alwaysEnforce { - t.Errorf("%s is not marked alwaysEnforce", fullMethod) - } - } - if found == 0 { - t.Fatal("found no AccessPolicy RPCs in Control_ServiceDesc") - } -} - func TestUnaryServerInterceptor_ActorAndTemplateChecks(t *testing.T) { ctx := context.Background() pool := startPostgres(t) @@ -460,7 +364,7 @@ func TestUnaryServerInterceptor_ActorAndTemplateChecks(t *testing.T) { {"global viewer cannot delete template", "global-viewer", deleteTemplate, codes.PermissionDenied}, } - interceptor := UnaryServerInterceptor(authorizer, true) + interceptor := UnaryServerInterceptor(authorizer) for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { userCtx := principal.InjectContext(ctx, principal.PrincipalInfo{ID: tc.user, Kind: principal.KindJWT}) @@ -652,7 +556,7 @@ func TestUnaryServerInterceptor_LifecycleEgressWorkerAndTagChecks(t *testing.T) {"bootstrap owner cannot mint actor certificate", "bootstrap-owner", mintCert, codes.PermissionDenied}, } - interceptor := UnaryServerInterceptor(authorizer, true) + interceptor := UnaryServerInterceptor(authorizer) for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { userCtx := principal.InjectContext(ctx, principal.PrincipalInfo{ID: tc.user, Kind: principal.KindJWT}) @@ -808,7 +712,7 @@ func TestUnaryServerInterceptor_SystemPrincipals(t *testing.T) { {"other mTLS identity cannot create worker", mtls(otherID), createWorker, codes.PermissionDenied}, } - interceptor := UnaryServerInterceptor(authorizer, true) + interceptor := UnaryServerInterceptor(authorizer) for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { userCtx := principal.InjectContext(ctx, tc.principal) diff --git a/cmd/ateapi/internal/authz/registry.go b/cmd/ateapi/internal/authz/registry.go index fb25590eb9..d850d03911 100644 --- a/cmd/ateapi/internal/authz/registry.go +++ b/cmd/ateapi/internal/authz/registry.go @@ -144,10 +144,6 @@ func both(aChecks []check, aErrs field.ErrorList, bChecks []check, bErrs field.E // rpcRule is the permission rule for one RPC. type rpcRule struct { extract checkExtractor - // alwaysEnforce marks RPCs that are checked even when enforcement is - // disabled. AccessPolicy RPCs set it so that nobody can grant themselves - // access while enforcement is off and keep that grant once it is turned on. - alwaysEnforce bool // authorizedInHandler, when set, says how the RPC's handler authorizes // the caller; the interceptor passes such RPCs through unchecked. authorizedInHandler string @@ -157,10 +153,6 @@ func rule(extract checkExtractor) rpcRule { return rpcRule{extract: extract} } -func governance(extract checkExtractor) rpcRule { - return rpcRule{extract: extract, alwaysEnforce: true} -} - // inHandler registers an RPC whose handler authorizes the caller, as how // describes. func inHandler(how string) rpcRule { @@ -168,8 +160,8 @@ func inHandler(how string) rpcRule { } // defaultRPCPermissions is the declarative registry mapping gRPC full method -// names to their permission rules. With enforcement on, the interceptor denies -// any RPC missing from it. +// names to their permission rules. The interceptor denies any RPC missing +// from it. var defaultRPCPermissions = map[string]rpcRule{ // Atespaces. ateapipb.Control_CreateAtespace_FullMethodName: rule(checksOf(func(*ateapipb.CreateAtespaceRequest) ([]check, field.ErrorList) { @@ -186,25 +178,25 @@ var defaultRPCPermissions = map[string]rpcRule{ })), // Access policies. - ateapipb.Control_GetGlobalAccessPolicy_FullMethodName: governance(checksOf(func(*ateapipb.GetGlobalAccessPolicyRequest) ([]check, field.ErrorList) { + ateapipb.Control_GetGlobalAccessPolicy_FullMethodName: rule(checksOf(func(*ateapipb.GetGlobalAccessPolicyRequest) ([]check, field.ErrorList) { return onGlobal(RelationCanGetAccessPolicy) })), - ateapipb.Control_CreateGlobalAccessPolicy_FullMethodName: governance(checksOf(func(*ateapipb.CreateGlobalAccessPolicyRequest) ([]check, field.ErrorList) { + ateapipb.Control_CreateGlobalAccessPolicy_FullMethodName: rule(checksOf(func(*ateapipb.CreateGlobalAccessPolicyRequest) ([]check, field.ErrorList) { return onGlobal(RelationCanCreateAccessPolicy) })), - ateapipb.Control_UpdateGlobalAccessPolicy_FullMethodName: governance(checksOf(func(*ateapipb.UpdateGlobalAccessPolicyRequest) ([]check, field.ErrorList) { + ateapipb.Control_UpdateGlobalAccessPolicy_FullMethodName: rule(checksOf(func(*ateapipb.UpdateGlobalAccessPolicyRequest) ([]check, field.ErrorList) { return onGlobal(RelationCanUpdateAccessPolicy) })), - ateapipb.Control_GetAtespaceAccessPolicy_FullMethodName: governance(checksOf(func(r *ateapipb.GetAtespaceAccessPolicyRequest) ([]check, field.ErrorList) { + ateapipb.Control_GetAtespaceAccessPolicy_FullMethodName: rule(checksOf(func(r *ateapipb.GetAtespaceAccessPolicyRequest) ([]check, field.ErrorList) { return onAtespace(RelationCanGetAccessPolicy, r.GetAtespace().GetName(), field.NewPath("atespace", "name")) })), - ateapipb.Control_CreateAtespaceAccessPolicy_FullMethodName: governance(checksOf(func(r *ateapipb.CreateAtespaceAccessPolicyRequest) ([]check, field.ErrorList) { + ateapipb.Control_CreateAtespaceAccessPolicy_FullMethodName: rule(checksOf(func(r *ateapipb.CreateAtespaceAccessPolicyRequest) ([]check, field.ErrorList) { return onAtespace(RelationCanCreateAccessPolicy, r.GetAtespace().GetName(), field.NewPath("atespace", "name")) })), - ateapipb.Control_UpdateAtespaceAccessPolicy_FullMethodName: governance(checksOf(func(r *ateapipb.UpdateAtespaceAccessPolicyRequest) ([]check, field.ErrorList) { + ateapipb.Control_UpdateAtespaceAccessPolicy_FullMethodName: rule(checksOf(func(r *ateapipb.UpdateAtespaceAccessPolicyRequest) ([]check, field.ErrorList) { return onAtespace(RelationCanUpdateAccessPolicy, r.GetAtespace().GetName(), field.NewPath("atespace", "name")) })), - ateapipb.Control_DeleteAtespaceAccessPolicy_FullMethodName: governance(checksOf(func(r *ateapipb.DeleteAtespaceAccessPolicyRequest) ([]check, field.ErrorList) { + ateapipb.Control_DeleteAtespaceAccessPolicy_FullMethodName: rule(checksOf(func(r *ateapipb.DeleteAtespaceAccessPolicyRequest) ([]check, field.ErrorList) { return onAtespace(RelationCanDeleteAccessPolicy, r.GetAtespace().GetName(), field.NewPath("atespace", "name")) })), diff --git a/cmd/ateapi/internal/authz/registry_test.go b/cmd/ateapi/internal/authz/registry_test.go index 05d4e26386..b6821ca7d3 100644 --- a/cmd/ateapi/internal/authz/registry_test.go +++ b/cmd/ateapi/internal/authz/registry_test.go @@ -709,9 +709,6 @@ func TestDefaultRPCPermissions_CoversEveryServedRPC(t *testing.T) { if (r.extract == nil) == (r.authorizedInHandler == "") { t.Errorf("%s: want exactly one of a check extractor or authorizedInHandler", fullMethod) } - if r.authorizedInHandler != "" && r.alwaysEnforce { - t.Errorf("%s: authorizedInHandler and alwaysEnforce are exclusive", fullMethod) - } } for fullMethod := range defaultRPCPermissions { if !served[fullMethod] { diff --git a/cmd/ateapi/internal/controlapi/access_policy_test.go b/cmd/ateapi/internal/controlapi/access_policy_test.go index fd24e4c68e..c50529b934 100644 --- a/cmd/ateapi/internal/controlapi/access_policy_test.go +++ b/cmd/ateapi/internal/controlapi/access_policy_test.go @@ -49,7 +49,7 @@ func TestAccessPolicy_GlobalAndAtespaceGovernance(t *testing.T) { persistence.SetPolicyManager(policyManager) svc := NewRPCService(persistence, nil, nil, nil, nil, nil, "", nil, nil, "", nil, nil, "") - interceptor := authz.UnaryServerInterceptor(authorizer, true) + interceptor := authz.UnaryServerInterceptor(authorizer) userCtx := func(id string) context.Context { return principal.InjectContext(ctx, principal.PrincipalInfo{ @@ -200,7 +200,7 @@ func TestAccessPolicy_GlobalAndAtespaceGovernance(t *testing.T) { if err != nil { t.Fatalf("authz.New after restart failed: %v", err) } - restartedInterceptor := authz.UnaryServerInterceptor(restarted, true) + restartedInterceptor := authz.UnaryServerInterceptor(restarted) getGlobalAfterRestart := func(c context.Context) error { _, err := restartedInterceptor(c, &ateapipb.GetGlobalAccessPolicyRequest{Name: resources.SingletonName}, &grpc.UnaryServerInfo{FullMethod: ateapipb.Control_GetGlobalAccessPolicy_FullMethodName}, func(c context.Context, r any) (any, error) { return svc.GetGlobalAccessPolicy(c, r.(*ateapipb.GetGlobalAccessPolicyRequest)) diff --git a/cmd/ateapi/internal/controlapi/atespace_test.go b/cmd/ateapi/internal/controlapi/atespace_test.go index 4564485076..6a649fda48 100644 --- a/cmd/ateapi/internal/controlapi/atespace_test.go +++ b/cmd/ateapi/internal/controlapi/atespace_test.go @@ -73,7 +73,7 @@ func TestAtespace_EndToEndOpenFGAScenarios(t *testing.T) { svc := &RPCService{ impl: newServiceImpl(persistence, nil), } - interceptor := authz.UnaryServerInterceptor(authorizer, true) + interceptor := authz.UnaryServerInterceptor(authorizer) asUser := func(id string) context.Context { return principal.InjectContext(ctx, principal.PrincipalInfo{ diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 9bc6d2888e..4f854b7fc4 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -85,11 +85,10 @@ var ( postgresOwnerRole = pflag.String("postgres-owner-role", "", "Required PostgreSQL role assumed by owner connections.") postgresSchema = pflag.String("postgres-schema", "substrate", "PostgreSQL schema for Substrate tables. This overrides a search_path connection parameter.") postgresPoolMaxConns = pflag.Int32("postgres-pool-max-conns", 0, "Maximum connections in the shared Substrate and OpenFGA read/write PostgreSQL pool. Does not affect the owner, watch, or readiness probe pools. The DSN or pgx default is used when unset.") - experimentalEnableAuthz = pflag.Bool("experimental-enable-authz", false, "Enforce OpenFGA authorization checks on all registered RPCs (experimental). AccessPolicy RPCs are always checked.") // TODO: Move the authz settings into the hot-reloadable config proto // (agent-substrate/substrate#2021) once it lands, so bootstrap owner // changes take effect without a restart. - authzBootstrapOwners = pflag.StringSlice("authz-bootstrap-owners", nil, "Principal IDs that are always global owners while listed, independent of the stored global AccessPolicy. Removing an ID revokes its access on restart. At least one is required when --experimental-enable-authz is set.") + authzBootstrapOwners = pflag.StringSlice("authz-bootstrap-owners", nil, "Principal IDs that are always global owners while listed, independent of the stored global AccessPolicy. Removing an ID revokes its access on restart. Required: at least one.") // These SPIFFE IDs hold the matching system component role when they call // over mTLS. controllerSPIFFEID = pflag.String("controller-spiffe-id", installdefaults.ControllerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of ate-controller. Holds the controller authz role, which manages workers.") @@ -232,13 +231,10 @@ func main() { // (atepg's outbox maintenance loop); stop it on shutdown before closing pool. defer persistence.Close() - // The authz stack is always wired so policy tuples stay in sync with the - // store; --experimental-enable-authz only controls enforcement in the - // interceptor. - if *experimentalEnableAuthz && len(*authzBootstrapOwners) == 0 { + if len(*authzBootstrapOwners) == 0 { // Without a bootstrap owner, nobody could create the global - // AccessPolicy, so the enforced API would be unusable. - serverboot.Fatal(ctx, "Invalid flags", fmt.Errorf("--authz-bootstrap-owners must list at least one principal when --experimental-enable-authz is set")) + // AccessPolicy, so the API would be unusable. + serverboot.Fatal(ctx, "Invalid flags", fmt.Errorf("--authz-bootstrap-owners must list at least one principal")) } // atelet shares ateapi's namespace in every supported deployment topology, // so we read it from Kubernetes' downward API rather than expose a flag. @@ -379,7 +375,7 @@ func main() { apiauthn.UnaryServerInterceptor(authCfg), ateinterceptors.MaxDeadlineUnaryInterceptor(maxRPCDeadline), ateinterceptors.ServerUnaryInterceptor, - authz.UnaryServerInterceptor(authorizer, *experimentalEnableAuthz), + authz.UnaryServerInterceptor(authorizer), ateinterceptors.RejectUnknownFieldsUnaryInterceptor, preview.UnaryServerInterceptor, } @@ -504,7 +500,6 @@ func logFlagValues(ctx context.Context) { slog.String("postgres-owner-role", *postgresOwnerRole), slog.String("postgres-schema", *postgresSchema), slog.Int("postgres-pool-max-conns", int(*postgresPoolMaxConns)), - slog.Bool("experimental-enable-authz", *experimentalEnableAuthz), slog.Any("authz-bootstrap-owners", *authzBootstrapOwners), slog.String("actor-id-jwt-pool", *actorIDJWTPoolFile), slog.String("actor-jwt-issuer", *actorJWTIssuer), diff --git a/docs/api-guide.md b/docs/api-guide.md index e883609471..1e465a5baf 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -675,7 +675,7 @@ Query the physical resource pool. ate-api-server mints credentials that identify an actor to services outside Substrate. They name the logical actor, so they stay the same as the actor moves between workers. This is distinct from the `actorMetadata` data source described under [SystemInfo Volumes](#systeminfo-volumes), which only tells an actor its own identity fields (name, atespace, uid). -Both RPCs are on `ateapi.Control`. With authorization on, as ate-setup installs it by default, only the egress gateway may call them; see [authorization.md](authorization.md). It calls `MintActorJwt` when an egress policy injects an actor JWT into a request. +Both RPCs are on `ateapi.Control`. Only the egress gateway may call them; see [authorization.md](authorization.md). It calls `MintActorJwt` when an egress policy injects an actor JWT into a request. ### `MintActorJwt` diff --git a/docs/architecture.md b/docs/architecture.md index 1bda9e6bca..7b40dbe937 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -547,8 +547,7 @@ Agent Substrate is built on a **Defense-in-Depth** model: security context even as they migrate across physical nodes or code versions. - * **API Authorization**: With enforcement on, the default in installs from - `ate-setup`, every `ate-api-server` RPC is authorized. Users + * **API Authorization**: Every `ate-api-server` RPC is authorized. Users and automation get owner, editor, or viewer roles on the global scope or on an atespace through AccessPolicy (`kubectl ate access-policy`), and Substrate's components hold fixed system roles bound to their pod diff --git a/docs/authorization.md b/docs/authorization.md index 1b4e16355d..2e6f1c79c7 100644 --- a/docs/authorization.md +++ b/docs/authorization.md @@ -24,15 +24,14 @@ callers authenticate, see [authentication.md](authentication.md). ## Settings -These are `ate-setup` settings, so the environment variable is read by -`ate-setup`. It passes them to `ate-api-server` as the -`--experimental-enable-authz` and `--authz-bootstrap-owners` flags on its -container; an install that doesn't use `ate-setup` sets those flags in its -manifest. A change rolls `ate-api-server`. +Authorization is always enforced. `ate-setup` reads the setting below and +passes it to `ate-api-server` as the `--authz-bootstrap-owners` flag on its +container; an install that doesn't use `ate-setup` sets that flag in its +manifest. `ate-api-server` won't start without a bootstrap owner. A change +rolls `ate-api-server`. | Setting | Environment variable | Example | |---|---|---| -| `ateapi.authz.enabled` | `ATE_API_EXPERIMENTAL_ENABLE_AUTHZ` | `true` (the default) | | `ateapi.authz.bootstrapOwners` | `ATE_API_AUTHZ_BOOTSTRAP_OWNERS` | `alice@example.com,bob@example.com` | If you don't set `ateapi.authz.bootstrapOwners`, `ate-setup` uses the shared @@ -40,14 +39,6 @@ If you don't set `ateapi.authz.bootstrapOwners`, `ate-setup` uses the shared so the demos, benchmarks and e2e tests work. See the warning at the top of this page. -```sh -ATE_API_EXPERIMENTAL_ENABLE_AUTHZ=false hack/install-ate.sh --deploy-ate-system # turn it off -``` - -With authorization off, access policy RPCs are still checked, so nobody can -grant themselves access while it is off and keep it once it is on. Bootstrap -owners still apply, so they can set up policies before turning it on. - ## Principals A principal is the ID authentication produces. Access policies name it as @@ -69,7 +60,7 @@ policies say. They write an install's first access policies and recover access if those break. - They come from configuration, not the API. `ate-api-server` reads them at - startup and exits if authorization is on and the list is empty. Restarting + startup and exits if the list is empty. Restarting it with a new list takes effect immediately. - Any principal `ate-api-server` can authenticate can be one. - `ate-setup` warns whenever `ate-client` is one. @@ -176,8 +167,6 @@ viewer, and a global role holds the same role in every atespace. So | `GetAtespaceAccessPolicy` | `can_get_access_policy`, the atespace | Atespace viewer | | `CreateAtespaceAccessPolicy`, `UpdateAtespaceAccessPolicy`, `DeleteAtespaceAccessPolicy` | `can_create_access_policy` / `can_update_access_policy` / `can_delete_access_policy`, the atespace | Atespace owner | -The access policy RPCs are checked even with authorization off. - ### Actor templates and actors | RPC | Permission, on | Allowed for | @@ -236,8 +225,6 @@ certificate. ### Every RPC -With authorization on: - - An RPC with no rule is denied, and a test fails the build if a served RPC has none. - An unauthenticated caller gets `Unauthenticated` before its request is read. diff --git a/docs/operator-install.md b/docs/operator-install.md index f4f8c2d45b..d8d6b7a26e 100644 --- a/docs/operator-install.md +++ b/docs/operator-install.md @@ -277,7 +277,6 @@ demos, the [benchmarks](../benchmarking/README.md#settings), and the | `actorJWT.algorithm` | `ACTOR_JWT_ALGORITHM` | `--actor-jwt-algorithm` | `ES256` | (global) | | `actorJWT.issuer` | `ACTOR_JWT_ISSUER` | `--actor-jwt-issuer` | — | (global) | | `ateapi.authz.bootstrapOwners` | `ATE_API_AUTHZ_BOOTSTRAP_OWNERS` | `--ateapi-authz-bootstrap-owners` | `system:serviceaccount::ate-client` | (global) | -| `ateapi.authz.enabled` | `ATE_API_EXPERIMENTAL_ENABLE_AUTHZ` | `--ateapi-experimental-enable-authz` | `true` | (global) | | `ateapi.expectedJWTIssuer` | `EXPECTED_JWT_ISSUER` | `--ateapi-expected-jwt-issuer` | — | (global) | | `ateapi.logLevel` | `ATE_API_LOG_LEVEL` | `--ateapi-log-level` | — | (global) | | `ateapi.postgres.cloudsql.gsa` | `ATE_API_POSTGRES_CLOUDSQL_GSA` | `--ateapi-postgres-cloudsql-gsa` | — | (global) | From 770abbd1a944873490e39c3ed218978c7eb77b0f Mon Sep 17 00:00:00 2001 From: Vu Le <52352161+lekhoavu2909@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:12:36 +0000 Subject: [PATCH 18/63] ateapi: check mounted volumes are attachable and test assigned_node detach on crash (#2611) Some storage backends (e.g. Cloud Filestore IP export rules) restrict a volume to the node it is published to via CSI `ControllerPublishVolume` / `ControllerUnpublishVolume`, so every publish must be matched by an unpublish to the same node. Since #2222 , detach targets `ActorStatus.assigned_node`, which survives crashes and worker loss. This PR adds checks before attach and test coverage for the paths where detach used to be skipped. Follows #2094, which covers the publish side. - *ateapi: check mounted volumes are attachable before publishing* - *test(ateapi): cover volume detach via assigned_node after crash or worker loss* #### Changes - Worker without a node name: `assignWorkerAttempt` rejects such a worker with when the actor mounts external volumes. Previously the resume failed in `ensureVolumesAttached`, after the worker was booked. - Volume checks before attach: `ensureVolumesAttached` checks that every mounted volume is `STATUS_CREATED` and has a `storage_volume_id` before any attach call, so one bad volume can't leave earlier volumes published. - Unit tests, functional tests - E2E: `assigned_node` equals the worker's `node_name` while RUNNING and is cleared after suspend and revert. The OSS NFS driver doesn't implement `ControllerPublishVolume`. - [x] Tests pass --- .../actor_volume_detach_test.go | 318 ++++++++++++++++++ .../workflow_reconcile_assignments_test.go | 1 + .../internal/controlapi/workflow_resume.go | 30 +- .../controlapi/workflow_resume_test.go | 208 ++++++++++++ .../combinedvolumes/combinedvolumes_test.go | 37 +- 5 files changed, 589 insertions(+), 5 deletions(-) create mode 100644 cmd/ateapi/internal/controlapi/functionaltest/actor_volume_detach_test.go diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_volume_detach_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_volume_detach_test.go new file mode 100644 index 0000000000..e12b10fb75 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_volume_detach_test.go @@ -0,0 +1,318 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package functionaltest + +import ( + "context" + "slices" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/volume" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// calls returns copies of the nodes the plugin attached to and detached from, +// in call order. +func (a *attachFailVolumePlugin) calls() (attached, detached []string) { + a.mu.Lock() + defer a.mu.Unlock() + return slices.Clone(a.attachedNodes), slices.Clone(a.detachedNodes) +} + +// assertAssignedNode checks the actor's persisted assigned_node in the store +// against want. +func assertAssignedNode(t *testing.T, tc *testContext, name, step, want string) { + t.Helper() + actor, err := tc.persistence.GetActor(context.Background(), resources.ActorRef{Atespace: testAtespace, Name: name}) + if err != nil { + t.Fatalf("GetActor(%s) from store: %v", name, err) + } + if got := actor.GetStatus().GetAssignedNode(); got != want { + t.Errorf("persisted assigned_node %s = %q, want %q", step, got, want) + } +} + +// assertDetachedNodes checks the nodes the plugin detached from, in call +// order. An empty want means no detach. +func assertDetachedNodes(t *testing.T, plugin *attachFailVolumePlugin, step string, want ...string) { + t.Helper() + _, detached := plugin.calls() + if diff := cmp.Diff(want, detached, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("detached nodes %s (-want +got):\n%s", step, diff) + } +} + +// createSingleVolumeActor creates a template mounting one external volume, +// worker-1 on node1, and a SUSPENDED actor named name from the template. It +// returns worker-1's Worker name. +func createSingleVolumeActor(t *testing.T, tc *testContext, ns, name string) string { + t.Helper() + preview.InitForTest(t, "ExternalVolumes") + volumes := []*ateapipb.Volume{{ + Name: "vol1", + External: &ateapipb.ExternalVolumeSource{Template: &ateapipb.ExternalVolumeTemplate{ + StorageClassName: "standard", + CapacityBytes: 10 << 30, + }}, + }} + mounts := []*ateapipb.VolumeMount{{Name: "vol1", MountPath: "/mnt/vol1"}} + createTemplateWithVolumes(t, tc, ns, volumes, mounts) + workerName := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + + if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor failed: %v", err) + } + return workerName +} + +// resumeToRunningOnNode1 resumes the actor and checks that its volume was +// published to node1 and that the store records node1 as its assigned_node. +func resumeToRunningOnNode1(t *testing.T, tc *testContext, plugin *attachFailVolumePlugin, name string) { + t.Helper() + if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + }); err != nil { + t.Fatalf("ResumeActor failed: %v", err) + } + if attached, _ := plugin.calls(); !cmp.Equal(attached, []string{"node1"}) { + t.Fatalf("attached nodes after resume = %v, want [node1]", attached) + } + assertAssignedNode(t, tc, name, "after resume", "node1") +} + +// assertCrashedKeepingAssignedNode checks that the actor is CRASHED with no +// worker assignment, and that its status still records node as assigned_node. +func assertCrashedKeepingAssignedNode(t *testing.T, tc *testContext, name, node string) { + t.Helper() + actor, err := tc.persistence.GetActor(context.Background(), resources.ActorRef{Atespace: testAtespace, Name: name}) + if err != nil { + t.Fatalf("GetActor(%s) from store: %v", name, err) + } + if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_CRASHED { + t.Fatalf("state = %v, want CRASHED", got) + } + if got := actor.GetStatus().GetWorkerAssignment(); got != nil { + t.Fatalf("worker assignment = %v, want nil after the crash", got) + } + assertAssignedNode(t, tc, name, "after the crash", node) +} + +// TestDeleteActor_CrashedWorkerGone_DetachesAssignedNode verifies that +// deleting an actor that crashed because its worker went away unpublishes its +// volume from the node recorded in assigned_node, although the crash cleared +// the worker assignment and the worker no longer exists. +func TestDeleteActor_CrashedWorkerGone_DetachesAssignedNode(t *testing.T) { + ns := namespaceForTest("ns-del-crashed-detach") + plugin := &attachFailVolumePlugin{} + tc := setupTestWithVolumePlugins(t, ns, map[string]volume.VolumePluginControlPlane{ + "substrate.io/mock": plugin, + }) + defer tc.cleanup() + + const name = "crashed-delete-actor" + createSingleVolumeActor(t, tc, ns, name) + resumeToRunningOnNode1(t, tc, plugin, name) + + // Deregistering the worker crashes the actor it hosts. + deleteWorkerPod(t, tc, ns, "worker-1") + assertCrashedKeepingAssignedNode(t, tc, name, "node1") + assertDetachedNodes(t, plugin, "after the crash") + + // CRASHED is deletable without AnyState. + if _, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + }); err != nil { + t.Fatalf("DeleteActor failed: %v", err) + } + assertDetachedNodes(t, plugin, "after delete", "node1") + plugin.mu.Lock() + if len(plugin.deleted) != 1 { + t.Errorf("deleted volumes = %v, want exactly one", plugin.deleted) + } + plugin.mu.Unlock() +} + +// TestResumeActor_RestoreCrash_DeleteDetaches verifies that when Restore fails +// with an error a retry cannot clear after the volume was published, the +// crashed actor keeps assigned_node on record and a later delete unpublishes +// the volume from it. +func TestResumeActor_RestoreCrash_DeleteDetaches(t *testing.T) { + ns := namespaceForTest("ns-restore-crash-detach") + plugin := &attachFailVolumePlugin{} + tc := setupTestWithVolumePlugins(t, ns, map[string]volume.VolumePluginControlPlane{ + "substrate.io/mock": plugin, + }) + defer tc.cleanup() + + const name = "restore-crash-actor" + createSingleVolumeActor(t, tc, ns, name) + tc.fakeAtelet.FailRestore = status.Error(codes.Internal, "injected restore failure") + + _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + }) + if got := status.Code(err); got != codes.Internal { + t.Fatalf("ResumeActor code = %v, want %v (err: %v)", got, codes.Internal, err) + } + if attached, _ := plugin.calls(); !cmp.Equal(attached, []string{"node1"}) { + t.Fatalf("attached nodes = %v, want [node1]: the volume must be published before Restore fails", attached) + } + assertCrashedKeepingAssignedNode(t, tc, name, "node1") + assertDetachedNodes(t, plugin, "after the crash") + + if _, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + }); err != nil { + t.Fatalf("DeleteActor failed: %v", err) + } + assertDetachedNodes(t, plugin, "after delete", "node1") +} + +// TestSuspendActor_WorkerRowGone_StillDetaches verifies that a suspend +// unpublishes the volume from the node recorded on the actor even when the +// worker row the assignment names is already gone, and clears assigned_node +// afterwards. +func TestSuspendActor_WorkerRowGone_StillDetaches(t *testing.T) { + ns := namespaceForTest("ns-suspend-worker-gone") + plugin := &attachFailVolumePlugin{} + tc := setupTestWithVolumePlugins(t, ns, map[string]volume.VolumePluginControlPlane{ + "substrate.io/mock": plugin, + }) + defer tc.cleanup() + + const name = "worker-gone-actor" + workerName := createSingleVolumeActor(t, tc, ns, name) + resumeToRunningOnNode1(t, tc, plugin, name) + + // Remove the worker row directly, bypassing the DeleteWorker workflow, so + // the actor stays RUNNING with an assignment to a worker that no longer + // exists. + if _, err := tc.persistence.DeleteWorker(context.Background(), workerName, store.DeletePreconditions{}); err != nil { + t.Fatalf("DeleteWorker(%s) from store: %v", workerName, err) + } + + suspended, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, + }) + if err != nil { + t.Fatalf("SuspendActor failed: %v", err) + } + if got := suspended.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + t.Errorf("state = %v, want SUSPENDED", got) + } + assertDetachedNodes(t, plugin, "after suspend", "node1") + assertAssignedNode(t, tc, name, "after suspend", "") +} + +// TestRevertActor_Crashed_DetachesAssignedNode verifies that reverting an +// actor that crashed because its worker went away unpublishes its volume from +// the node recorded in assigned_node, clears assigned_node, and leaves the +// actor SUSPENDED. +func TestRevertActor_Crashed_DetachesAssignedNode(t *testing.T) { + ns := namespaceForTest("ns-revert-crashed-detach") + plugin := &attachFailVolumePlugin{} + tc := setupTestWithVolumePlugins(t, ns, map[string]volume.VolumePluginControlPlane{ + "substrate.io/mock": plugin, + }) + defer tc.cleanup() + + const name = "crashed-revert-actor" + createSingleVolumeActor(t, tc, ns, name) + resumeToRunningOnNode1(t, tc, plugin, name) + + deleteWorkerPod(t, tc, ns, "worker-1") + assertCrashedKeepingAssignedNode(t, tc, name, "node1") + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.TerminateCalled = false + tc.fakeAtelet.Lock.Unlock() + + reverted, err := tc.client.RevertActor(context.Background(), &ateapipb.RevertActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + }) + if err != nil { + t.Fatalf("RevertActor failed: %v", err) + } + if got := reverted.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + t.Errorf("state = %v, want SUSPENDED", got) + } + assertDetachedNodes(t, plugin, "after revert", "node1") + assertAssignedNode(t, tc, name, "after revert", "") + tc.fakeAtelet.Lock.Lock() + if !tc.fakeAtelet.TerminateCalled { + t.Errorf("expected Terminate call to clean up assigned node") + } else if gotUID := tc.fakeAtelet.TerminateRequest.GetWorkerPodUid(); gotUID != "" { + t.Errorf("TerminateRequest.WorkerPodUid = %q, want empty", gotUID) + } + tc.fakeAtelet.Lock.Unlock() +} + +// TestRevertActor_WorkerNoLongerHosts_Detaches verifies that reverting a +// RUNNING actor whose worker no longer hosts it skips the ateom workload +// termination (WorkerPodUid == "") but still unpublishes the volume from the +// node recorded on the actor and clears assigned_node. +func TestRevertActor_WorkerNoLongerHosts_Detaches(t *testing.T) { + ns := namespaceForTest("ns-revert-unhosted-detach") + plugin := &attachFailVolumePlugin{} + tc := setupTestWithVolumePlugins(t, ns, map[string]volume.VolumePluginControlPlane{ + "substrate.io/mock": plugin, + }) + defer tc.cleanup() + + const name = "unhosted-revert-actor" + workerName := createSingleVolumeActor(t, tc, ns, name) + resumeToRunningOnNode1(t, tc, plugin, name) + + actor, err := tc.persistence.GetActor(context.Background(), resources.ActorRef{Atespace: testAtespace, Name: name}) + if err != nil { + t.Fatalf("GetActor(%s) from store: %v", name, err) + } + // Drop the worker's assignment row only; the actor still names the worker. + if _, err := tc.persistence.ReleaseActorFromWorker(context.Background(), workerName, actor.GetMetadata().GetUid()); err != nil { + t.Fatalf("ReleaseActorFromWorker failed: %v", err) + } + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.TerminateCalled = false + tc.fakeAtelet.Lock.Unlock() + + reverted, err := tc.client.RevertActor(context.Background(), &ateapipb.RevertActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + }) + if err != nil { + t.Fatalf("RevertActor failed: %v", err) + } + if got := reverted.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + t.Errorf("state = %v, want SUSPENDED", got) + } + assertDetachedNodes(t, plugin, "after revert", "node1") + assertAssignedNode(t, tc, name, "after revert", "") + tc.fakeAtelet.Lock.Lock() + if !tc.fakeAtelet.TerminateCalled { + t.Errorf("expected Terminate call to clean up assigned node") + } else if gotUID := tc.fakeAtelet.TerminateRequest.GetWorkerPodUid(); gotUID != "" { + t.Errorf("TerminateRequest.WorkerPodUid = %q, want empty", gotUID) + } + tc.fakeAtelet.Lock.Unlock() +} diff --git a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go index f30691fc57..c8555135ca 100644 --- a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go @@ -573,6 +573,7 @@ func TestAssignWorkerAttempt_StampsWorkerEpoch(t *testing.T) { WorkerPool: "pool", WorkerPod: "pod-1", WorkerPodUid: name, + NodeName: "node-1", SandboxClass: "gvisor", Epoch: 1, Status: &ateapipb.WorkerStatus{ diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index 253e523be5..f667e0717e 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -477,7 +477,7 @@ func (w *ActorWorkflow) workerHoldingStaleClaim(ctx context.Context, actor *atea } // The Actor's allocation makes HasRoom unsuitable for an existing claim. - if w.appliesWithFallback(worker, constraints, relaxable) { + if worker.GetNodeName() != "" && w.appliesWithFallback(worker, constraints, relaxable) { return worker, nil } @@ -544,6 +544,10 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour constraints.RequiredRuntime = hardware.Requirement{} } + if assignedWorker.GetNodeName() == "" { + return nil, nil, apierror.FailedPrecondition("worker %s has no node name", assignedWorker.GetMetadata().GetName()) + } + assignment := &ateapipb.ActorAssignment{ Actor: &ateapipb.ObjectRef{ Atespace: actor.GetMetadata().GetAtespace(), @@ -745,8 +749,9 @@ func (w *ActorWorkflow) appliesWithFallback(worker *ateapipb.Worker, constraints // ensureVolumesAttached attaches the actor's mounted external volumes to the // assigned worker's node and returns each driver's attachment metadata by -// volume name, for the Restore call that follows. Attachment is idempotent, so -// a re-entered workflow safely runs it again. +// volume name, for the Restore call that follows. Every volume must be +// provisioned with a storage volume ID; otherwise nothing is attached. +// Attachment is idempotent, so a re-entered workflow safely runs it again. // TODO replace re-execution with a proper check on the volumes' attach state. func (w *ActorWorkflow) ensureVolumesAttached(ctx context.Context, actor *ateapipb.Actor, worker *ateapipb.Worker, actorTemplate *ateapipb.ActorTemplate) (_ map[string]map[string]string, err error) { ctx, done := stepSpan(ctx, "AttachVolumes") @@ -762,8 +767,14 @@ func (w *ActorWorkflow) ensureVolumesAttached(ctx context.Context, actor *ateapi } ref := &ateapipb.ObjectRef{Atespace: actor.GetMetadata().GetAtespace(), Name: actor.GetMetadata().GetName()} + volumes := getMountedActorVolumes(ctx, ref, actor.GetStatus().GetExternalVolumes().GetVolumes(), actorTemplate) + // Check every volume before attaching any, so a bad one cannot leave the + // ones before it published. + if err := checkVolumesAttachable(volumes); err != nil { + return nil, err + } volumePublishContexts := make(map[string]map[string]string) - for _, vol := range getMountedActorVolumes(ctx, ref, actor.GetStatus().GetExternalVolumes().GetVolumes(), actorTemplate) { + for _, vol := range volumes { slog.InfoContext(ctx, "Attaching volume to node", slog.String("volume_id", vol.GetStorageVolumeId()), slog.String("node", node)) plugin, err := w.pluginRegistry.GetPlugin(ctx, vol.GetVolumeType()) if err != nil { @@ -780,6 +791,17 @@ func (w *ActorWorkflow) ensureVolumesAttached(ctx context.Context, actor *ateapi return volumePublishContexts, nil } +// checkVolumesAttachable returns FailedPrecondition unless every volume has a +// storage volume ID. +func checkVolumesAttachable(volumes []*ateapipb.ExternalVolumeStatus) error { + for _, vol := range volumes { + if vol.GetStorageVolumeId() == "" { + return apierror.FailedPrecondition("volume %q has no storage volume ID to attach", vol.GetName()) + } + } + return nil +} + // ensureAteletRestored brings the workload up on the assigned worker: // restoring the actor's local snapshot when one exists, else its external // snapshot, else cold-booting from the template spec. This is the diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index fd651e99e1..2122bb0b07 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "net" + "strings" "sync" "testing" "time" @@ -267,6 +268,7 @@ func TestAssignWorkerAttempt_ReleasesIneligibleStaleWorker(t *testing.T) { WorkerPool: "pool-a", WorkerPod: "stale-pod", WorkerPodUid: testWorkerUID("stale-pod"), + NodeName: "node-1", SandboxClass: "gvisor", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, @@ -280,6 +282,7 @@ func TestAssignWorkerAttempt_ReleasesIneligibleStaleWorker(t *testing.T) { WorkerPool: "pool-b", WorkerPod: "free-pod", WorkerPodUid: testWorkerUID("free-pod"), + NodeName: "node-1", SandboxClass: "gvisor", Labels: map[string]string{"pool": "b"}, Status: &ateapipb.WorkerStatus{ @@ -341,6 +344,7 @@ func TestAssignWorkerAttempt_RetryAfterConflictPicksFreshWorker(t *testing.T) { WorkerPool: "pool", WorkerPod: "contested-pod", WorkerPodUid: testWorkerUID("contested-pod"), + NodeName: "node-1", SandboxClass: "gvisor", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, @@ -354,6 +358,7 @@ func TestAssignWorkerAttempt_RetryAfterConflictPicksFreshWorker(t *testing.T) { WorkerPool: "pool", WorkerPod: "fallback-pod", WorkerPodUid: testWorkerUID("fallback-pod"), + NodeName: "node-1", SandboxClass: "gvisor", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, @@ -449,6 +454,7 @@ func seedAssignFixture(t *testing.T, ctx context.Context, persistence store.Inte WorkerPool: "pool", WorkerPod: "pod-1", WorkerPodUid: testWorkerUID("pod-1"), + NodeName: "node-1", SandboxClass: "gvisor", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, @@ -2192,6 +2198,7 @@ func contendedWorker(podName string) *ateapipb.Worker { WorkerPool: "pool", WorkerPod: podName, WorkerPodUid: testWorkerUID(podName), + NodeName: "node-1", SandboxClass: "gvisor", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, @@ -2965,6 +2972,159 @@ func TestRequiredSnapshotRuntime(t *testing.T) { } } +// mockVolume returns a volume served by the "mock" plugin. +func mockVolume(name, storageID string) *ateapipb.ExternalVolumeStatus { + return &ateapipb.ExternalVolumeStatus{Name: name, StorageVolumeId: storageID, VolumeType: "mock"} +} + +// volumeTemplate returns a template that declares the volumes named in +// declared and mounts those named in mounted. +func volumeTemplate(declared []string, mounted ...string) *ateapipb.ActorTemplate { + tmpl := &ateapipb.ActorTemplate{ + Containers: []*ateapipb.Container{{Name: "main", Image: "img"}}, + } + for _, name := range declared { + tmpl.Volumes = append(tmpl.Volumes, &ateapipb.Volume{Name: name, External: &ateapipb.ExternalVolumeSource{Template: &ateapipb.ExternalVolumeTemplate{StorageClassName: "sc"}}}) + } + for _, name := range mounted { + tmpl.Containers[0].VolumeMounts = append(tmpl.Containers[0].VolumeMounts, &ateapipb.VolumeMount{Name: name, MountPath: "/" + name}) + } + return tmpl +} + +// seedVolumeAssignFixture stores one free gvisor worker on node and a +// SUSPENDED actor holding volumes, and returns the stored actor plus a +// started worker cache. +func seedVolumeAssignFixture(t *testing.T, ctx context.Context, persistence store.Interface, node string, volumes ...*ateapipb.ExternalVolumeStatus) (*ateapipb.Actor, *workercache.Cache) { + t.Helper() + if _, err := persistence.CreateWorker(ctx, &ateapipb.Worker{ + Metadata: &ateapipb.ResourceMetadata{Name: testWorkerUID("pod-1")}, + WorkerNamespace: "worker-ns", + WorkerPool: "pool", + WorkerPod: "pod-1", + WorkerPodUid: testWorkerUID("pod-1"), + NodeName: node, + SandboxClass: "gvisor", + Status: &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + DefaultRuntime: testSandboxRuntime(), + }, + }); err != nil { + t.Fatalf("CreateWorker: %v", err) + } + storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "id1"}, + Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, ExternalVolumes: &ateapipb.ExternalVolumes{Volumes: volumes}}, + }) + actor, err := persistence.GetActor(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}) + if err != nil { + t.Fatalf("GetActor: %v", err) + } + cacheCtx, cancel := context.WithCancel(ctx) + t.Cleanup(cancel) + wc := workercache.New(persistence, time.Minute) + if err := wc.Start(cacheCtx); err != nil { + t.Fatalf("workercache.Start: %v", err) + } + return actor, wc +} + +// newVolumeAssignWorkflow returns a workflow that schedules from wc and +// serves "mock" volumes with plugin. +func newVolumeAssignWorkflow(st store.Interface, wc *workercache.Cache, plugin volume.VolumePluginControlPlane) *ActorWorkflow { + return &ActorWorkflow{ + store: st, + workerCache: wc, + scheduler: scheduling.New(wc), + pluginRegistry: &mockPluginRegistry{plugins: map[string]volume.VolumePluginControlPlane{"mock": plugin}}, + } +} + +// TestResume_ChecksMountedVolumesBeforeAttach runs the assignment write and +// the attach step together: the assignment write records AssignedNode on the +// actor, and ensureVolumesAttached publishes only mounted volumes to it. A +// selected volume without a storage volume ID fails the attach step with +// FailedPrecondition before any publish call, even when a valid volume +// precedes it. +func TestResume_ChecksMountedVolumesBeforeAttach(t *testing.T) { + preview.InitForTest(t, "ExternalVolumes") + tests := []struct { + name string + // last is the second mounted volume, placed after every other volume. + last *ateapipb.ExternalVolumeStatus + // wantErr is a substring of the expected FailedPrecondition; empty + // means the attach succeeds. + wantErr string + }{{ + name: "all provisioned", + last: mockVolume("b", "id-b"), + }, { + name: "mounted volume without storage ID", + last: mockVolume("b", ""), + wantErr: `volume "b" has no storage volume ID`, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} + actor, wc := seedVolumeAssignFixture(t, ctx, persistence, "node-1", + mockVolume("a", "id-a"), + mockVolume("u", ""), + mockVolume("o", ""), + tt.last, + ) + plugin := &publishContextVolumePlugin{} + w := newVolumeAssignWorkflow(persistence, wc, plugin) + // u is declared but not mounted; o is not declared at all. + tmpl := volumeTemplate([]string{"a", "u", "b"}, "a", "b") + + assigned, worker, err := w.assignWorkerAttempt(ctx, actorRef, actor, tmpl) + if err != nil { + t.Fatalf("assignWorkerAttempt: %v", err) + } + if got := assigned.GetStatus().GetAssignedNode(); got != "node-1" { + t.Errorf("assigned actor AssignedNode = %q, want node-1", got) + } + _, attachErr := w.ensureVolumesAttached(ctx, assigned, worker, tmpl) + + nameByID := make(map[string]string) + for _, vol := range assigned.GetStatus().GetExternalVolumes().GetVolumes() { + nameByID[vol.GetStorageVolumeId()] = vol.GetName() + } + + if tt.wantErr != "" { + if got := apierror.Code(attachErr); got != codes.FailedPrecondition { + t.Fatalf("ensureVolumesAttached error = %v (code %v), want FailedPrecondition", attachErr, got) + } + if !strings.Contains(attachErr.Error(), tt.wantErr) { + t.Errorf("ensureVolumesAttached error = %q, want it to contain %q", attachErr, tt.wantErr) + } + if len(plugin.attachCalls) != 0 { + t.Errorf("attach calls = %v, want none", plugin.attachCalls) + } + return + } + if attachErr != nil { + t.Fatalf("ensureVolumesAttached: %v", attachErr) + } + var published []string + for _, call := range plugin.attachCalls { + if call.Node != "node-1" { + t.Errorf("attach of %q went to node %q, want node-1", call.VolumeID, call.Node) + } + published = append(published, nameByID[call.VolumeID]) + } + if diff := cmp.Diff([]string{"a", "b"}, published); diff != "" { + t.Errorf("published volumes mismatch (-want +got):\n%s", diff) + } + }) + } +} + +// TestCanRelaxRuntime pins when a resume may drop its runtime constraint: only +// when it has one, and never while a local pause checkpoint pins the actor. func TestCanRelaxRuntime(t *testing.T) { rt := testSandboxRuntime() memory := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY @@ -3136,3 +3296,51 @@ func TestToAteletSandboxRuntime(t *testing.T) { }) } } + +// TestAssignWorkerAttempt_WorkerWithoutNodeRejectsVolumes verifies a worker +// with no node name is not claimed: the attempt fails with FailedPrecondition +// before the bind, leaving no worker claim and no actor write. +func TestAssignWorkerAttempt_WorkerWithoutNodeRejectsVolumes(t *testing.T) { + tests := []struct { + name string + mounted []string + }{ + {name: "mounted volume", mounted: []string{"vol"}}, + {name: "unmounted volume", mounted: nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} + actor, wc := seedVolumeAssignFixture(t, ctx, persistence, "", + mockVolume("vol", "storage-vol"), + ) + plugin := &publishContextVolumePlugin{} + w := newVolumeAssignWorkflow(persistence, wc, plugin) + tmpl := volumeTemplate([]string{"vol"}, tt.mounted...) + + _, _, err := w.assignWorkerAttempt(ctx, actorRef, actor, tmpl) + stored, gerr := persistence.GetActor(ctx, actorRef) + if gerr != nil { + t.Fatalf("GetActor: %v", gerr) + } + claim := firstAssignment(t, persistence, testWorkerUID("pod-1")) + if got := apierror.Code(err); got != codes.FailedPrecondition { + t.Fatalf("assignWorkerAttempt error = %v (code %v), want FailedPrecondition", err, got) + } + if want := "has no node name"; !strings.Contains(err.Error(), want) { + t.Errorf("assignWorkerAttempt error = %q, want it to contain %q", err, want) + } + if claim != nil { + t.Errorf("worker claim = %v, want none", claim) + } + if diff := cmp.Diff(actor, stored, protocmp.Transform()); diff != "" { + t.Errorf("assignment wrote to the stored actor (-before +after):\n%s", diff) + } + if len(plugin.attachCalls) != 0 { + t.Errorf("attach calls = %v, want none", plugin.attachCalls) + } + }) + } +} diff --git a/internal/e2e/suites/combinedvolumes/combinedvolumes_test.go b/internal/e2e/suites/combinedvolumes/combinedvolumes_test.go index b494549a02..07ba9bb5dc 100644 --- a/internal/e2e/suites/combinedvolumes/combinedvolumes_test.go +++ b/internal/e2e/suites/combinedvolumes/combinedvolumes_test.go @@ -324,6 +324,33 @@ func requireSharedWrite(ctx context.Context, t *testing.T, router *e2e.RouterCli requireContentAtBoth(ctx, t, router, actorRef, writePath, aliasPath, probeWrittenContent) } +// requireAssignedNode fails unless the actor's status records want as its +// assigned_node. An empty want means no node is recorded. +func requireAssignedNode(t *testing.T, actor *ateapipb.Actor, want string) { + t.Helper() + + if got := actor.GetStatus().GetAssignedNode(); got != want { + t.Errorf("actor assigned_node = %q, want %q", got, want) + } +} + +// requireAssignedToWorkerNode fails unless the running actor's assigned_node +// matches the node of its worker assignment. It returns the assigned node. +func requireAssignedToWorkerNode(ctx context.Context, t *testing.T, clients *e2e.Clients, actorRef resources.ActorRef) string { + t.Helper() + + actor, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{Actor: actorRef.ToObjectRef()}) + if err != nil { + t.Fatalf("GetActor: %v", err) + } + node := actor.GetStatus().GetWorkerAssignment().GetNodeName() + if node == "" { + t.Fatalf("running actor's worker assignment names no node: %v", actor.GetStatus().GetWorkerAssignment()) + } + requireAssignedNode(t, actor, node) + return node +} + func TestCombinedVolumes(t *testing.T) { repo := os.Getenv("KO_DOCKER_REPO") if repo == "" { @@ -357,6 +384,7 @@ func TestCombinedVolumes(t *testing.T) { if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{Actor: actorRef.ToObjectRef()}); err != nil { t.Fatalf("ResumeActor: %v", err) } + requireAssignedToWorkerNode(ctx, t, clients, actorRef) router, err := e2e.NewRouterClient(t, ctx) if err != nil { @@ -404,9 +432,14 @@ func TestCombinedVolumes(t *testing.T) { }) t.Run("SurvivesSuspendResume", func(t *testing.T) { - if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef.ToObjectRef(), Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}); err != nil { + runningNode := requireAssignedToWorkerNode(ctx, t, clients, actorRef) + suspended, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef.ToObjectRef(), Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) + if err != nil { t.Fatalf("SuspendActor: %v", err) } + // Synchronous suspend preserves the actor's node binding while its + // local snapshot copy is retained. + requireAssignedNode(t, suspended.GetActor(), runningNode) // This suite checks volume restore, which can outlast the router's // request-parking budget on a busy node. Resume through the control @@ -424,6 +457,7 @@ func TestCombinedVolumes(t *testing.T) { t.Run("ExternalVolumeReattached", func(t *testing.T) { requireExternalVolume(t) requireContentAtBoth(resumeCtx, t, router, actorRef, extPathA+"/multi.txt", extPathB+"/multi.txt", probeWrittenContent) + requireAssignedToWorkerNode(ctx, t, clients, actorRef) }) }) @@ -443,6 +477,7 @@ func TestCombinedVolumes(t *testing.T) { if got := reverted.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Fatalf("state after revert = %v, want SUSPENDED", got) } + requireAssignedNode(t, reverted.GetActor(), "") resumeCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() From 94c91b21fb9ea068a986df4f44811f4d707dab6a Mon Sep 17 00:00:00 2001 From: Eren Akbulut Date: Tue, 20 Oct 2026 21:32:38 +0000 Subject: [PATCH 19/63] Revert and delete no longer hang when an actor's node is deleted (#2486) After a node is deleted, revert can still hang for a CRASHED actor or a paused actor whose latest upload is PENDING. Delete can hang for any actor still bound to that node. #2883 already lets revert and resume forget an unreachable node when the actor has no worker assignment or crash and the latest local snapshot has a COMPLETED durable copy. When no atelet is indexed on the assigned node, `ensureAteletTerminated` uses `NodeGone` to check the Node through the live Kubernetes API. It skips termination only on NotFound, after checking again that no atelet appeared. Existing nodes and lookup failures stay retryable. The API server gains `get` permission on nodes. External volumes still go through `DetachVolume`: nil and NotFound succeed, other errors propagate. Kubernetes Node deletion does not fence a machine that is still running; resources on such a machine may remain. Rebased onto main `872ab2077`. The whole-workflow test comment now identifies its paused actor's PENDING upload. Validation: - Targeted revert/delete/detach and `TestNodeGone` tests pass with `-race -count=3 -shuffle=on` and Docker-backed database tests enabled. - The full race suite passes all 15 ateapi packages with tests. Overall it reports 10,601 passing test/subtest events, 8 skipped, and two failures in `cmd/ateworker-microvm/internal/child`: `TestReapedWhenItDiesOnItsOwn` and `TestStop`. Both report `unsupported operation` on macOS and reproduce from unchanged main `872ab2077`; the non-Linux pidfd implementation returns `errors.ErrUnsupported`. - `hack/verify-all.sh` passes (using local Python 3.13 and clang-format 23.1.3). - A fresh independent review found no issues in the rebased patch. `git diff --check` passes. Earlier kind runs exercised real worker deletion, paused/crashed revert and delete, and the kept-node control. Those live runs were not repeated for this rebase. Current volume tests use a mock plugin; no real CSI-driver validation is claimed. Part of #2470. Resume without a completed latest upload remains separate. AI assistance: Claude Code helped write the original code and tests. Codex clarified the fixture comment, rebased the branch, reviewed the integration, and ran validation. --- cmd/ateapi/internal/controlapi/dialer.go | 29 +++ cmd/ateapi/internal/controlapi/dialer_test.go | 46 +++++ .../internal/controlapi/workflow_delete.go | 13 ++ .../controlapi/workflow_delete_test.go | 174 ++++++++++++++++++ cmd/ateapi/main.go | 7 +- manifests/ate-install/ate-api-server.yaml | 5 + 6 files changed, 273 insertions(+), 1 deletion(-) diff --git a/cmd/ateapi/internal/controlapi/dialer.go b/cmd/ateapi/internal/controlapi/dialer.go index 4b96a60f42..56fc7a4f1c 100644 --- a/cmd/ateapi/internal/controlapi/dialer.go +++ b/cmd/ateapi/internal/controlapi/dialer.go @@ -15,6 +15,7 @@ package controlapi import ( + "context" "crypto/tls" "crypto/x509" "errors" @@ -35,6 +36,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/tools/cache" "k8s.io/utils/lru" ) @@ -53,6 +55,9 @@ type ateletConn struct { // AteletDialer handles gRPC connections to Atelet pods. type AteletDialer struct { ateletIndexer cache.Indexer + // getNode, when set, reads a node from the API server. NodeGone uses it to + // tell a deleted node from an atelet that is only missing for now. + getNode func(ctx context.Context, nodeName string) error // mu makes the lookup, stale eviction and insert in DialForAteletOnNode // atomic, so concurrent callers cannot evict each other's fresh conn. mu sync.Mutex @@ -74,6 +79,30 @@ func WithDialCredentials(build func(expectedPodUID string) (credentials.Transpor return func(d *AteletDialer) { d.dialCredentials = build } } +// WithNodeGetter sets how NodeGone reads a node. getNode returns the API +// server's error, so a NotFound error means that the node is deleted. +func WithNodeGetter(getNode func(ctx context.Context, nodeName string) error) DialerOption { + return func(d *AteletDialer) { d.getNode = getNode } +} + +// NodeGone reports whether nodeName is deleted from the cluster and no atelet +// pod is known on it. It reads the node live, so a stale cache cannot report a +// node as gone. Without a node getter it always reports false. +func (d *AteletDialer) NodeGone(ctx context.Context, nodeName string) (bool, error) { + if d.getNode == nil { + return false, nil + } + if err := d.getNode(ctx, nodeName); !apierrors.IsNotFound(err) { + return false, err // nil when the node exists + } + // An atelet that registered while the node was read means the node is back. + atelets, err := d.ateletIndexer.ByIndex(byNode, nodeName) + if err != nil { + return false, fmt.Errorf("while finding atelet on node %q: %w", nodeName, err) + } + return len(atelets) == 0, nil +} + // NewAteletDialer creates a new AteletDialer. clientBundlePath and serverCAPath // are used to build the per-atelet mTLS credentials used for every atelet // connection, and ateletSPIFFEID is the identity those credentials expect on diff --git a/cmd/ateapi/internal/controlapi/dialer_test.go b/cmd/ateapi/internal/controlapi/dialer_test.go index 43215e7942..fa56a87f4d 100644 --- a/cmd/ateapi/internal/controlapi/dialer_test.go +++ b/cmd/ateapi/internal/controlapi/dialer_test.go @@ -15,6 +15,7 @@ package controlapi import ( + "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" @@ -24,6 +25,7 @@ import ( "errors" "math/big" "net/url" + "slices" "strconv" "sync" "testing" @@ -39,7 +41,9 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/cache" ) @@ -297,6 +301,48 @@ func TestVerifyAteletServerCert(t *testing.T) { }) } +// nodeGetter answers like the API server: the named nodes exist, others are +// NotFound, and lookupErr, when set, fails every read. +func nodeGetter(lookupErr error, names ...string) func(context.Context, string) error { + return func(_ context.Context, node string) error { + if lookupErr != nil { + return lookupErr + } + if slices.Contains(names, node) { + return nil + } + return apierrors.NewNotFound(schema.GroupResource{Resource: "nodes"}, node) + } +} + +func TestNodeGone(t *testing.T) { + spiffeID := installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace) + atelet := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ate-system", Name: "atelet-1"}, + Spec: corev1.PodSpec{NodeName: "node1"}, + } + unavailable := apierrors.NewServiceUnavailable("etcd is down") + for _, tc := range []struct { + name string + dialer *AteletDialer + wantGone bool + wantErr bool + }{ + {name: "node deleted", dialer: NewAteletDialer(newTestAteletIndexer(t), spiffeID, "", "", WithNodeGetter(nodeGetter(nil, "node2"))), wantGone: true}, + {name: "node exists", dialer: NewAteletDialer(newTestAteletIndexer(t), spiffeID, "", "", WithNodeGetter(nodeGetter(nil, "node1")))}, + {name: "lookup fails", dialer: NewAteletDialer(newTestAteletIndexer(t), spiffeID, "", "", WithNodeGetter(nodeGetter(unavailable))), wantErr: true}, + {name: "atelet registered meanwhile", dialer: NewAteletDialer(newTestAteletIndexer(t, atelet), spiffeID, "", "", WithNodeGetter(nodeGetter(nil)))}, + {name: "no node getter", dialer: NewAteletDialer(newTestAteletIndexer(t), spiffeID, "", "")}, + } { + t.Run(tc.name, func(t *testing.T) { + gone, err := tc.dialer.NodeGone(context.Background(), "node1") + if gone != tc.wantGone || (err != nil) != tc.wantErr { + t.Fatalf("NodeGone = %v, %v; want %v, error %v", gone, err, tc.wantGone, tc.wantErr) + } + }) + } +} + // newTestAteletIndexer builds an indexer with the production byNode index // shape, holding the given atelet pods. func newTestAteletIndexer(t *testing.T, pods ...*corev1.Pod) cache.Indexer { diff --git a/cmd/ateapi/internal/controlapi/workflow_delete.go b/cmd/ateapi/internal/controlapi/workflow_delete.go index 8317791a82..7529bd8853 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete.go @@ -154,6 +154,19 @@ func (w *ActorWorkflow) ensureAteletTerminated(ctx context.Context, actorRef res } conn, err := w.dialer.DialForAteletOnNode(nodeName) + if errors.Is(err, ErrNoAteletOnNode) { + gone, nodeErr := w.dialer.NodeGone(ctx, nodeName) + if nodeErr != nil { + return fmt.Errorf("while connecting to atelet on node %q: %w (and while checking the node: %w)", nodeName, err, nodeErr) + } + if gone { + // A deleted Node object counts as the node being gone for good. If the + // machine still runs, whatever atelet holds for the actor stays there. + slog.WarnContext(ctx, "assigned node no longer exists, skipping atelet terminate request", + slog.String("node", nodeName), slog.Any("actor", actorRef)) + return nil + } + } if err != nil { return fmt.Errorf("while connecting to atelet on node %q: %w", nodeName, err) } diff --git a/cmd/ateapi/internal/controlapi/workflow_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_delete_test.go index 4c6edeedd4..61a93e83e1 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete_test.go @@ -17,17 +17,21 @@ package controlapi import ( "context" "errors" + "slices" "strings" "testing" "go.ate.dev/substrate/cmd/ateapi/internal/store" "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/installdefaults" "go.ate.dev/substrate/internal/objectstore/objectstoretest" "go.ate.dev/substrate/internal/objectstoreplugin/objectstoreplugintest" "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/volume" "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func TestDeleteActorWorkflow_ExecutionPaths(t *testing.T) { @@ -92,6 +96,176 @@ func TestDeleteActorWorkflow_ExecutionPaths(t *testing.T) { } } +// A deleted Node object counts as the node being gone for good, so revert and +// delete do not wait for its atelet. A node that is still there may get its +// atelet back, and a failed lookup proves nothing, so both stay retryable. +func TestEnsureAteletTerminated_NodeGone(t *testing.T) { + actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} + actor := &ateapipb.Actor{Status: &ateapipb.ActorStatus{AssignedNode: "node1"}} + for _, tc := range []struct { + name string + nodes []string + lookupErr error + wantErr bool + }{ + {name: "node gone", nodes: []string{"node2"}}, + {name: "node still there", nodes: []string{"node1"}, wantErr: true}, + {name: "node lookup fails", lookupErr: errors.New("connection refused"), wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + w := newTestActorWorkflow(t, newTestPersistence(t), "team-a", "tmpl") + w.dialer = NewAteletDialer(newTestAteletIndexer(t), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "", + WithNodeGetter(nodeGetter(tc.lookupErr, tc.nodes...))) + err := w.ensureAteletTerminated(context.Background(), actorRef, actor, nil, "revert") + if tc.wantErr != (err != nil) { + t.Fatalf("ensureAteletTerminated = %v, want error %v", err, tc.wantErr) + } + if tc.wantErr && !errors.Is(err, ErrNoAteletOnNode) { + t.Fatalf("ensureAteletTerminated = %v, want ErrNoAteletOnNode", err) + } + if tc.lookupErr != nil && !strings.Contains(err.Error(), tc.lookupErr.Error()) { + t.Fatalf("ensureAteletTerminated = %v, want the node lookup error in it", err) + } + }) + } +} + +// detachDeletePlugin is a volume plugin for a whole delete: it records +// detaches like mockDetachVolumePlugin and accepts the volume delete. +type detachDeletePlugin struct{ mockDetachVolumePlugin } + +func (*detachDeletePlugin) DeleteVolume(context.Context, string) error { return nil } + +// TestRevertAndDelete_AssignedNodeGone runs the whole revert and delete +// workflows for a paused actor with a PENDING upload and a crashed actor whose +// assigned node has no atelet (#2470). If the Node object is gone, both finish, +// and revert clears assigned_node. If it still exists, both fail with +// ErrNoAteletOnNode and leave the actor REVERTING or DELETING for a retry. +// A volume detach on the gone node still runs: OK and NotFound from the plugin +// count as detached, and any other error fails the step as before. +func TestRevertAndDelete_AssignedNodeGone(t *testing.T) { + memory := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + completed := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + for _, tc := range []struct { + name string + state ateapipb.ActorState + paused bool + nodeKept bool + volume bool + detachErr error + wantErr error // ErrNoAteletOnNode for a kept node + // wantDetachErr expects the plugin's detach error back. + wantDetachErr bool + }{ + {name: "paused, node gone", state: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, paused: true}, + {name: "crashed, node gone", state: ateapipb.ActorState_ACTOR_STATE_CRASHED}, + {name: "paused, node kept", state: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, paused: true, nodeKept: true, wantErr: ErrNoAteletOnNode}, + {name: "crashed, node kept", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, nodeKept: true, wantErr: ErrNoAteletOnNode}, + {name: "node gone, volume detach OK", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, volume: true}, + {name: "node gone, volume detach NotFound", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, volume: true, detachErr: status.Error(codes.NotFound, "no such node")}, + {name: "node gone, volume detach fails", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, volume: true, detachErr: status.Error(codes.Internal, "instance not found"), wantDetachErr: true}, + } { + for _, verb := range []string{"revert", "delete"} { + t.Run(tc.name+"/"+verb, func(t *testing.T) { + ctx := context.Background() + st, cleanup := storetest.SetupTestStore(t) + defer cleanup() + w := newTestActorWorkflow(t, st, "ns", "tmpl1") + nodes := []string{"node-2"} + if tc.nodeKept { + nodes = append(nodes, "node-1") + } + w.dialer = NewAteletDialer(newTestAteletIndexer(t), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "", + WithNodeGetter(nodeGetter(nil, nodes...))) + plugin := &detachDeletePlugin{mockDetachVolumePlugin{detachErrs: map[string]error{"storage-vol-1": tc.detachErr}}} + w.pluginRegistry = &mockPluginRegistry{plugins: map[string]volume.VolumePluginControlPlane{"mock": plugin}} + tmpl := &ateapipb.ActorTemplate{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "ns", Name: "tmpl-vol"}, + SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://snapshots"}, + Volumes: []*ateapipb.Volume{{Name: "vol1"}}, + Containers: []*ateapipb.Container{{VolumeMounts: []*ateapipb.VolumeMount{{Name: "vol1"}}}}, + } + if _, err := st.CreateActorTemplate(ctx, tmpl); err != nil { + t.Fatalf("create ActorTemplate: %v", err) + } + + actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} + seedWorkflowActor(t, ctx, st, actorRef, "ns", "tmpl-vol", tc.state) + actor, err := st.GetActor(ctx, actorRef) + if err != nil { + t.Fatalf("get seeded actor: %v", err) + } + mustUpdateActorStatus(t, ctx, st, actor, func(s *ateapipb.ActorStatus) { + s.AssignedNode = "node-1" + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", "kept", + mustActorSnapshotURI(t, tmpl, actor, "kept").String(), completed)} + if tc.paused { + local := newPausedSnapshot(t, 2, memory, "", "local", "node-1") + local.DurableSnapshot.Object.SnapshotUri = mustActorSnapshotURI(t, tmpl, actor, "local").String() + s.LastAssignedGeneration = 2 + s.Snapshots = append(s.Snapshots, local) + } + if tc.volume { + s.ExternalVolumes = &ateapipb.ExternalVolumes{Volumes: []*ateapipb.ExternalVolumeStatus{ + {Name: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}}} + } + }) + + wantState := ateapipb.ActorState_ACTOR_STATE_REVERTING + if verb == "revert" { + _, err = w.RevertActor(ctx, actorRef) + } else { + wantState = ateapipb.ActorState_ACTOR_STATE_DELETING + _, err = w.DeleteActor(ctx, actorRef, store.DeletePreconditions{}) + } + switch { + case tc.wantDetachErr: + if err == nil || !strings.Contains(err.Error(), "instance not found") { + t.Fatalf("%s = %v, want the detach error", verb, err) + } + case tc.wantErr != nil: + if !errors.Is(err, tc.wantErr) { + t.Fatalf("%s = %v, want %v", verb, err, tc.wantErr) + } + case err != nil: + t.Fatalf("%s: %v", verb, err) + } + if tc.volume { + if want := []detachCall{{VolumeID: "storage-vol-1", Node: "node-1"}}; !slices.Equal(plugin.detachCalls, want) { + t.Errorf("detach calls = %v, want %v", plugin.detachCalls, want) + } + } + + got, getErr := st.GetActor(ctx, actorRef) + switch { + case tc.wantErr != nil || tc.wantDetachErr: + if getErr != nil { + t.Fatalf("get actor after failed %s: %v", verb, getErr) + } + if s := got.GetStatus(); s.GetState() != wantState || s.GetAssignedNode() != "node-1" { + t.Errorf("after failed %s: state %v, assigned node %q, want %v on node-1", verb, s.GetState(), s.GetAssignedNode(), wantState) + } + case verb == "revert": + if getErr != nil { + t.Fatalf("get actor after revert: %v", getErr) + } + if s := got.GetStatus(); s.GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED || s.GetAssignedNode() != "" { + t.Errorf("after revert: state %v, assigned node %q, want SUSPENDED with no node", s.GetState(), s.GetAssignedNode()) + } + if snaps := got.GetStatus().GetSnapshots(); len(snaps) != 1 || snaps[0].GetGeneration() != 1 || snaps[0].GetDurableSnapshot().GetStatus() != completed { + t.Errorf("after revert: snapshots %v, want only the completed durable snapshot at generation 1", snaps) + } + default: + if !errors.Is(getErr, store.ErrNotFound) { + t.Errorf("get actor after delete = %v, want ErrNotFound", getErr) + } + } + }) + } + } +} + // TestEnsureMarkedDeleting_AllStates checks that every actor state can be // marked DELETING. func TestEnsureMarkedDeleting_AllStates(t *testing.T) { diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 4f854b7fc4..4ae67ab320 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -59,6 +59,7 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/reflection" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" @@ -325,7 +326,11 @@ func main() { } volPlugins := make(map[string]volume.VolumePluginControlPlane) - ateletDialer := controlapi.NewAteletDialer(ateletPodInformer.GetIndexer(), ateletSPIFFEID, *ateletClientCredBundle, *podIdentityCACerts) + ateletDialer := controlapi.NewAteletDialer(ateletPodInformer.GetIndexer(), ateletSPIFFEID, *ateletClientCredBundle, *podIdentityCACerts, + controlapi.WithNodeGetter(func(ctx context.Context, nodeName string) error { + _, err := clientset.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + return err + })) actorIDCAPool, err := localca.NewRefreshingPool(*actorIDCAPoolFile) if err != nil { diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index 11b4e6bcbd..92ff9cff30 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -22,6 +22,11 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["get", "watch", "list"] +# Tells a deleted node from a missing atelet, so revert and delete of an actor +# on a node that is gone do not wait for that atelet forever. +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] - apiGroups: ["ate.dev"] resources: ["workerpools", "csidriverconfigs"] verbs: ["get", "watch", "list"] From 2a2d4a84706e59f94e485812f24053a9bafecd79 Mon Sep 17 00:00:00 2001 From: Luiz Oliveira Date: Tue, 20 Oct 2026 21:46:22 +0000 Subject: [PATCH 20/63] Do not crash DELETING actors (#3004) * If DeleteActor fails midway, it leaves the actor in DELETING state. * If the worker pod goes away, or the worker restarts, before the retry, crashBoundActor moves the actor to CRASHED. * RevertActor accepts CRASHED, so someone can revert the actor and resume it. This is not safe, because there's no guarantee that a DELETING actor can be recovered (volumes may have already been detached, or snapshots cleared). _____________________________ This also addresses https://github.com/agent-substrate/substrate/pull/1981#discussion_r4186140245 --- .../controlapi/functionaltest/actor_test.go | 63 +++++++++++++++++++ .../workflow_reconcile_assignments_test.go | 1 + .../controlapi/workflow_worker_delete.go | 15 +++-- .../controlapi/workflow_worker_delete_test.go | 7 ++- 4 files changed, 79 insertions(+), 7 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index ee43b6c876..a8b013e724 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -1381,6 +1381,69 @@ func TestDeleteActor_Crashed(t *testing.T) { assertGrpcError(t, err, codes.NotFound, "Actor test-atespace/id1 not found") } +// A delete that fails partway leaves the actor DELETING. Losing its worker +// before the retry must keep it DELETING, since a CRASHED actor can be +// reverted and resumed, and the retry must still finish the delete. +func TestDeleteActor_RetryAfterWorkerDelete(t *testing.T) { + ns := namespaceForTest("ns-delete-retry-worker-gone") + tc := setupTest(t, ns) + defer tc.cleanup() + + createTemplate(t, tc, ns) + createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + + ctx := context.Background() + actorRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"} + if _, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor failed: %v", err) + } + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { + t.Fatalf("ResumeActor failed: %v", err) + } + + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.FailTerminate = status.Error(codes.Unavailable, "node unavailable") + tc.fakeAtelet.Lock.Unlock() + if _, err := tc.client.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: actorRef}); err == nil { + t.Fatal("DeleteActor succeeded, want it to fail on Terminate") + } + + // This would crash a RUNNING Actor. But, should keep a DELETING actor + // unchanged. + deleteWorkerPod(t, tc, ns, "worker-1") + actor, err := tc.client.GetActor(ctx, &ateapipb.GetActorRequest{Actor: actorRef}) + if err != nil { + t.Fatalf("GetActor failed: %v", err) + } + if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_DELETING { + t.Fatalf("state after worker delete = %v, want DELETING", got) + } + + // Retry deleting the actor after its worker is gone. + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.FailTerminate = nil + tc.fakeAtelet.TerminateRequest = nil + tc.fakeAtelet.Lock.Unlock() + if _, err := tc.client.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: actorRef}); err != nil { + t.Fatalf("retried DeleteActor failed: %v", err) + } + + // The worker is gone, so the retry must not name its pod. + tc.fakeAtelet.Lock.Lock() + terminate := tc.fakeAtelet.TerminateRequest + tc.fakeAtelet.Lock.Unlock() + if terminate == nil { + t.Error("retried DeleteActor did not call atelet Terminate") + } else if gotUID := terminate.GetWorkerPodUid(); gotUID != "" { + t.Errorf("TerminateRequest.WorkerPodUid = %q, want empty", gotUID) + } + _, err = tc.client.GetActor(ctx, &ateapipb.GetActorRequest{Actor: actorRef}) + assertGrpcError(t, err, codes.NotFound, "Actor test-atespace/id1 not found") +} + func TestDeleteActor_NotFound(t *testing.T) { ns := namespaceForTest("ns-delete-notfound") tc := setupTest(t, ns) diff --git a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go index c8555135ca..00e1a61149 100644 --- a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go @@ -304,6 +304,7 @@ func TestReconcileAssignments_ReleasesByActorState(t *testing.T) { {name: "running elsewhere is released", state: ateapipb.ActorState_ACTOR_STATE_RUNNING, pointsAt: otherWorker, wantState: ateapipb.ActorState_ACTOR_STATE_RUNNING, wantRelease: true}, {name: "suspended elsewhere is released", state: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, pointsAt: otherWorker, wantState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, wantRelease: true}, {name: "resuming crashes", state: ateapipb.ActorState_ACTOR_STATE_RESUMING, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantRelease: true}, + {name: "deleting stays deleting", state: ateapipb.ActorState_ACTOR_STATE_DELETING, wantState: ateapipb.ActorState_ACTOR_STATE_DELETING, wantRelease: true}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go index 5aeec6555d..b26f9698b2 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go @@ -198,10 +198,19 @@ func (w *WorkerWorkflow) releaseBoundActor(ctx context.Context, worker *ateapipb // crashBoundActor moves an Actor that lost its sandbox on the Worker to // ACTOR_STATE_CRASHED and clears its pointers at the Worker. crashMsg is -// recorded in the Actor's status. The Actor's assignment row in the worker is -// left for the caller to release. +// recorded in the Actor's status. An Actor in ACTOR_STATE_DELETING is left +// untouched. The Actor's assignment row in the worker is left for the caller to +// release. // A concurrent write to the Actor fails this as ABORTED. func (w *WorkerWorkflow) crashBoundActor(ctx context.Context, worker *ateapipb.Worker, actorRef resources.ActorRef, actor *ateapipb.Actor, logMsg, crashMsg string) error { + // DELETING is final. A CRASHED actor can be reverted and resumed, and the + // delete may already have collected its volumes and snapshots. The delete's + // retry clears the actor's pointers. + if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_DELETING { + markSkipped(ctx, "actor is DELETING") + return nil + } + name := worker.GetMetadata().GetName() opName := ateattr.OperationUnknown switch actor.GetStatus().GetState() { @@ -211,8 +220,6 @@ func (w *WorkerWorkflow) crashBoundActor(ctx context.Context, worker *ateapipb.W opName = ateattr.OperationSuspend case ateapipb.ActorState_ACTOR_STATE_REVERTING: opName = ateattr.OperationRevert - case ateapipb.ActorState_ACTOR_STATE_DELETING: - opName = ateattr.OperationDelete } wasAlreadyCrashed := actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_CRASHED diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go index 9db03ecf27..efeaaa3bfa 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go @@ -147,8 +147,9 @@ func TestDeleteWorkerWorkflow_ReleasesBoundActor(t *testing.T) { // The Actor's state when its pod vanished decides what the release does: one // that had already suspended saved its state cleanly and stays resumable, while -// anything still in flight crashes and is counted. An Actor already counted as -// crashed is released again without being counted twice. +// anything still in flight crashes and is counted. An Actor being deleted stays +// DELETING so it can never be resumed. An Actor already counted as crashed is +// released again without being counted twice. func TestDeleteWorkerWorkflow_ReleasedActorStateTransitions(t *testing.T) { tests := []struct { name string @@ -161,7 +162,7 @@ func TestDeleteWorkerWorkflow_ReleasedActorStateTransitions(t *testing.T) { {name: "resuming becomes crashed", start: ateapipb.ActorState_ACTOR_STATE_RESUMING, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantOp: ateattr.OperationResume, wantMetric: true}, {name: "suspending becomes crashed", start: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantOp: ateattr.OperationSuspend, wantMetric: true}, {name: "reverting becomes crashed", start: ateapipb.ActorState_ACTOR_STATE_REVERTING, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantOp: ateattr.OperationRevert, wantMetric: true}, - {name: "deleting becomes crashed", start: ateapipb.ActorState_ACTOR_STATE_DELETING, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantOp: ateattr.OperationDelete, wantMetric: true}, + {name: "deleting stays deleting", start: ateapipb.ActorState_ACTOR_STATE_DELETING, wantState: ateapipb.ActorState_ACTOR_STATE_DELETING, wantMetric: false}, {name: "suspended stays suspended", start: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, wantState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, wantMetric: false}, {name: "crashed is not counted twice", start: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantMetric: false}, } From 5f3fe1cb07e256aaab92c2910de187456cd72dd9 Mon Sep 17 00:00:00 2001 From: Max Thompson Date: Tue, 20 Oct 2026 21:47:16 +0000 Subject: [PATCH 21/63] ate-api-server accepts actor JWT issuers with a path (#2781) Part of #1756. - ate-api-server's `--actor-jwt-issuer` again accepts any OpenID issuer identifier: https, a host, and no user info, query, or fragment. A path is allowed. ate-api-server only writes the value into `iss`, so it shouldn't depend on where the discovery document is served. - The `https://` rule stays in ate-idp-server and in ate-setup, which deploys ate-idp-server with every install. ate-idp-server serves the discovery document and key set at fixed root paths, so it only works with an issuer that has no path. - The flag's help text says the same. For reviewers: installs through ate-setup behave as before. Only an operator who runs ate-api-server from their own manifests, and serves the documents somewhere else, can now use an issuer with a path. Testing: unit tests for the issuer check. I didn't run e2e, since nothing an install deploys changes. - [ ] Tests pass - [ ] Appropriate changes to documentation are included in the PR --- cmd/ateapi/main.go | 26 +++++++++++++++++++++++--- cmd/ateapi/main_test.go | 6 +++++- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 4ae67ab320..de75d6126a 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -21,9 +21,11 @@ import ( "fmt" "log/slog" "net" + "net/url" "os" "os/signal" "strconv" + "strings" "syscall" "time" @@ -36,7 +38,6 @@ import ( "go.ate.dev/substrate/cmd/ateapi/internal/store/atepg" "go.ate.dev/substrate/cmd/ateapi/internal/workercache" "go.ate.dev/substrate/cmd/ateapi/internal/workerservice" - "go.ate.dev/substrate/internal/actoridjwt" "go.ate.dev/substrate/internal/ateinterceptors" "go.ate.dev/substrate/internal/credbundle" "go.ate.dev/substrate/internal/installdefaults" @@ -98,7 +99,7 @@ var ( apiServerSidecarSPIFFEID = pflag.String("api-server-sidecar-spiffe-id", installdefaults.APIServerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of the ate-api-server pod, which its sidecars share. Holds the api_server_sidecar role, which reads actors so the objectstore control plugin can scope their storage credentials.") actorIDJWTPoolFile = pflag.String("actor-id-jwt-pool", "", "The file that contains the serialized JWT authority pool for signing actor JWTs") - actorJWTIssuer = pflag.String("actor-jwt-issuer", "", "Issuer URL placed in the iss claim of actor JWTs. Relying parties fetch /.well-known/openid-configuration to verify them. Must be https://, with no path. Required.") + actorJWTIssuer = pflag.String("actor-jwt-issuer", "", "Issuer URL placed in the iss claim of actor JWTs. Relying parties fetch /.well-known/openid-configuration to verify them. Must be https with no query or fragment. Required.") defaultEgressGatewayAddress = pflag.String("default-egress-gateway-address", "", "Default address (host:port) of the egress PEP that each actor's atunnel dials. Sent on every atelet Run and Restore, so it takes effect at the actor's next activation. Empty leaves actors with no TCP egress.") actorIDCAPoolFile = pflag.String("actor-id-ca-pool", "", "The file that contains the CA pool for signing actor JWTs") @@ -687,9 +688,28 @@ func buildJWTProviders(ctx context.Context, cfg *apiauthn.AuthenticationConfig) return serverCfg, nil } +// validateActorJWTIssuer checks that issuer is an OpenID issuer identifier: an +// https URL with a host and no user info, query, or fragment. A path is +// allowed. func validateActorJWTIssuer(issuer string) error { if issuer == "" { return errors.New("required") } - return actoridjwt.ValidateIssuer(issuer) + u, err := url.Parse(issuer) + if err != nil { + return fmt.Errorf("invalid issuer: %w", err) + } + switch { + case u.Scheme != "https": + return fmt.Errorf("issuer %q must use the https scheme", issuer) + case u.Hostname() == "": + return fmt.Errorf("issuer %q has no host", issuer) + case u.User != nil: + return fmt.Errorf("issuer %q must not contain user info", issuer) + case strings.Contains(issuer, "?"): + return fmt.Errorf("issuer %q must not contain a query", issuer) + case strings.Contains(issuer, "#"): + return fmt.Errorf("issuer %q must not contain a fragment", issuer) + } + return nil } diff --git a/cmd/ateapi/main_test.go b/cmd/ateapi/main_test.go index 8ab0193c6d..1a4c1ada12 100644 --- a/cmd/ateapi/main_test.go +++ b/cmd/ateapi/main_test.go @@ -105,7 +105,11 @@ func TestValidateActorJWTIssuer(t *testing.T) { {issuer: "", wantErr: true}, {issuer: "http://idp.example.com", wantErr: true}, {issuer: "https://idp.ate-system.svc"}, - {issuer: "https://idp.example.com/prod", wantErr: true}, + {issuer: "https://idp.example.com/prod"}, + {issuer: "https:///prod", wantErr: true}, + {issuer: "https://user@idp.example.com", wantErr: true}, + {issuer: "https://idp.example.com?env=prod", wantErr: true}, + {issuer: "https://idp.example.com#prod", wantErr: true}, } { err := validateActorJWTIssuer(tt.issuer) if gotErr := err != nil; gotErr != tt.wantErr { From 0a262402052040c6e2d092ef3694075715b976a8 Mon Sep 17 00:00:00 2001 From: Sairaj Pokale Date: Tue, 20 Oct 2026 22:41:20 +0000 Subject: [PATCH 22/63] fix(benchmarking): use ate-setup subcommands in install-ate.sh invocations (#3007) #2718 removed the legacy `--deploy-ate-system`, `--delete-ate-system`, and `--setup-csi=nfs` flag-translation shim from `hack/install-ate.sh` and `hack/install-ate-kind.sh`, forwarding arguments directly to `go run ./cmd/ate-setup "$@"`. Update the remaining callers in `benchmarking/` and `hack/run-microvm-demo.sh` to pass the `ate-setup` subcommands (`deploy ate-system`, `delete ate-system`, `setup csi nfs`) instead of the removed flags. Fixes #3006 - [x] Tests pass - [x] Docs updated --- benchmarking/README.md | 12 +++++------- benchmarking/automation/README.md | 4 ++-- benchmarking/automation/orchestrator.py | 8 ++++---- benchmarking/automation/test_orchestrator.py | 7 ++++++- benchmarking/automation/tests.yaml | 4 ++-- benchmarking/nighthawk-egress/README.md | 6 +++--- benchmarking/nighthawk-ingress/README.md | 4 ++-- benchmarking/nighthawk-ingress/run-dev.sh | 2 +- hack/run-microvm-demo.sh | 4 ++-- 9 files changed, 27 insertions(+), 24 deletions(-) diff --git a/benchmarking/README.md b/benchmarking/README.md index 92238a7141..553597aec2 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -28,7 +28,7 @@ scenario ladder, read [observability.md](observability.md). > benchmark call is denied: > ```bash > ATE_API_AUTHZ_BOOTSTRAP_OWNERS=system:serviceaccount:ate-system:ate-client,spiffe://cluster.local/ns/benchmarking/sa/default \ -> ./hack/install-ate.sh --deploy-ate-system +> ./hack/install-ate.sh deploy ate-system > ``` > The nightly automation sets this itself; see > [automation/README.md](automation/README.md). @@ -74,8 +74,6 @@ go run ./cmd/ate-setup delete benchmarks `ate-setup` reads these settings in addition to those in [docs/operator-install.md](../docs/operator-install.md#settings), through the same flag, environment variable, and configuration document channels. -`hack/install-ate.sh` forwards them as `--benchmark-worker-count`, -`--benchmark-sandbox-class`, and `--benchmark-actor-memory`. | Key | Environment variable | Flag | Default | Applies to | |--------------------------|-------------------------------|----------------------------|----------|------------------------------------------| @@ -154,7 +152,7 @@ and state restoration latency when a durable directory is attached to the actor. * `glutton-storage`: Attaches an external CSI volume instead of a durable directory. `deploy_locust.sh` and `workloads/deploy.sh` deploy it only when `--storage-class-name` names an existing StorageClass. To run it on - NFS, install with `hack/install-ate.sh --setup-csi=nfs` and pass + NFS, install with `hack/install-ate.sh setup csi nfs` and pass `--storage-class-name csi-nfs-sc`, the class the e2e tests also use. #### DurDir Reported Metrics @@ -597,7 +595,7 @@ limits of ate-api-server and Postgres. Isolate mode takes the nodes out by installing substrate with a fake data plane: ```bash -hack/install-ate.sh --deploy-ate-system --fake-data-plane +hack/install-ate.sh deploy ate-system --fake-data-plane ./benchmarking/workloads/deploy.sh --deploy --worker-count 200 \ --fake-atelet-env FAKE_ATELET_DELAY=500ms ``` @@ -636,8 +634,8 @@ Fake actors need neither. fake-atelet takes the storage settings of atelet's objectstore-plugin sidecar, whose writes it takes over, so its snapshot placeholders go to the storage the install points atelet at. The install refuses a cluster whose WorkerPools still have worker pods, which would -outlive ate-controller. A normal `hack/install-ate.sh --deploy-ate-system` -puts the real data plane back, and `--delete-ate-system` removes the fakes +outlive ate-controller. A normal `hack/install-ate.sh deploy ate-system` +puts the real data plane back, and `delete ate-system` removes the fakes either way. Never use the mode on a cluster that serves real actors: actors placed on fake Workers do not exist. diff --git a/benchmarking/automation/README.md b/benchmarking/automation/README.md index 3849e6ce53..daf3ffec82 100644 --- a/benchmarking/automation/README.md +++ b/benchmarking/automation/README.md @@ -18,7 +18,7 @@ the router capacity benchmark — see 4. `docker build && docker push` builds the runner image for each test type in use, tagged with the commit hash: `${KO_DOCKER_REPO}/locust-test:` and/or `${KO_DOCKER_REPO}/nighthawk-ingress-test:`. -5. `hack/install-ate.sh --deploy-ate-system` (with `ATE_CREDENTIAL_PROVIDER` +5. `hack/install-ate.sh deploy ate-system` (with `ATE_CREDENTIAL_PROVIDER` in `.ate-dev-env.sh`, or `--credential-provider` in the test's `ateArgs`) + `benchmarking/workloads/deploy.sh --deploy --sandbox-class ` (these build & push substrate / workload @@ -144,7 +144,7 @@ principal://iam.googleapis.com/projects//locations/global/worklo - `roles/container.admin` on the test cluster's project — required to manage cluster-scoped resources (CRDs, ClusterRoles, ClusterRoleBindings, - Namespaces) that `hack/install-ate.sh --deploy-ate-system` creates. + Namespaces) that `hack/install-ate.sh deploy ate-system` creates. `container.developer` is not enough — it intentionally omits the `container.clusterRoles.*` and `container.customResourceDefinitions.*` permissions. diff --git a/benchmarking/automation/orchestrator.py b/benchmarking/automation/orchestrator.py index 97af6142dc..03a465d402 100644 --- a/benchmarking/automation/orchestrator.py +++ b/benchmarking/automation/orchestrator.py @@ -390,9 +390,9 @@ def validate_and_normalize_tests(tests: list[dict[str, Any]]) -> None: def substrate_args(test: dict[str, Any]) -> list[str]: - """Return the hack/install-ate.sh --deploy-ate-system arguments for a + """Return the hack/install-ate.sh deploy ate-system arguments for a tests.yaml entry: its ateArgs, then the fake data plane flag its isolate - block asks for. The teardown needs no matching flag: --delete-ate-system + block asks for. The teardown needs no matching flag: delete ate-system removes the fake data plane whether or not it was installed.""" return [*test.get("ateArgs", []), *isolate_install_args(test)] @@ -411,11 +411,11 @@ def deploy_substrate(ate_args: Iterable[str] = ()) -> None: **os.environ, "ATE_API_AUTHZ_BOOTSTRAP_OWNERS": f"{CLIENT_PRINCIPAL},{BENCHMARK_RUNNER_PRINCIPAL}", } - run(["hack/install-ate.sh", "--deploy-ate-system", *(str(a) for a in ate_args)], env=env) + run(["hack/install-ate.sh", "deploy", "ate-system", *(str(a) for a in ate_args)], env=env) def teardown_substrate() -> None: - run_no_check(["hack/install-ate.sh", "--delete-ate-system"]) + run_no_check(["hack/install-ate.sh", "delete", "ate-system"]) def install_microvm_deps() -> None: diff --git a/benchmarking/automation/test_orchestrator.py b/benchmarking/automation/test_orchestrator.py index aef06afed9..67ede8f23f 100644 --- a/benchmarking/automation/test_orchestrator.py +++ b/benchmarking/automation/test_orchestrator.py @@ -67,7 +67,7 @@ class DeploySubstrateTest(unittest.TestCase): def test_runner_and_client_are_bootstrap_owners(self, run): orchestrator.deploy_substrate(["--credential-provider={}"]) cmd = run.call_args.args[0] - self.assertEqual(cmd, ["hack/install-ate.sh", "--deploy-ate-system", "--credential-provider={}"]) + self.assertEqual(cmd, ["hack/install-ate.sh", "deploy", "ate-system", "--credential-provider={}"]) owners = run.call_args.kwargs["env"]["ATE_API_AUTHZ_BOOTSTRAP_OWNERS"].split(",") self.assertEqual( owners, @@ -83,6 +83,11 @@ def test_keeps_the_rest_of_the_environment(self, run): orchestrator.deploy_substrate() self.assertEqual(run.call_args.kwargs["env"]["KO_DOCKER_REPO"], "example") + @mock.patch("orchestrator.run_no_check") + def test_teardown_substrate(self, run_no_check): + orchestrator.teardown_substrate() + run_no_check.assert_called_once_with(["hack/install-ate.sh", "delete", "ate-system"]) + class DeployWorkloadsTest(unittest.TestCase): @mock.patch("orchestrator.run") diff --git a/benchmarking/automation/tests.yaml b/benchmarking/automation/tests.yaml index 6bc63d6495..efd3bcf77d 100644 --- a/benchmarking/automation/tests.yaml +++ b/benchmarking/automation/tests.yaml @@ -52,14 +52,14 @@ # large workerCount fleets whose image pulls or scheduling # outrun the default. # ateArgs Optional. List of strings appended verbatim to -# `hack/install-ate.sh --deploy-ate-system` when this +# `hack/install-ate.sh deploy ate-system` when this # test's substrate is deployed (e.g. ["--atenet-dataplane=agentgateway"]). # The deploy requires --credential-provider; set it here # as one YAML string, '--credential-provider={"name":"k8s.io"}' # (no shell quoting: the args are passed without a shell), # or as ATE_CREDENTIAL_PROVIDER in .ate-dev-env.sh. # isolate Optional. Isolate mode: installs substrate with the fake -# data plane (hack/install-ate.sh --fake-data-plane), which +# data plane (hack/install-ate.sh deploy ate-system --fake-data-plane), which # serves the WorkerPool without worker pods: fake-atelet # takes atelet's place on every atelet node and answers each # lifecycle call after a delay, and fake-workersync honors diff --git a/benchmarking/nighthawk-egress/README.md b/benchmarking/nighthawk-egress/README.md index a0796bae9f..6ce6a76c62 100644 --- a/benchmarking/nighthawk-egress/README.md +++ b/benchmarking/nighthawk-egress/README.md @@ -36,15 +36,15 @@ variable when deploying ATE. 1. **Cluster with Agent Substrate deployed**: - **Existing cluster** (with `.ate-dev-env.sh` configured): ```bash - hack/install-ate.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' + hack/install-ate.sh deploy ate-system --credential-provider='{"name":"k8s.io"}' ``` - **Local `kind` cluster**: ```bash - hack/install-ate-kind.sh --deploy-ate-system --credential-provider='{"enabled":false}' + hack/install-ate-kind.sh deploy ate-system --credential-provider='{"enabled":false}' ``` - To set the `atenet-egress` Envoy worker thread count (`--concurrency`), set `E2E_ATE_ENVOY_CONCURRENCY` when deploying: ```bash - E2E_ATE_ENVOY_CONCURRENCY=1 hack/install-ate-kind.sh --deploy-ate-system --credential-provider='{"enabled":false}' + E2E_ATE_ENVOY_CONCURRENCY=1 hack/install-ate-kind.sh deploy ate-system --credential-provider='{"enabled":false}' ``` 2. **`kubectl-ate` installed in `PATH`**: ```bash diff --git a/benchmarking/nighthawk-ingress/README.md b/benchmarking/nighthawk-ingress/README.md index 46ad4e9fda..0c32545dc0 100644 --- a/benchmarking/nighthawk-ingress/README.md +++ b/benchmarking/nighthawk-ingress/README.md @@ -120,7 +120,7 @@ requests 2×`envoyCpu` CPUs; the runner requests `clientConcurrency`+1). ```bash # .ate-dev-env.sh at the repo root, then: -hack/install-ate.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' +hack/install-ate.sh deploy ate-system --credential-provider='{"name":"k8s.io"}' benchmarking/workloads/deploy.sh --deploy --worker-count 50 --sandbox-class gvisor ``` @@ -147,7 +147,7 @@ Flags: `--envoy-cpu`, `--actors`, `--tail-latency-slo-ms`, `--atespace`, Notes: - The script does not redeploy substrate — if substrate code changed, - rerun `hack/install-ate.sh --deploy-ate-system` first. + rerun `hack/install-ate.sh deploy ate-system` first. - `--atespace` isolates actor *fleets* between experiments sharing a cluster; capacity runs themselves still need the router exclusively (the cpu pin is global and the run saturates it). diff --git a/benchmarking/nighthawk-ingress/run-dev.sh b/benchmarking/nighthawk-ingress/run-dev.sh index a905597909..bfb382fb44 100755 --- a/benchmarking/nighthawk-ingress/run-dev.sh +++ b/benchmarking/nighthawk-ingress/run-dev.sh @@ -80,7 +80,7 @@ docker info >/dev/null 2>&1 || { kubectl get deployment atenet-router -n ate-system >/dev/null 2>&1 || { echo "ERROR: substrate is not deployed on ${CLUSTER_NAME}. Run:" >&2 - echo " hack/install-ate.sh --deploy-ate-system --credential-provider='{\"name\":\"k8s.io\"}'" >&2 + echo " hack/install-ate.sh deploy ate-system --credential-provider='{\"name\":\"k8s.io\"}'" >&2 exit 1 } diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index eff1360f99..fb06982948 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -79,13 +79,13 @@ log() { } # --- 1. deploy the control plane ------------------------------------------- -log "Deploying the ate control plane (--deploy-ate-system)..." +log "Deploying the ate control plane (deploy ate-system)..." if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh deploy ate-system else # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh depoy ate-system + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh deploy ate-system fi # --- 2. install micro-VM deps (assets + cluster-wide SandboxConfig) -------- From dcc169d4fab21333481423947b825e5e6e1d5feb Mon Sep 17 00:00:00 2001 From: Yufan Su Date: Tue, 20 Oct 2026 22:42:55 +0000 Subject: [PATCH 23/63] atenet-egress: raise the envoy config for a million tunnels case (#2732) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What Raises the circuit breaker limits on three clusters in `manifests/ate-install/atenet-egress.yaml`: | Cluster | Before | After | |---|---|---| | `mitm_internal` | 16,384 | 1,048,400 | | `egress_forward_proxy` (HTTPS to destinations) | unset (Envoy default 1,024) | 65,536 | | `egress_forward_proxy_cleartext` (HTTP to destinations) | unset (Envoy default 1,024) | 65,536 | Each cluster gets the same value for `max_connections`, `max_pending_requests` and `max_requests`. ### Why **`mitm_internal` caps the tunnels per replica.** Each open tunnel holds one connection and one request on this cluster. HTTP/2 multiplexing of CONNECT doesn't change that, because it only shares the actor-to-gateway connections. At 16,384, the next CONNECT gets a 503. **The destination clusters were running at Envoy's default of 1,024 connections.** Envoy keeps a separate connection pool per worker thread per destination, so the count grows with threads × destinations, not with tunnels. At 1,000 destinations and 16 threads that's already 16,000 pools. Past the limit, requests queue and then fail with `503 UO`. - Scale tests measured 16–17k destination connections at 16 threads, with 250k–500k tunnels. - 65,536 covers 1,000 destinations × 32 threads × 2 requests at once per destination per thread. ### Consequence - **Increased memory usage**: Up to 1,048,400 tunnels, about 49–56 GiB at ~49 KiB (HTTP) or ~56 KiB (HTTPS) each. - Currently the Envoy container has no memory request or limit. - [ ] Tests pass - [ ] Appropriate changes to documentation are included in the PR --- manifests/ate-install/atenet-egress.yaml | 26 ++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 7b266df961..1554712f85 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -925,15 +925,14 @@ data: # Pools are create using hashes of filter state, which includes filter state objects that use # envoy.hashable_string factory. connect_timeout: 1s - # Each open tunnel holds one connection and one request here, so these - # cap concurrent tunnels per replica (Envoy's default is 1024); past - # them a CONNECT gets a 503. + # Envoy always opens a new pool's first connection. + # Past the limit, a second tunnel in the same pool waits for one to close. circuit_breakers: thresholds: - priority: DEFAULT - max_connections: 16384 - max_requests: 16384 - max_pending_requests: 16384 + max_connections: 1048400 + max_requests: 1048400 + max_pending_requests: 1048400 typed_extension_protocol_options: envoy.extensions.upstreams.tcp.v3.TcpProtocolOptions: "@type": type.googleapis.com/envoy.extensions.upstreams.tcp.v3.TcpProtocolOptions @@ -1030,6 +1029,14 @@ data: - name: egress_forward_proxy lb_policy: CLUSTER_PROVIDED connect_timeout: 5s + # Fits 1000 destinations x 32 worker threads x 2 requests at once per + # destination per thread = 64000 connections. + circuit_breakers: + thresholds: + - priority: DEFAULT + max_connections: 65536 + max_pending_requests: 65536 + max_requests: 65536 cluster_type: name: envoy.clusters.dynamic_forward_proxy typed_config: @@ -1101,6 +1108,13 @@ data: - name: egress_forward_proxy_cleartext lb_policy: CLUSTER_PROVIDED connect_timeout: 5s + # Sized like egress_forward_proxy, for the same reason. + circuit_breakers: + thresholds: + - priority: DEFAULT + max_connections: 65536 + max_pending_requests: 65536 + max_requests: 65536 cluster_type: name: envoy.clusters.dynamic_forward_proxy typed_config: From 812478056925706b545df527b1f56c1a239c3202 Mon Sep 17 00:00:00 2001 From: Chenyi Wang Date: Tue, 20 Oct 2026 22:43:01 +0000 Subject: [PATCH 24/63] ateworker: clear partial checkpoint files and preserve microVM base on merge (#2623) If a checkpoint attempt fails mid-flight, retrying `CheckpointWorkload` can fail due to leftover or displaced files on disk: 1. **gVisor**: `checkpoint-state/` was not cleared before checkpointing, leaving partial files from a failed attempt behind. 2. **microVM**: `MergeDeltaIntoBase` moved `restore-state/memory-ranges` into `checkpoint-state/` before merging, so a failed checkpoint followed by a retry (`os.RemoveAll(checkpointDir)`) permanently deleted the restore base. **Fix:** - Clear `checkpoint-state/` before running gVisor checkpoint (matching microVM). - In microVM `MergeDeltaIntoBase`, merge delta pages onto `baseFile` in place under `restore-state/` and rename into `checkpoint-state/` only after the merge succeeds. --- cmd/ateworker-gvisor/main.go | 4 + cmd/ateworker-microvm/internal/ch/merge.go | 93 +++++++++------- .../internal/ch/merge_test.go | 105 +++++++++++++++++- cmd/ateworker-microvm/restore_test.go | 14 ++- 4 files changed, 170 insertions(+), 46 deletions(-) diff --git a/cmd/ateworker-gvisor/main.go b/cmd/ateworker-gvisor/main.go index d876d92e7f..9c201942aa 100644 --- a/cmd/ateworker-gvisor/main.go +++ b/cmd/ateworker-gvisor/main.go @@ -792,6 +792,10 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke } checkpointPath := req.GetActorDirs().GetCheckpointDir() + // Clear any partial files from a previous failed attempt. + if err := os.RemoveAll(checkpointPath); err != nil { + return nil, fmt.Errorf("while clearing checkpoint directory: %w", err) + } err = os.MkdirAll(checkpointPath, 0o700) timing.prep = lap(&tLast) if err != nil { diff --git a/cmd/ateworker-microvm/internal/ch/merge.go b/cmd/ateworker-microvm/internal/ch/merge.go index 1e56b94f4f..2406057ed7 100644 --- a/cmd/ateworker-microvm/internal/ch/merge.go +++ b/cmd/ateworker-microvm/internal/ch/merge.go @@ -23,6 +23,7 @@ import ( "fmt" "io" "os" + "sync" "syscall" "golang.org/x/sys/unix" @@ -75,7 +76,7 @@ func MergeSparseOverlay(ctx context.Context, baseFile, deltaFile, outFile string } defer o.Close() - if _, err := copySparseRegions(d, o); err != nil { + if _, err := copySparseRegions(ctx, d, o); err != nil { return err } // No fsync: atelet ships the merged image to GCS (the durability point), so a @@ -99,21 +100,22 @@ func MergeSparseOverlay(ctx context.Context, baseFile, deltaFile, outFile string } // MergeDeltaIntoBase overlays deltaFile's populated pages onto baseFile in place -// and leaves the complete merged snapshot at deltaFile's path — the same result as +// and links the complete merged snapshot at deltaFile's path — the same result as // MergeSparseOverlay, but WITHOUT copying baseFile's working set on every suspend. // // baseFile is the per-actor restore staging file (restore-state/memory-ranges), // demand-paged only by the now-paused CH we are about to tear down and discarded // afterward. So rather than `cp`-ing its whole working set (e.g. ~150MiB of a 2GiB -// guest, ~0.8s on the suspend critical path), we rename baseFile next to deltaFile, -// overlay deltaFile's (small) faulted pages onto it, and swap it into deltaFile's -// place — turning an O(working-set) copy into an O(delta) write plus two renames. +// guest, ~0.8s on the suspend critical path), we overlay deltaFile's (small) +// faulted pages onto baseFile in place while it stays in restore-state/, and +// hard-link baseFile into deltaFile's place once the overlay succeeds. Keeping +// baseFile in restore-state/ ensures a retry can still find it if a sibling +// checkpoint step (e.g. rootfs or volume tar) fails and wipes checkpoint-state/. // // baseFile and deltaFile are siblings under the actor dir (restore-state/ and -// checkpoint-state/), so the renames are same-filesystem (metadata-only). If they -// straddle a mount boundary (EXDEV) it falls back to the copying MergeSparseOverlay -// (baseFile is untouched until the first rename succeeds), as it does when baseFile -// carries a second link and so cannot be overlaid in place. +// checkpoint-state/), so the link is same-filesystem (metadata-only). If they +// straddle a mount boundary or baseFile carries a second link, it falls back to +// the copying MergeSparseOverlay. func MergeDeltaIntoBase(ctx context.Context, baseFile, deltaFile string) error { bi, err := os.Stat(baseFile) if err != nil { @@ -129,60 +131,71 @@ func MergeDeltaIntoBase(ctx context.Context, baseFile, deltaFile string) error { return fmt.Errorf("MergeDeltaIntoBase: size mismatch base=%d delta=%d", bi.Size(), di.Size()) } - // The fast path below renames baseFile and mutates its inode in place, so it - // is only for an expendable staging file. Preserved snapshots use - // MergeSparseOverlay directly; a shared inode (Nlink > 1) falls back to it too. - if st, ok := bi.Sys().(*syscall.Stat_t); ok && st.Nlink > 1 { + // Only mutate an unshared staging file on the same filesystem in place. + bst, bOk := bi.Sys().(*syscall.Stat_t) + dst, dOk := di.Sys().(*syscall.Stat_t) + if (bOk && bst.Nlink > 1) || (bOk && dOk && bst.Dev != dst.Dev) { return MergeSparseOverlay(ctx, baseFile, deltaFile, deltaFile) } - // Move baseFile (with its already-on-disk working set) next to deltaFile. If this - // fails with EXDEV the two are on different filesystems and baseFile is still - // intact, so fall back to the copying merge. - merged := deltaFile + ".merged.tmp" - _ = os.Remove(merged) - if err := os.Rename(baseFile, merged); err != nil { - if errors.Is(err, unix.EXDEV) { - return MergeSparseOverlay(ctx, baseFile, deltaFile, deltaFile) - } - return fmt.Errorf("rename base->merged: %w", err) - } - d, err := os.Open(deltaFile) if err != nil { return fmt.Errorf("open delta %q: %w", deltaFile, err) } - defer d.Close() - m, err := os.OpenFile(merged, os.O_RDWR, 0o600) + closeD := sync.OnceValue(d.Close) + defer func() { _ = closeD() }() + b, err := os.OpenFile(baseFile, os.O_RDWR, 0o600) if err != nil { return err } - defer m.Close() - if _, err := copySparseRegions(d, m); err != nil { + closeB := sync.OnceValue(b.Close) + defer func() { _ = closeB() }() + if _, err := copySparseRegions(ctx, d, b); err != nil { return err } // No fsync: atelet ships the merged image to GCS (the durability point), so a // partial local file after a crash is just discarded + the suspend retried. - if err := m.Close(); err != nil { + if err := closeB(); err != nil { + return err + } + if err := closeD(); err != nil { + return err + } + if err := ctx.Err(); err != nil { return err } // Put the merged image at deltaFile's name. Unlink the old delta FIRST, then - // rename onto the now-free name: renaming OVER an existing file makes ext4 - // (data=ordered) synchronously write back the renamed file's dirty pages, and - // `merged` carries ~150MiB of dirty download pages, so a replace-rename costs - // ~0.5-0.8s. Renaming to a non-existent name skips that flush (the dirty pages - // stay in page cache for atelet to ship), taking the merge ~840ms→~5ms. + // hard-link baseFile onto the now-free name. Keeping baseFile linked in + // restore-state/ means that if a concurrent checkpoint step (e.g. tar) fails + // after MergeDeltaIntoBase finishes and a retry wipes checkpoint-state/, + // restore-state/memory-ranges remains intact with Nlink back at 1 for the + // retry to merge into again. if err := os.Remove(deltaFile); err != nil { return fmt.Errorf("remove old delta: %w", err) } - return os.Rename(merged, deltaFile) + return os.Link(baseFile, deltaFile) +} + +type contextReader struct { + ctx context.Context + r io.Reader +} + +func (cr contextReader) Read(p []byte) (int, error) { + if err := cr.ctx.Err(); err != nil { + return 0, err + } + return cr.r.Read(p) } // copySparseRegions overwrites dst with every populated (non-hole) region of src // at the same byte offsets, leaving dst's other bytes untouched. Holes in src are // located via SEEK_DATA/SEEK_HOLE and skipped. src and dst are assumed to be the // same logical size (the caller validates this). -func copySparseRegions(src, dst *os.File) (copied int64, err error) { +func copySparseRegions(ctx context.Context, src, dst *os.File) (copied int64, err error) { + if err := ctx.Err(); err != nil { + return 0, err + } si, err := src.Stat() if err != nil { return 0, err @@ -190,8 +203,12 @@ func copySparseRegions(src, dst *os.File) (copied int64, err error) { size := si.Size() sfd := int(src.Fd()) buf := make([]byte, 1<<20) + cr := contextReader{ctx: ctx, r: src} off := int64(0) for off < size { + if err := ctx.Err(); err != nil { + return copied, err + } // Next populated region [ds, de) in src. ds, err := unix.Seek(sfd, off, unix.SEEK_DATA) if err != nil { @@ -216,7 +233,7 @@ func copySparseRegions(src, dst *os.File) (copied int64, err error) { if n > remaining { n = remaining } - r, err := io.ReadFull(src, buf[:n]) + r, err := io.ReadFull(cr, buf[:n]) if r > 0 { if _, werr := dst.Write(buf[:r]); werr != nil { return copied, werr diff --git a/cmd/ateworker-microvm/internal/ch/merge_test.go b/cmd/ateworker-microvm/internal/ch/merge_test.go index ddb3620889..3d3b9b54a5 100644 --- a/cmd/ateworker-microvm/internal/ch/merge_test.go +++ b/cmd/ateworker-microvm/internal/ch/merge_test.go @@ -64,7 +64,7 @@ func fill(seed, n int) []byte { return b } -// TestMergeDeltaIntoBase asserts the fast (rename+overlay) merge produces exactly +// TestMergeDeltaIntoBase asserts the fast (in-place overlay + link) merge produces // base-with-delta-overlaid — byte-identical to the reference copying merge // (MergeSparseOverlay). This is the guest memory image, so any divergence = a dead // or corrupted guest on resume. @@ -95,7 +95,7 @@ func TestMergeDeltaIntoBase(t *testing.T) { ctx := context.Background() - // --- fast path: MergeDeltaIntoBase (consumes base, result lands in delta) --- + // --- fast path: MergeDeltaIntoBase (overlays base in place, links result at delta) --- dir := t.TempDir() base := filepath.Join(dir, "restore-state", "memory-ranges") delta := filepath.Join(dir, "checkpoint-state", "memory-ranges") @@ -118,9 +118,18 @@ func TestMergeDeltaIntoBase(t *testing.T) { if !bytes.Equal(got, want) { t.Fatalf("MergeDeltaIntoBase result != expected (len got=%d want=%d)", len(got), len(want)) } - // base must have been consumed (renamed away), not left behind as a stale copy. - if _, err := os.Stat(base); !os.IsNotExist(err) { - t.Errorf("base still present after MergeDeltaIntoBase (err=%v); expected it consumed", err) + // base remains in restore-state/ hard-linked to delta (same inode, no extra blocks) + // so a retry after a sibling checkpoint step fails and wipes checkpoint-state/ still has base. + bi, err := os.Stat(base) + if err != nil { + t.Fatalf("stat base after MergeDeltaIntoBase: %v", err) + } + di, err := os.Stat(delta) + if err != nil { + t.Fatalf("stat delta after MergeDeltaIntoBase: %v", err) + } + if !os.SameFile(bi, di) { + t.Errorf("expected base and delta to share an inode after fast-path merge") } // The merged image must stay sparse (the holes between regions are not allocated). if fi, err := os.Stat(delta); err == nil { @@ -279,3 +288,89 @@ func TestCopySparseFileLeavesZeroBlocksAsHoles(t *testing.T) { t.Errorf("dst allocates %d bytes, want the zero runs left as holes", allocated) } } + +func TestMergeDeltaIntoBaseSurvivesCancelledAttemptAndCheckpointDirWipe(t *testing.T) { + const size = 4 << 20 // 4 MiB logical + baseRegions := []region{ + {off: 0, data: fill(1, 4096)}, + {off: 1 << 20, data: fill(2, 65536)}, + } + deltaRegions := []region{ + {off: 1 << 20, data: fill(99, 65536)}, + {off: 2 << 20, data: fill(42, 8192)}, + } + want := make([]byte, size) + for _, r := range baseRegions { + copy(want[r.off:], r.data) + } + for _, r := range deltaRegions { + copy(want[r.off:], r.data) + } + + dir := t.TempDir() + restoreDir := filepath.Join(dir, "restore-state") + checkpointDir := filepath.Join(dir, "checkpoint-state") + if err := os.MkdirAll(restoreDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(checkpointDir, 0o700); err != nil { + t.Fatal(err) + } + base := filepath.Join(restoreDir, "memory-ranges") + delta := filepath.Join(checkpointDir, "memory-ranges") + writeSparse(t, base, size, baseRegions) + writeSparse(t, delta, size, deltaRegions) + + // Attempt 1: context already cancelled when delta is all holes (SEEK_DATA returns ENXIO). + writeSparse(t, delta, size, nil) + cancelledCtx, cancel := context.WithCancel(context.Background()) + cancel() + if err := MergeDeltaIntoBase(cancelledCtx, base, delta); err == nil { + t.Fatal("expected cancelled MergeDeltaIntoBase on empty delta to fail, got nil") + } + + // Attempt 2: context already cancelled when delta has data regions. + writeSparse(t, delta, size, deltaRegions) + if err := MergeDeltaIntoBase(cancelledCtx, base, delta); err == nil { + t.Fatal("expected cancelled MergeDeltaIntoBase to fail, got nil") + } + + // Attempt 3: MergeDeltaIntoBase succeeds (hard-linking base -> delta), then a + // sibling checkpoint step (e.g. rootfs or volume tar) fails, causing the next + // CheckpointWorkload retry to wipe checkpoint-state/ and re-snapshot delta. + writeSparse(t, delta, size, deltaRegions) + if err := MergeDeltaIntoBase(context.Background(), base, delta); err != nil { + t.Fatalf("MergeDeltaIntoBase before sibling failure: %v", err) + } + if err := os.RemoveAll(checkpointDir); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(checkpointDir, 0o700); err != nil { + t.Fatal(err) + } + writeSparse(t, delta, size, deltaRegions) + + // Attempt 4: retry must still succeed on the fast path because unlinking delta + // dropped base's Nlink back to 1 in restore-state/. + if err := MergeDeltaIntoBase(context.Background(), base, delta); err != nil { + t.Fatalf("retry MergeDeltaIntoBase failed: %v", err) + } + got, err := os.ReadFile(delta) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatal("retry MergeDeltaIntoBase result != expected") + } + bi, err := os.Stat(base) + if err != nil { + t.Fatal(err) + } + di, err := os.Stat(delta) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(bi, di) { + t.Errorf("expected retry MergeDeltaIntoBase to stay on fast path (same inode)") + } +} diff --git a/cmd/ateworker-microvm/restore_test.go b/cmd/ateworker-microvm/restore_test.go index 926885a76b..42caeb77df 100644 --- a/cmd/ateworker-microvm/restore_test.go +++ b/cmd/ateworker-microvm/restore_test.go @@ -446,7 +446,7 @@ func TestMergeOnDemandDeltaPreservesRestoreSource(t *testing.T) { } }) - t.Run("preserveRestoreSource=false moves expendable base into checkpointDir", func(t *testing.T) { + t.Run("preserveRestoreSource=false merges expendable base in place and links into checkpointDir", func(t *testing.T) { restoreDir := t.TempDir() checkpointDir := t.TempDir() basePath := filepath.Join(restoreDir, "memory-ranges") @@ -459,8 +459,16 @@ func TestMergeOnDemandDeltaPreservesRestoreSource(t *testing.T) { t.Fatalf("mergeOnDemandDelta: %v", err) } - if _, err := os.Stat(basePath); !os.IsNotExist(err) { - t.Errorf("expendable base memory-ranges still in restoreDir (stat err = %v), want moved", err) + bi, err := os.Stat(basePath) + if err != nil { + t.Fatalf("stat base memory-ranges in restoreDir: %v", err) + } + di, err := os.Stat(deltaPath) + if err != nil { + t.Fatalf("stat merged memory-ranges in checkpointDir: %v", err) + } + if !os.SameFile(bi, di) { + t.Errorf("expected basePath and deltaPath to share an inode after in-place merge") } gotMerged, err := os.ReadFile(deltaPath) if err != nil { From 1a927e295decd10e107e780c16382141d55ed170 Mon Sep 17 00:00:00 2001 From: Max Smythe Date: Tue, 20 Oct 2026 22:51:54 +0000 Subject: [PATCH 25/63] glutton: add RunScript, a script tree run inside the actor as one request (#2162) A benchmark that wants an actor to do a whole on-sandbox activity, a build, a test run, a file churn, had to issue every resource operation as its own request through the router. That costs a round trip per operation, and it leaves the driver unable to hold the actor busy for a chosen length of time without knowing how long each operation takes on the sandbox at hand. RunScript takes the activity as a script and runs it in-process, holding the request open until it finishes. A script is a tree. A Block runs its steps in order and may cycle them for a wall-clock budget; a Step is a nested Block or an Operation; an Operation runs its requests at the same time and waits for all of them. A budget is checked between passes: the pass under way finishes, so a loop overruns by at most one pass and no request is cut short, and a nested block stops cycling once any ancestor's budget has elapsed the same way. The only errors are the caller's context ending and a failing request, which names the request's path in the tree and keeps its status code. The request set is the service's work requests plus sleep. UseCPU and OpenFD are left out because they install a background load that outlives the script; BurnCPU is the CPU spike. The response mirrors the tree with summed vital stats at every node: requests run, bytes written and read to disk and RAM, burn iterations, time slept, and the node's own wall clock and entry count. A node inside a looped block sums over its passes, so the response is the size of the script however long it ran, and the root carries the totals. The fake actor answers a script with the mirrored tree, so drivers can be tested against it. Fixes # > It's a good idea to open an issue first for discussion. - [ x ] Tests pass - [ x ] Appropriate changes to documentation are included in the PR --- internal/benchmarking/glutton/fake/server.go | 53 ++ internal/benchmarking/glutton/glutton.go | 1 + internal/benchmarking/glutton/script.go | 319 ++++++++ internal/benchmarking/glutton/script_test.go | 321 ++++++++ internal/benchmarking/glutton/server.go | 1 + internal/proto/glutton/glutton.pb.go | 811 ++++++++++++++++++- internal/proto/glutton/glutton.proto | 112 +++ internal/proto/glutton/glutton_grpc.pb.go | 54 ++ 8 files changed, 1642 insertions(+), 30 deletions(-) create mode 100644 internal/benchmarking/glutton/script.go create mode 100644 internal/benchmarking/glutton/script_test.go diff --git a/internal/benchmarking/glutton/fake/server.go b/internal/benchmarking/glutton/fake/server.go index 49d9c19536..1dc689064c 100644 --- a/internal/benchmarking/glutton/fake/server.go +++ b/internal/benchmarking/glutton/fake/server.go @@ -41,6 +41,7 @@ const ( IngestRoute = glutton.IngestRoute PingRoute = glutton.PingRoute UseCPURoute = glutton.UseCPURoute + RunScriptRoute = glutton.RunScriptRoute ) // Server is an httptest-backed stand-in for a glutton actor holding one file. @@ -72,6 +73,7 @@ type Server struct { burnMillis []int64 ingestSizes []int64 cpuRequests []*gluttonpb.UseCPURequest + scripts []*gluttonpb.RunScriptRequest } func (s *Server) reportedDigest() []byte { @@ -135,6 +137,13 @@ func (s *Server) RecordedRAMReadSizes() []string { return append([]string(nil), s.ramReadSizes...) } +// RecordedScripts returns each /runscript request. +func (s *Server) RecordedScripts() []*gluttonpb.RunScriptRequest { + s.mu.Lock() + defer s.mu.Unlock() + return append([]*gluttonpb.RunScriptRequest(nil), s.scripts...) +} + // RecordedCPURequests returns each /usecpu request. func (s *Server) RecordedCPURequests() []*gluttonpb.UseCPURequest { s.mu.Lock() @@ -317,6 +326,26 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) { resp, _ := proto.Marshal(&gluttonpb.UseCPUResponse{NumCores: req.GetNumCores()}) _, _ = w.Write(resp) + case RunScriptRoute: + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + var req gluttonpb.RunScriptRequest + if err := proto.Unmarshal(body, &req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + s.mu.Lock() + s.scripts = append(s.scripts, &req) + s.mu.Unlock() + + // The fake does no work, so the script is instant: every node + // entered once, every request run. + resp, _ := proto.Marshal(&gluttonpb.RunScriptResponse{Result: mirrorBlock(req.GetScript())}) + _, _ = w.Write(resp) + default: http.NotFound(w, r) } @@ -335,3 +364,27 @@ func (s *Server) RecordedIngestSizes() []int64 { defer s.mu.Unlock() return append([]int64(nil), s.ingestSizes...) } + +// mirrorBlock builds the result tree the real glutton would return for a +// script that cost nothing: one pass per node and one request per leaf. +func mirrorBlock(block *gluttonpb.Block) *gluttonpb.StepResult { + result := &gluttonpb.StepResult{Stats: &gluttonpb.Stats{Passes: 1}} + for _, step := range block.GetSteps() { + var child *gluttonpb.StepResult + switch kind := step.GetKind().(type) { + case *gluttonpb.Step_Block: + child = mirrorBlock(kind.Block) + case *gluttonpb.Step_Operation: + child = &gluttonpb.StepResult{Stats: &gluttonpb.Stats{Passes: 1}} + for range kind.Operation.GetRequests() { + child.Children = append(child.Children, &gluttonpb.StepResult{Stats: &gluttonpb.Stats{Passes: 1, RequestsRun: 1}}) + child.Stats.RequestsRun++ + } + default: + child = &gluttonpb.StepResult{Stats: &gluttonpb.Stats{}} + } + result.Children = append(result.Children, child) + result.Stats.RequestsRun += child.GetStats().GetRequestsRun() + } + return result +} diff --git a/internal/benchmarking/glutton/glutton.go b/internal/benchmarking/glutton/glutton.go index 1f978e26f8..bf855764dd 100644 --- a/internal/benchmarking/glutton/glutton.go +++ b/internal/benchmarking/glutton/glutton.go @@ -44,4 +44,5 @@ const ( BurnCPURoute = "/burncpu" IngestRoute = "/ingest" UseCPURoute = "/usecpu" + RunScriptRoute = "/runscript" ) diff --git a/internal/benchmarking/glutton/script.go b/internal/benchmarking/glutton/script.go new file mode 100644 index 0000000000..c4250f84a0 --- /dev/null +++ b/internal/benchmarking/glutton/script.go @@ -0,0 +1,319 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package glutton + +import ( + "context" + "fmt" + "time" + + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" +) + +// RunScript runs the request's script tree in-process and holds the +// request open until it finishes. The tree is checked for shape before +// anything runs; a request's own argument errors surface when it runs, +// with its path in the tree and kind prepended and its status code kept, +// so a caller can tell a bad script from a failed request. See the proto +// for what a block, an operation, and the stats mean. +func (s *Service) RunScript(ctx context.Context, req *gluttonpb.RunScriptRequest) (*gluttonpb.RunScriptResponse, error) { + if req.GetScript() == nil { + return nil, status.Error(codes.InvalidArgument, "script is required") + } + if err := validateBlock(req.GetScript(), "script"); err != nil { + return nil, err + } + result, err := s.runBlock(ctx, req.GetScript(), "script", time.Time{}) + if err != nil { + return nil, err + } + return &gluttonpb.RunScriptResponse{Result: result}, nil +} + +// validateBlock rejects what can be seen without running anything: an +// empty block or operation, a step or request with nothing set, a negative +// budget or sleep. path names the node in messages. +func validateBlock(block *gluttonpb.Block, path string) error { + if len(block.GetSteps()) == 0 { + return status.Errorf(codes.InvalidArgument, "%s: steps is required", path) + } + if block.GetLoopDurationMs() < 0 { + return status.Errorf(codes.InvalidArgument, "%s: loop_duration_ms must be non-negative", path) + } + for i, step := range block.GetSteps() { + stepPath := fmt.Sprintf("%s.steps[%d]", path, i) + switch kind := step.GetKind().(type) { + case *gluttonpb.Step_Block: + if err := validateBlock(kind.Block, stepPath+".block"); err != nil { + return err + } + case *gluttonpb.Step_Operation: + if err := validateOperation(kind.Operation, stepPath+".operation"); err != nil { + return err + } + default: + return status.Errorf(codes.InvalidArgument, "%s: no kind set", stepPath) + } + } + return nil +} + +func validateOperation(op *gluttonpb.Operation, path string) error { + if len(op.GetRequests()) == 0 { + return status.Errorf(codes.InvalidArgument, "%s: requests is required", path) + } + for i, request := range op.GetRequests() { + requestPath := fmt.Sprintf("%s.requests[%d]", path, i) + switch kind := request.GetKind().(type) { + case nil: + return status.Errorf(codes.InvalidArgument, "%s: no kind set", requestPath) + case *gluttonpb.Request_Sleep: + if kind.Sleep.GetDurationMs() < 0 { + return status.Errorf(codes.InvalidArgument, "%s (sleep): duration_ms must be non-negative", requestPath) + } + } + } + return nil +} + +// runBlock runs the block's steps in order and, when the block loops, +// again and again until a budget has elapsed. deadline is the earliest +// budget among the block's ancestors, zero for none; the block's own +// budget tightens it for the block and everything beneath. A budget is +// checked between passes only: the pass in progress finishes, so a loop +// overruns its budget by at most one pass and nothing is cut mid-flight. +// The only errors are the caller's context ending and a failing request. +// The context is checked before every step, since most requests run to +// completion without looking at it: a looped block of CPU burns or disk +// writes stops at the next step once the caller is gone, not at its budget. +func (s *Service) runBlock(ctx context.Context, block *gluttonpb.Block, path string, deadline time.Time) (*gluttonpb.StepResult, error) { + start := time.Now() + result := newResult(len(block.GetSteps())) + if budget := block.GetLoopDurationMs(); budget > 0 { + own := start.Add(time.Duration(budget) * time.Millisecond) + if deadline.IsZero() || own.Before(deadline) { + deadline = own + } + } + for { + result.Stats.Passes++ + for i, step := range block.GetSteps() { + if err := ctx.Err(); err != nil { + result.Stats.ElapsedMs = time.Since(start).Milliseconds() + return result, status.FromContextError(err).Err() + } + child, err := s.runStep(ctx, step, fmt.Sprintf("%s.steps[%d]", path, i), deadline) + if child != nil { + addResult(result.Children[i], child) + addTotals(result.Stats, child.Stats) + } + if err != nil { + result.Stats.ElapsedMs = time.Since(start).Milliseconds() + return result, err + } + } + if block.GetLoopDurationMs() == 0 || !time.Now().Before(deadline) { + break + } + } + result.Stats.ElapsedMs = time.Since(start).Milliseconds() + return result, nil +} + +func (s *Service) runStep(ctx context.Context, step *gluttonpb.Step, path string, deadline time.Time) (*gluttonpb.StepResult, error) { + switch kind := step.GetKind().(type) { + case *gluttonpb.Step_Block: + return s.runBlock(ctx, kind.Block, path+".block", deadline) + case *gluttonpb.Step_Operation: + return s.runOperation(ctx, kind.Operation, path+".operation") + default: + return nil, status.Errorf(codes.InvalidArgument, "%s: no kind set", path) + } +} + +// runOperation runs the operation's requests at the same time. The first +// failure cancels the rest and is the error returned; the result still +// carries the stats of every request that ran, failed ones included. +func (s *Service) runOperation(ctx context.Context, op *gluttonpb.Operation, path string) (*gluttonpb.StepResult, error) { + start := time.Now() + result := newResult(len(op.GetRequests())) + group, groupCtx := errgroup.WithContext(ctx) + for i, request := range op.GetRequests() { + group.Go(func() error { + stats, err := s.runRequest(groupCtx, request) + if stats != nil { + result.Children[i].Stats = stats + } + if err != nil { + return prefixError(fmt.Sprintf("%s.requests[%d] (%s)", path, i, requestKind(request)), err) + } + return nil + }) + } + err := group.Wait() + for _, child := range result.Children { + addTotals(result.Stats, child.Stats) + } + result.Stats.Passes = 1 + result.Stats.ElapsedMs = time.Since(start).Milliseconds() + return result, err +} + +// runRequest dispatches one request to the service method it names and +// reports what it did. Results other than the stats are dropped. +func (s *Service) runRequest(ctx context.Context, request *gluttonpb.Request) (*gluttonpb.Stats, error) { + start := time.Now() + stats := &gluttonpb.Stats{Passes: 1, RequestsRun: 1} + var err error + switch kind := request.GetKind().(type) { + case *gluttonpb.Request_WriteRam: + _, err = s.WriteRAM(ctx, kind.WriteRam) + if err == nil { + stats.RamBytesWritten, _ = parseBytes(kind.WriteRam.GetSize()) + } + case *gluttonpb.Request_ReadRam: + var resp *gluttonpb.ReadRAMResponse + resp, err = s.ReadRAM(ctx, kind.ReadRam) + stats.RamBytesRead = resp.GetSize() + case *gluttonpb.Request_WriteDisk: + _, err = s.WriteDisk(ctx, kind.WriteDisk) + if err == nil { + stats.DiskBytesWritten = int64(kind.WriteDisk.GetSize()) + } + case *gluttonpb.Request_ReadDisk: + var resp *gluttonpb.ReadDiskResponse + resp, err = s.ReadDisk(ctx, kind.ReadDisk) + stats.DiskBytesRead = resp.GetSize() + case *gluttonpb.Request_BurnCpu: + var resp *gluttonpb.BurnCPUResponse + resp, err = s.BurnCPU(ctx, kind.BurnCpu) + stats.BurnIterations = resp.GetIterations() + case *gluttonpb.Request_Ingest: + _, err = s.Ingest(ctx, kind.Ingest) + if err == nil { + stats.DiskBytesWritten = int64(len(kind.Ingest.GetPayload())) + } + case *gluttonpb.Request_Sleep: + err = sleep(ctx, time.Duration(kind.Sleep.GetDurationMs())*time.Millisecond) + stats.SleptMs = time.Since(start).Milliseconds() + default: + err = status.Errorf(codes.InvalidArgument, "unknown request %T", kind) + } + stats.ElapsedMs = time.Since(start).Milliseconds() + return stats, err +} + +// sleep idles for duration or until ctx ends, whichever comes first. +func sleep(ctx context.Context, duration time.Duration) error { + if duration <= 0 { + return nil + } + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return status.FromContextError(ctx.Err()).Err() + } +} + +// newResult is a node with zeroed stats and n zeroed children, ready to +// have each pass added in. +func newResult(n int) *gluttonpb.StepResult { + result := &gluttonpb.StepResult{Stats: &gluttonpb.Stats{}, Children: make([]*gluttonpb.StepResult, n)} + for i := range result.Children { + result.Children[i] = &gluttonpb.StepResult{Stats: &gluttonpb.Stats{}} + } + return result +} + +// addResult folds one entry's result tree into the accumulated node of the +// same shape. A child the entry did not produce (an operation's request +// that never started) adds nothing. +func addResult(dst, src *gluttonpb.StepResult) { + addStats(dst.Stats, src.Stats) + for len(dst.Children) < len(src.Children) { + dst.Children = append(dst.Children, &gluttonpb.StepResult{Stats: &gluttonpb.Stats{}}) + } + for i, child := range src.Children { + if child != nil { + addResult(dst.Children[i], child) + } + } +} + +// addStats sums src into dst, field by field: one node's later entry +// added to its earlier ones. +func addStats(dst, src *gluttonpb.Stats) { + if src == nil { + return + } + dst.ElapsedMs += src.ElapsedMs + dst.Passes += src.Passes + addTotals(dst, src) +} + +// addTotals sums a child's work into its parent: every field but passes, +// which count the parent's own entries, and elapsed, which the parent +// measures itself. +func addTotals(dst, src *gluttonpb.Stats) { + if src == nil { + return + } + dst.RequestsRun += src.RequestsRun + dst.DiskBytesWritten += src.DiskBytesWritten + dst.DiskBytesRead += src.DiskBytesRead + dst.RamBytesWritten += src.RamBytesWritten + dst.RamBytesRead += src.RamBytesRead + dst.BurnIterations += src.BurnIterations + dst.SleptMs += src.SleptMs +} + +// requestKind is the proto field name of the request that is set. +func requestKind(request *gluttonpb.Request) string { + switch request.GetKind().(type) { + case *gluttonpb.Request_WriteRam: + return "write_ram" + case *gluttonpb.Request_ReadRam: + return "read_ram" + case *gluttonpb.Request_WriteDisk: + return "write_disk" + case *gluttonpb.Request_ReadDisk: + return "read_disk" + case *gluttonpb.Request_BurnCpu: + return "burn_cpu" + case *gluttonpb.Request_Ingest: + return "ingest" + case *gluttonpb.Request_Sleep: + return "sleep" + } + return "unset" +} + +// prefixError puts the failing request's path ahead of err's message, +// keeping its status code so the HTTP mapping and the caller's +// classification still apply. +func prefixError(where string, err error) error { + st, ok := status.FromError(err) + if !ok { + return status.Errorf(codes.Internal, "%s: %v", where, err) + } + return status.Error(st.Code(), fmt.Sprintf("%s: %s", where, st.Message())) +} diff --git a/internal/benchmarking/glutton/script_test.go b/internal/benchmarking/glutton/script_test.go new file mode 100644 index 0000000000..22ac88d51b --- /dev/null +++ b/internal/benchmarking/glutton/script_test.go @@ -0,0 +1,321 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package glutton + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" +) + +// Script builders, so the tests read like the trees they send. + +func writeDisk(key string, size int32) *gluttonpb.Request { + return &gluttonpb.Request{Kind: &gluttonpb.Request_WriteDisk{WriteDisk: &gluttonpb.WriteDiskRequest{Key: key, Size: size}}} +} + +func readDisk(key string) *gluttonpb.Request { + return &gluttonpb.Request{Kind: &gluttonpb.Request_ReadDisk{ReadDisk: &gluttonpb.ReadDiskRequest{Key: key, ReadMode: gluttonpb.ReadMode_READ_MODE_DIGEST_ONLY}}} +} + +func writeRAM(key, size string) *gluttonpb.Request { + return &gluttonpb.Request{Kind: &gluttonpb.Request_WriteRam{WriteRam: &gluttonpb.WriteRAMRequest{Key: key, Size: size}}} +} + +func readRAM(key string) *gluttonpb.Request { + return &gluttonpb.Request{Kind: &gluttonpb.Request_ReadRam{ReadRam: &gluttonpb.ReadRAMRequest{Key: key}}} +} + +func burn(ms int64) *gluttonpb.Request { + return &gluttonpb.Request{Kind: &gluttonpb.Request_BurnCpu{BurnCpu: &gluttonpb.BurnCPURequest{DurationMs: ms}}} +} + +func sleepFor(ms int64) *gluttonpb.Request { + return &gluttonpb.Request{Kind: &gluttonpb.Request_Sleep{Sleep: &gluttonpb.SleepRequest{DurationMs: ms}}} +} + +// op is a step holding one operation of the given requests, run together. +func op(requests ...*gluttonpb.Request) *gluttonpb.Step { + return &gluttonpb.Step{Kind: &gluttonpb.Step_Operation{Operation: &gluttonpb.Operation{Requests: requests}}} +} + +// seq is a step holding a block of the given steps, run in order once. +func seq(steps ...*gluttonpb.Step) *gluttonpb.Step { + return &gluttonpb.Step{Kind: &gluttonpb.Step_Block{Block: &gluttonpb.Block{Steps: steps}}} +} + +// loop is a step holding a block that cycles its steps for budgetMs. +func loop(budgetMs int64, steps ...*gluttonpb.Step) *gluttonpb.Step { + return &gluttonpb.Step{Kind: &gluttonpb.Step_Block{Block: &gluttonpb.Block{Steps: steps, LoopDurationMs: budgetMs}}} +} + +func script(steps ...*gluttonpb.Step) *gluttonpb.RunScriptRequest { + return &gluttonpb.RunScriptRequest{Script: &gluttonpb.Block{Steps: steps}} +} + +func run(t *testing.T, svc *Service, req *gluttonpb.RunScriptRequest) *gluttonpb.StepResult { + t.Helper() + resp, err := svc.RunScript(context.Background(), req) + if err != nil { + t.Fatalf("RunScript: %v", err) + } + return resp.GetResult() +} + +// Steps run in order, each request has its usual effect, and the result +// mirrors the tree with the totals summed to the root. +func TestRunScriptRunsStepsInOrderAndSumsStats(t *testing.T) { + svc := newTestService(t) + result := run(t, svc, script( + op(writeDisk("scripted", 1024)), + op(readDisk("scripted")), + seq(op(writeRAM("arr", "8Ki")), op(readRAM("arr"))), + op(burn(1), sleepFor(1)), + )) + root := result.GetStats() + if root.GetPasses() != 1 || root.GetRequestsRun() != 6 { + t.Errorf("root passes=%d requests_run=%d, want 1/6", root.GetPasses(), root.GetRequestsRun()) + } + if root.GetDiskBytesWritten() != 1024 || root.GetDiskBytesRead() != 1024 { + t.Errorf("disk bytes written=%d read=%d, want 1024/1024", root.GetDiskBytesWritten(), root.GetDiskBytesRead()) + } + if root.GetRamBytesWritten() != 8192 || root.GetRamBytesRead() != 8192 { + t.Errorf("ram bytes written=%d read=%d, want 8192/8192", root.GetRamBytesWritten(), root.GetRamBytesRead()) + } + if root.GetBurnIterations() < 1 || root.GetSleptMs() < 1 { + t.Errorf("burn_iterations=%d slept_ms=%d, want both positive", root.GetBurnIterations(), root.GetSleptMs()) + } + if len(result.GetChildren()) != 4 { + t.Fatalf("root has %d children, want 4 steps", len(result.GetChildren())) + } + nested := result.GetChildren()[2] + if len(nested.GetChildren()) != 2 || nested.GetStats().GetRequestsRun() != 2 || nested.GetStats().GetRamBytesWritten() != 8192 { + t.Errorf("nested block = %v, want two children and the RAM totals", nested) + } + // An operation's children are its requests, each with its own stats. + parallel := result.GetChildren()[3] + if len(parallel.GetChildren()) != 2 || parallel.GetChildren()[0].GetStats().GetBurnIterations() < 1 || parallel.GetChildren()[1].GetStats().GetSleptMs() < 1 { + t.Errorf("operation = %v, want a burn child and a sleep child", parallel) + } +} + +// Requests in one operation overlap: two sleeps of 100ms take about 100ms, +// and the operation's elapsed is its longest request, not their sum. +func TestRunScriptOperationRunsRequestsTogether(t *testing.T) { + svc := newTestService(t) + start := time.Now() + result := run(t, svc, script(op(sleepFor(100), sleepFor(100), sleepFor(100)))) + if elapsed := time.Since(start); elapsed > 250*time.Millisecond { + t.Errorf("three parallel 100ms sleeps took %v", elapsed) + } + stats := result.GetChildren()[0].GetStats() + if stats.GetSleptMs() < 300 || stats.GetElapsedMs() > 250 { + t.Errorf("operation slept_ms=%d elapsed_ms=%d, want ~300 summed and ~100 wall", stats.GetSleptMs(), stats.GetElapsedMs()) + } +} + +// A looped block cycles until its budget and finishes the pass it is in: +// every pass is whole, the loop overruns by at most one pass, and the +// stats count every pass. +func TestRunScriptLoopsUntilTheBudget(t *testing.T) { + svc := newTestService(t) + start := time.Now() + result := run(t, svc, script(loop(100, op(sleepFor(30)), op(writeDisk("churn", 16))))) + elapsed := time.Since(start) + if elapsed < 100*time.Millisecond || elapsed > 300*time.Millisecond { + t.Errorf("a 100ms loop returned after %v", elapsed) + } + looped := result.GetChildren()[0].GetStats() + if looped.GetPasses() < 4 || looped.GetPasses() > 5 { + t.Errorf("passes = %d, want 4 whole 30ms passes to cover 100ms", looped.GetPasses()) + } + // Whole passes: as many writes as sleeps, and the sleeps complete. + if looped.GetDiskBytesWritten() != 16*looped.GetPasses() || looped.GetSleptMs() < 30*(looped.GetPasses()-1) { + t.Errorf("loop stats = %v, want %d whole passes", looped, looped.GetPasses()) + } + if root := result.GetStats(); root.GetPasses() != 1 || root.GetDiskBytesWritten() != looped.GetDiskBytesWritten() { + t.Errorf("root = %v, want one pass and the loop's totals", root) + } +} + +// An inner loop stops cycling once the outer one's budget has elapsed, +// finishing its pass; that is not an error. +func TestRunScriptNestedLoopsShareTheBudget(t *testing.T) { + svc := newTestService(t) + start := time.Now() + result := run(t, svc, script(loop(100, loop(5000, op(sleepFor(20)))))) + if elapsed := time.Since(start); elapsed > 300*time.Millisecond { + t.Errorf("a 5s loop inside a 100ms loop ran for %v", elapsed) + } + inner := result.GetChildren()[0].GetChildren()[0].GetStats() + if inner.GetPasses() < 5 || inner.GetSleptMs() < 20*inner.GetPasses() { + t.Errorf("inner loop = %v, want ~5 whole 20ms passes", inner) + } +} + +// Shape problems are rejected before any step runs, with the node named. +func TestRunScriptRejectsShape(t *testing.T) { + svc := newTestService(t) + for _, tc := range []struct { + name string + req *gluttonpb.RunScriptRequest + msg string + }{ + {"no script", &gluttonpb.RunScriptRequest{}, "script is required"}, + {"empty block", script(), "script: steps is required"}, + {"unset step", script(&gluttonpb.Step{}), "script.steps[0]: no kind set"}, + {"empty operation", script(op()), "script.steps[0].operation: requests is required"}, + {"unset request", script(op(&gluttonpb.Request{})), "script.steps[0].operation.requests[0]: no kind set"}, + {"negative sleep", script(op(sleepFor(-1))), "requests[0] (sleep)"}, + {"negative budget", script(loop(-1, op(sleepFor(0)))), "script.steps[0].block: loop_duration_ms"}, + {"deep empty block", script(seq(seq(seq()))), "script.steps[0].block.steps[0].block.steps[0].block: steps is required"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := svc.RunScript(context.Background(), tc.req) + if status.Code(err) != codes.InvalidArgument { + t.Fatalf("error = %v, want InvalidArgument", err) + } + if !strings.Contains(err.Error(), tc.msg) { + t.Errorf("error %q does not mention %q", err, tc.msg) + } + }) + } + if _, err := svc.ReadDisk(context.Background(), &gluttonpb.ReadDiskRequest{Key: "scripted"}); status.Code(err) != codes.NotFound { + t.Errorf("a rejected script left a file behind (ReadDisk err = %v)", err) + } +} + +// A request's own error keeps its code and names the request's path; the +// script stops there. +func TestRunScriptRequestErrorsKeepTheirCode(t *testing.T) { + svc := newTestService(t) + for _, tc := range []struct { + name string + req *gluttonpb.RunScriptRequest + code codes.Code + msg string + }{ + {"bad key", script(op(sleepFor(0)), op(writeDisk("../x", 1))), codes.InvalidArgument, "script.steps[1].operation.requests[0] (write_disk)"}, + {"missing file", script(seq(op(readDisk("nothing")))), codes.NotFound, "script.steps[0].block.steps[0].operation.requests[0] (read_disk)"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := svc.RunScript(context.Background(), tc.req) + st, ok := status.FromError(err) + if !ok || st.Code() != tc.code { + t.Fatalf("error = %v, want code %v", err, tc.code) + } + if !strings.Contains(st.Message(), tc.msg) { + t.Errorf("message %q does not mention %q", st.Message(), tc.msg) + } + }) + } + // A failing request inside a loop is an error, not the end of the loop. + _, err := svc.RunScript(context.Background(), script(loop(100, op(readDisk("nothing"))))) + if status.Code(err) != codes.NotFound { + t.Errorf("a failing request in a loop returned %v, want NotFound", err) + } + // One failing request cancels its siblings in the operation. + start := time.Now() + _, err = svc.RunScript(context.Background(), script(op(readDisk("nothing"), sleepFor(5000)))) + if status.Code(err) != codes.NotFound || time.Since(start) > time.Second { + t.Errorf("sibling failure: err %v after %v, want NotFound promptly", err, time.Since(start)) + } +} + +// A caller that goes away ends the script, mid-sleep or at the next +// request, so a disconnected request does not keep the sandbox busy. That +// is an error, unlike a budget running out. +func TestRunScriptStopsWhenTheContextEnds(t *testing.T) { + svc := newTestService(t) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + start := time.Now() + _, err := svc.RunScript(ctx, script(loop(10000, op(sleepFor(5000))))) + if status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("error = %v, want DeadlineExceeded", err) + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Errorf("took %v to notice the context ended", elapsed) + } +} + +// Requests that do not watch the context, such as RAM and disk writes, run +// to completion; the block still stops at the next step rather than cycling +// until its budget, so a looped block of them cannot outlive its caller. +func TestRunScriptStopsBetweenStepsWhenTheContextEnds(t *testing.T) { + svc := newTestService(t) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + start := time.Now() + _, err := svc.RunScript(ctx, script(loop(10000, op(writeRAM("churn", "4Ki")), op(writeDisk("churn", 16)), op(burn(1))))) + if status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("error = %v, want DeadlineExceeded", err) + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Errorf("took %v to notice the context ended", elapsed) + } +} + +// The HTTP route carries the same request and maps a request's status code +// the way every other route does. +func TestRunScriptHTTPRoute(t *testing.T) { + svc := newTestService(t) + handler, err := Handler(ModeHTTP, svc) + if err != nil { + t.Fatal(err) + } + ts := httptest.NewServer(handler) + defer ts.Close() + + post := func(req *gluttonpb.RunScriptRequest) (*http.Response, []byte) { + t.Helper() + body, _ := proto.Marshal(req) + res, err := http.Post(ts.URL+RunScriptRoute, "application/x-protobuf", bytes.NewReader(body)) + if err != nil { + t.Fatalf("POST %s: %v", RunScriptRoute, err) + } + defer res.Body.Close() + out, _ := io.ReadAll(res.Body) + return res, out + } + + res, body := post(script(op(writeDisk("viahttp", 64)), op(readDisk("viahttp")))) + if res.StatusCode != http.StatusOK { + t.Fatalf("status %d: %s", res.StatusCode, body) + } + var resp gluttonpb.RunScriptResponse + if err := proto.Unmarshal(body, &resp); err != nil { + t.Fatal(err) + } + if resp.GetResult().GetStats().GetRequestsRun() != 2 { + t.Errorf("requests_run = %d, want 2", resp.GetResult().GetStats().GetRequestsRun()) + } + + res, body = post(script(op(readDisk("absent")))) + if res.StatusCode != http.StatusNotFound || !strings.Contains(string(body), "(read_disk)") { + t.Errorf("missing file over HTTP: status %d body %q, want 404 naming the request", res.StatusCode, body) + } +} diff --git a/internal/benchmarking/glutton/server.go b/internal/benchmarking/glutton/server.go index 03f625d285..a7eab72586 100644 --- a/internal/benchmarking/glutton/server.go +++ b/internal/benchmarking/glutton/server.go @@ -121,6 +121,7 @@ func newMux(svc *Service) *http.ServeMux { mux.HandleFunc(BurnCPURoute, protoRoute("BurnCPU", svc.BurnCPU)) mux.HandleFunc(IngestRoute, protoRoute("Ingest", svc.Ingest)) mux.HandleFunc(UseCPURoute, protoRoute("UseCPU", svc.UseCPU)) + mux.HandleFunc(RunScriptRoute, protoRoute("RunScript", svc.RunScript)) return mux } diff --git a/internal/proto/glutton/glutton.pb.go b/internal/proto/glutton/glutton.pb.go index f7be48fa91..490941706a 100644 --- a/internal/proto/glutton/glutton.pb.go +++ b/internal/proto/glutton/glutton.pb.go @@ -1222,6 +1222,676 @@ func (x *UseCPUResponse) GetNumCores() int32 { return 0 } +// A Block is a list of steps run in order. With loop_duration_ms set, the +// block cycles through its steps until the budget elapses. A budget is +// checked between passes: the pass in progress finishes, so a loop +// overruns its budget by at most one pass and no request is cut short. +// A nested block stops cycling once any ancestor's budget has elapsed, +// the same way. +type Block struct { + state protoimpl.MessageState `protogen:"open.v1"` + // At least one step is required. + Steps []*Step `protobuf:"bytes,1,rep,name=steps,proto3" json:"steps,omitempty"` + // Wall-clock budget to cycle through the steps for, in milliseconds; + // the pass under way when it elapses finishes. Zero runs the steps once. + LoopDurationMs int64 `protobuf:"varint,2,opt,name=loop_duration_ms,json=loopDurationMs,proto3" json:"loop_duration_ms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Block) Reset() { + *x = Block{} + mi := &file_glutton_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Block) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Block) ProtoMessage() {} + +func (x *Block) ProtoReflect() protoreflect.Message { + mi := &file_glutton_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Block.ProtoReflect.Descriptor instead. +func (*Block) Descriptor() ([]byte, []int) { + return file_glutton_proto_rawDescGZIP(), []int{21} +} + +func (x *Block) GetSteps() []*Step { + if x != nil { + return x.Steps + } + return nil +} + +func (x *Block) GetLoopDurationMs() int64 { + if x != nil { + return x.LoopDurationMs + } + return 0 +} + +// A Step is either a nested Block or an Operation. +type Step struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Kind: + // + // *Step_Block + // *Step_Operation + Kind isStep_Kind `protobuf_oneof:"kind"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Step) Reset() { + *x = Step{} + mi := &file_glutton_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Step) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Step) ProtoMessage() {} + +func (x *Step) ProtoReflect() protoreflect.Message { + mi := &file_glutton_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Step.ProtoReflect.Descriptor instead. +func (*Step) Descriptor() ([]byte, []int) { + return file_glutton_proto_rawDescGZIP(), []int{22} +} + +func (x *Step) GetKind() isStep_Kind { + if x != nil { + return x.Kind + } + return nil +} + +func (x *Step) GetBlock() *Block { + if x != nil { + if x, ok := x.Kind.(*Step_Block); ok { + return x.Block + } + } + return nil +} + +func (x *Step) GetOperation() *Operation { + if x != nil { + if x, ok := x.Kind.(*Step_Operation); ok { + return x.Operation + } + } + return nil +} + +type isStep_Kind interface { + isStep_Kind() +} + +type Step_Block struct { + Block *Block `protobuf:"bytes,1,opt,name=block,proto3,oneof"` +} + +type Step_Operation struct { + Operation *Operation `protobuf:"bytes,2,opt,name=operation,proto3,oneof"` +} + +func (*Step_Block) isStep_Kind() {} + +func (*Step_Operation) isStep_Kind() {} + +// An Operation runs its requests at the same time and waits for all of +// them. The first request to fail cancels the rest. RAM requests take +// turns on the glutton's single RAM lock, so only CPU, disk, and sleep +// requests overlap in practice. +type Operation struct { + state protoimpl.MessageState `protogen:"open.v1"` + // At least one request is required. + Requests []*Request `protobuf:"bytes,1,rep,name=requests,proto3" json:"requests,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Operation) Reset() { + *x = Operation{} + mi := &file_glutton_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Operation) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Operation) ProtoMessage() {} + +func (x *Operation) ProtoReflect() protoreflect.Message { + mi := &file_glutton_proto_msgTypes[23] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Operation.ProtoReflect.Descriptor instead. +func (*Operation) Descriptor() ([]byte, []int) { + return file_glutton_proto_rawDescGZIP(), []int{23} +} + +func (x *Operation) GetRequests() []*Request { + if x != nil { + return x.Requests + } + return nil +} + +// A Request is one of the service's work requests. UseCPU and OpenFD are +// left out because they set a background load that outlives the script +// (BurnCPU is the CPU spike), and Gossip because it starts a loop. +type Request struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Kind: + // + // *Request_WriteRam + // *Request_ReadRam + // *Request_WriteDisk + // *Request_ReadDisk + // *Request_BurnCpu + // *Request_Ingest + // *Request_Sleep + Kind isRequest_Kind `protobuf_oneof:"kind"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Request) Reset() { + *x = Request{} + mi := &file_glutton_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Request) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Request) ProtoMessage() {} + +func (x *Request) ProtoReflect() protoreflect.Message { + mi := &file_glutton_proto_msgTypes[24] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Request.ProtoReflect.Descriptor instead. +func (*Request) Descriptor() ([]byte, []int) { + return file_glutton_proto_rawDescGZIP(), []int{24} +} + +func (x *Request) GetKind() isRequest_Kind { + if x != nil { + return x.Kind + } + return nil +} + +func (x *Request) GetWriteRam() *WriteRAMRequest { + if x != nil { + if x, ok := x.Kind.(*Request_WriteRam); ok { + return x.WriteRam + } + } + return nil +} + +func (x *Request) GetReadRam() *ReadRAMRequest { + if x != nil { + if x, ok := x.Kind.(*Request_ReadRam); ok { + return x.ReadRam + } + } + return nil +} + +func (x *Request) GetWriteDisk() *WriteDiskRequest { + if x != nil { + if x, ok := x.Kind.(*Request_WriteDisk); ok { + return x.WriteDisk + } + } + return nil +} + +func (x *Request) GetReadDisk() *ReadDiskRequest { + if x != nil { + if x, ok := x.Kind.(*Request_ReadDisk); ok { + return x.ReadDisk + } + } + return nil +} + +func (x *Request) GetBurnCpu() *BurnCPURequest { + if x != nil { + if x, ok := x.Kind.(*Request_BurnCpu); ok { + return x.BurnCpu + } + } + return nil +} + +func (x *Request) GetIngest() *IngestRequest { + if x != nil { + if x, ok := x.Kind.(*Request_Ingest); ok { + return x.Ingest + } + } + return nil +} + +func (x *Request) GetSleep() *SleepRequest { + if x != nil { + if x, ok := x.Kind.(*Request_Sleep); ok { + return x.Sleep + } + } + return nil +} + +type isRequest_Kind interface { + isRequest_Kind() +} + +type Request_WriteRam struct { + WriteRam *WriteRAMRequest `protobuf:"bytes,1,opt,name=write_ram,json=writeRam,proto3,oneof"` +} + +type Request_ReadRam struct { + ReadRam *ReadRAMRequest `protobuf:"bytes,2,opt,name=read_ram,json=readRam,proto3,oneof"` +} + +type Request_WriteDisk struct { + WriteDisk *WriteDiskRequest `protobuf:"bytes,3,opt,name=write_disk,json=writeDisk,proto3,oneof"` +} + +type Request_ReadDisk struct { + ReadDisk *ReadDiskRequest `protobuf:"bytes,4,opt,name=read_disk,json=readDisk,proto3,oneof"` +} + +type Request_BurnCpu struct { + BurnCpu *BurnCPURequest `protobuf:"bytes,5,opt,name=burn_cpu,json=burnCpu,proto3,oneof"` +} + +type Request_Ingest struct { + Ingest *IngestRequest `protobuf:"bytes,6,opt,name=ingest,proto3,oneof"` +} + +type Request_Sleep struct { + Sleep *SleepRequest `protobuf:"bytes,7,opt,name=sleep,proto3,oneof"` +} + +func (*Request_WriteRam) isRequest_Kind() {} + +func (*Request_ReadRam) isRequest_Kind() {} + +func (*Request_WriteDisk) isRequest_Kind() {} + +func (*Request_ReadDisk) isRequest_Kind() {} + +func (*Request_BurnCpu) isRequest_Kind() {} + +func (*Request_Ingest) isRequest_Kind() {} + +func (*Request_Sleep) isRequest_Kind() {} + +type SleepRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Wall-clock time to idle, in milliseconds. The glutton stays awake and + // does nothing, which paces the requests around it. + DurationMs int64 `protobuf:"varint,1,opt,name=duration_ms,json=durationMs,proto3" json:"duration_ms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SleepRequest) Reset() { + *x = SleepRequest{} + mi := &file_glutton_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SleepRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SleepRequest) ProtoMessage() {} + +func (x *SleepRequest) ProtoReflect() protoreflect.Message { + mi := &file_glutton_proto_msgTypes[25] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SleepRequest.ProtoReflect.Descriptor instead. +func (*SleepRequest) Descriptor() ([]byte, []int) { + return file_glutton_proto_rawDescGZIP(), []int{25} +} + +func (x *SleepRequest) GetDurationMs() int64 { + if x != nil { + return x.DurationMs + } + return 0 +} + +type RunScriptRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Script *Block `protobuf:"bytes,1,opt,name=script,proto3" json:"script,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RunScriptRequest) Reset() { + *x = RunScriptRequest{} + mi := &file_glutton_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RunScriptRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RunScriptRequest) ProtoMessage() {} + +func (x *RunScriptRequest) ProtoReflect() protoreflect.Message { + mi := &file_glutton_proto_msgTypes[26] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RunScriptRequest.ProtoReflect.Descriptor instead. +func (*RunScriptRequest) Descriptor() ([]byte, []int) { + return file_glutton_proto_rawDescGZIP(), []int{26} +} + +func (x *RunScriptRequest) GetScript() *Block { + if x != nil { + return x.Script + } + return nil +} + +type RunScriptResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The script's root block. + Result *StepResult `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RunScriptResponse) Reset() { + *x = RunScriptResponse{} + mi := &file_glutton_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RunScriptResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RunScriptResponse) ProtoMessage() {} + +func (x *RunScriptResponse) ProtoReflect() protoreflect.Message { + mi := &file_glutton_proto_msgTypes[27] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RunScriptResponse.ProtoReflect.Descriptor instead. +func (*RunScriptResponse) Descriptor() ([]byte, []int) { + return file_glutton_proto_rawDescGZIP(), []int{27} +} + +func (x *RunScriptResponse) GetResult() *StepResult { + if x != nil { + return x.Result + } + return nil +} + +// A StepResult mirrors one node of the script: a block, an operation, or +// a request. Its children are in the request's order: a block's steps or +// an operation's requests; a request has none. A node inside a looped +// block is entered once per pass, and its stats sum over every entry, so +// the response is the size of the script however long it ran. +type StepResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + Stats *Stats `protobuf:"bytes,1,opt,name=stats,proto3" json:"stats,omitempty"` + Children []*StepResult `protobuf:"bytes,2,rep,name=children,proto3" json:"children,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StepResult) Reset() { + *x = StepResult{} + mi := &file_glutton_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StepResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StepResult) ProtoMessage() {} + +func (x *StepResult) ProtoReflect() protoreflect.Message { + mi := &file_glutton_proto_msgTypes[28] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StepResult.ProtoReflect.Descriptor instead. +func (*StepResult) Descriptor() ([]byte, []int) { + return file_glutton_proto_rawDescGZIP(), []int{28} +} + +func (x *StepResult) GetStats() *Stats { + if x != nil { + return x.Stats + } + return nil +} + +func (x *StepResult) GetChildren() []*StepResult { + if x != nil { + return x.Children + } + return nil +} + +// Stats are a node's vital signs. Every field but elapsed_ms is the sum +// over the node's children and entries, so the root carries the totals. +type Stats struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Wall clock spent in this node, summed over its entries. A block's is + // the sum of its steps'; an operation's is its longest request's, since + // they overlap; so a parent's elapsed is not the sum of its children's. + ElapsedMs int64 `protobuf:"varint,1,opt,name=elapsed_ms,json=elapsedMs,proto3" json:"elapsed_ms,omitempty"` + // Times this node was entered: a looped block's pass count, and for + // every other node the entries its enclosing loops gave it. + Passes int64 `protobuf:"varint,2,opt,name=passes,proto3" json:"passes,omitempty"` + // Requests run beneath this node. + RequestsRun int64 `protobuf:"varint,3,opt,name=requests_run,json=requestsRun,proto3" json:"requests_run,omitempty"` + // Bytes WriteDisk generated and Ingest received, and bytes ReadDisk read. + DiskBytesWritten int64 `protobuf:"varint,4,opt,name=disk_bytes_written,json=diskBytesWritten,proto3" json:"disk_bytes_written,omitempty"` + DiskBytesRead int64 `protobuf:"varint,5,opt,name=disk_bytes_read,json=diskBytesRead,proto3" json:"disk_bytes_read,omitempty"` + // Bytes WriteRAM was asked to write, and bytes ReadRAM walked. + RamBytesWritten int64 `protobuf:"varint,6,opt,name=ram_bytes_written,json=ramBytesWritten,proto3" json:"ram_bytes_written,omitempty"` + RamBytesRead int64 `protobuf:"varint,7,opt,name=ram_bytes_read,json=ramBytesRead,proto3" json:"ram_bytes_read,omitempty"` + // Hash iterations BurnCPU completed. + BurnIterations int64 `protobuf:"varint,8,opt,name=burn_iterations,json=burnIterations,proto3" json:"burn_iterations,omitempty"` + // Milliseconds slept. + SleptMs int64 `protobuf:"varint,9,opt,name=slept_ms,json=sleptMs,proto3" json:"slept_ms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Stats) Reset() { + *x = Stats{} + mi := &file_glutton_proto_msgTypes[29] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Stats) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Stats) ProtoMessage() {} + +func (x *Stats) ProtoReflect() protoreflect.Message { + mi := &file_glutton_proto_msgTypes[29] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Stats.ProtoReflect.Descriptor instead. +func (*Stats) Descriptor() ([]byte, []int) { + return file_glutton_proto_rawDescGZIP(), []int{29} +} + +func (x *Stats) GetElapsedMs() int64 { + if x != nil { + return x.ElapsedMs + } + return 0 +} + +func (x *Stats) GetPasses() int64 { + if x != nil { + return x.Passes + } + return 0 +} + +func (x *Stats) GetRequestsRun() int64 { + if x != nil { + return x.RequestsRun + } + return 0 +} + +func (x *Stats) GetDiskBytesWritten() int64 { + if x != nil { + return x.DiskBytesWritten + } + return 0 +} + +func (x *Stats) GetDiskBytesRead() int64 { + if x != nil { + return x.DiskBytesRead + } + return 0 +} + +func (x *Stats) GetRamBytesWritten() int64 { + if x != nil { + return x.RamBytesWritten + } + return 0 +} + +func (x *Stats) GetRamBytesRead() int64 { + if x != nil { + return x.RamBytesRead + } + return 0 +} + +func (x *Stats) GetBurnIterations() int64 { + if x != nil { + return x.BurnIterations + } + return 0 +} + +func (x *Stats) GetSleptMs() int64 { + if x != nil { + return x.SleptMs + } + return 0 +} + var File_glutton_proto protoreflect.FileDescriptor const file_glutton_proto_rawDesc = "" + @@ -1290,14 +1960,55 @@ const file_glutton_proto_rawDesc = "" + "\x11cap_at_gomaxprocs\x18\x04 \x01(\bH\x00R\x0fcapAtGomaxprocs\x88\x01\x01B\x14\n" + "\x12_cap_at_gomaxprocs\"-\n" + "\x0eUseCPUResponse\x12\x1b\n" + - "\tnum_cores\x18\x01 \x01(\x05R\bnumCores*_\n" + + "\tnum_cores\x18\x01 \x01(\x05R\bnumCores\"V\n" + + "\x05Block\x12#\n" + + "\x05steps\x18\x01 \x03(\v2\r.glutton.StepR\x05steps\x12(\n" + + "\x10loop_duration_ms\x18\x02 \x01(\x03R\x0eloopDurationMs\"j\n" + + "\x04Step\x12&\n" + + "\x05block\x18\x01 \x01(\v2\x0e.glutton.BlockH\x00R\x05block\x122\n" + + "\toperation\x18\x02 \x01(\v2\x12.glutton.OperationH\x00R\toperationB\x06\n" + + "\x04kind\"9\n" + + "\tOperation\x12,\n" + + "\brequests\x18\x01 \x03(\v2\x10.glutton.RequestR\brequests\"\x8c\x03\n" + + "\aRequest\x127\n" + + "\twrite_ram\x18\x01 \x01(\v2\x18.glutton.WriteRAMRequestH\x00R\bwriteRam\x124\n" + + "\bread_ram\x18\x02 \x01(\v2\x17.glutton.ReadRAMRequestH\x00R\areadRam\x12:\n" + + "\n" + + "write_disk\x18\x03 \x01(\v2\x19.glutton.WriteDiskRequestH\x00R\twriteDisk\x127\n" + + "\tread_disk\x18\x04 \x01(\v2\x18.glutton.ReadDiskRequestH\x00R\breadDisk\x124\n" + + "\bburn_cpu\x18\x05 \x01(\v2\x17.glutton.BurnCPURequestH\x00R\aburnCpu\x120\n" + + "\x06ingest\x18\x06 \x01(\v2\x16.glutton.IngestRequestH\x00R\x06ingest\x12-\n" + + "\x05sleep\x18\a \x01(\v2\x15.glutton.SleepRequestH\x00R\x05sleepB\x06\n" + + "\x04kind\"/\n" + + "\fSleepRequest\x12\x1f\n" + + "\vduration_ms\x18\x01 \x01(\x03R\n" + + "durationMs\":\n" + + "\x10RunScriptRequest\x12&\n" + + "\x06script\x18\x01 \x01(\v2\x0e.glutton.BlockR\x06script\"@\n" + + "\x11RunScriptResponse\x12+\n" + + "\x06result\x18\x01 \x01(\v2\x13.glutton.StepResultR\x06result\"c\n" + + "\n" + + "StepResult\x12$\n" + + "\x05stats\x18\x01 \x01(\v2\x0e.glutton.StatsR\x05stats\x12/\n" + + "\bchildren\x18\x02 \x03(\v2\x13.glutton.StepResultR\bchildren\"\xcd\x02\n" + + "\x05Stats\x12\x1d\n" + + "\n" + + "elapsed_ms\x18\x01 \x01(\x03R\telapsedMs\x12\x16\n" + + "\x06passes\x18\x02 \x01(\x03R\x06passes\x12!\n" + + "\frequests_run\x18\x03 \x01(\x03R\vrequestsRun\x12,\n" + + "\x12disk_bytes_written\x18\x04 \x01(\x03R\x10diskBytesWritten\x12&\n" + + "\x0fdisk_bytes_read\x18\x05 \x01(\x03R\rdiskBytesRead\x12*\n" + + "\x11ram_bytes_written\x18\x06 \x01(\x03R\x0framBytesWritten\x12$\n" + + "\x0eram_bytes_read\x18\a \x01(\x03R\framBytesRead\x12'\n" + + "\x0fburn_iterations\x18\b \x01(\x03R\x0eburnIterations\x12\x19\n" + + "\bslept_ms\x18\t \x01(\x03R\asleptMs*_\n" + "\tWriteMode\x12\x17\n" + "\x13WRITE_MODE_TRUNCATE\x10\x00\x12\x18\n" + "\x14WRITE_MODE_OVERWRITE\x10\x01\x12\x1f\n" + "\x1bWRITE_MODE_OVERWRITE_ROTATE\x10\x02*9\n" + "\bReadMode\x12\x12\n" + "\x0eREAD_MODE_DATA\x10\x00\x12\x19\n" + - "\x15READ_MODE_DIGEST_ONLY\x10\x012\x80\x05\n" + + "\x15READ_MODE_DIGEST_ONLY\x10\x012\xc6\x05\n" + "\aGlutton\x12A\n" + "\bWriteRAM\x12\x18.glutton.WriteRAMRequest\x1a\x19.glutton.WriteRAMResponse\"\x00\x12>\n" + "\aReadRAM\x12\x17.glutton.ReadRAMRequest\x1a\x18.glutton.ReadRAMResponse\"\x00\x12D\n" + @@ -1308,7 +2019,8 @@ const file_glutton_proto_rawDesc = "" + "\x06Gossip\x12\x16.glutton.GossipRequest\x1a\x17.glutton.GossipResponse\"\x00\x12>\n" + "\aBurnCPU\x12\x17.glutton.BurnCPURequest\x1a\x18.glutton.BurnCPUResponse\"\x00\x12;\n" + "\x06Ingest\x12\x16.glutton.IngestRequest\x1a\x17.glutton.IngestResponse\"\x00\x12;\n" + - "\x06UseCPU\x12\x16.glutton.UseCPURequest\x1a\x17.glutton.UseCPUResponse\"\x00B-Z+go.ate.dev/substrate/internal/proto/gluttonb\x06proto3" + "\x06UseCPU\x12\x16.glutton.UseCPURequest\x1a\x17.glutton.UseCPUResponse\"\x00\x12D\n" + + "\tRunScript\x12\x19.glutton.RunScriptRequest\x1a\x1a.glutton.RunScriptResponse\"\x00B-Z+go.ate.dev/substrate/internal/proto/gluttonb\x06proto3" var ( file_glutton_proto_rawDescOnce sync.Once @@ -1323,7 +2035,7 @@ func file_glutton_proto_rawDescGZIP() []byte { } var file_glutton_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_glutton_proto_msgTypes = make([]protoimpl.MessageInfo, 21) +var file_glutton_proto_msgTypes = make([]protoimpl.MessageInfo, 30) var file_glutton_proto_goTypes = []any{ (WriteMode)(0), // 0: glutton.WriteMode (ReadMode)(0), // 1: glutton.ReadMode @@ -1348,37 +2060,63 @@ var file_glutton_proto_goTypes = []any{ (*IngestResponse)(nil), // 20: glutton.IngestResponse (*UseCPURequest)(nil), // 21: glutton.UseCPURequest (*UseCPUResponse)(nil), // 22: glutton.UseCPUResponse + (*Block)(nil), // 23: glutton.Block + (*Step)(nil), // 24: glutton.Step + (*Operation)(nil), // 25: glutton.Operation + (*Request)(nil), // 26: glutton.Request + (*SleepRequest)(nil), // 27: glutton.SleepRequest + (*RunScriptRequest)(nil), // 28: glutton.RunScriptRequest + (*RunScriptResponse)(nil), // 29: glutton.RunScriptResponse + (*StepResult)(nil), // 30: glutton.StepResult + (*Stats)(nil), // 31: glutton.Stats } var file_glutton_proto_depIdxs = []int32{ 0, // 0: glutton.WriteRAMRequest.write_mode:type_name -> glutton.WriteMode 0, // 1: glutton.WriteDiskRequest.write_mode:type_name -> glutton.WriteMode 1, // 2: glutton.ReadDiskRequest.read_mode:type_name -> glutton.ReadMode 16, // 3: glutton.GossipRequest.peers:type_name -> glutton.Peer - 2, // 4: glutton.Glutton.WriteRAM:input_type -> glutton.WriteRAMRequest - 4, // 5: glutton.Glutton.ReadRAM:input_type -> glutton.ReadRAMRequest - 6, // 6: glutton.Glutton.WriteDisk:input_type -> glutton.WriteDiskRequest - 8, // 7: glutton.Glutton.ReadDisk:input_type -> glutton.ReadDiskRequest - 10, // 8: glutton.Glutton.OpenFD:input_type -> glutton.OpenFDRequest - 12, // 9: glutton.Glutton.Ping:input_type -> glutton.PingRequest - 14, // 10: glutton.Glutton.Gossip:input_type -> glutton.GossipRequest - 17, // 11: glutton.Glutton.BurnCPU:input_type -> glutton.BurnCPURequest - 19, // 12: glutton.Glutton.Ingest:input_type -> glutton.IngestRequest - 21, // 13: glutton.Glutton.UseCPU:input_type -> glutton.UseCPURequest - 3, // 14: glutton.Glutton.WriteRAM:output_type -> glutton.WriteRAMResponse - 5, // 15: glutton.Glutton.ReadRAM:output_type -> glutton.ReadRAMResponse - 7, // 16: glutton.Glutton.WriteDisk:output_type -> glutton.WriteDiskResponse - 9, // 17: glutton.Glutton.ReadDisk:output_type -> glutton.ReadDiskResponse - 11, // 18: glutton.Glutton.OpenFD:output_type -> glutton.OpenFDResponse - 13, // 19: glutton.Glutton.Ping:output_type -> glutton.PingResponse - 15, // 20: glutton.Glutton.Gossip:output_type -> glutton.GossipResponse - 18, // 21: glutton.Glutton.BurnCPU:output_type -> glutton.BurnCPUResponse - 20, // 22: glutton.Glutton.Ingest:output_type -> glutton.IngestResponse - 22, // 23: glutton.Glutton.UseCPU:output_type -> glutton.UseCPUResponse - 14, // [14:24] is the sub-list for method output_type - 4, // [4:14] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 24, // 4: glutton.Block.steps:type_name -> glutton.Step + 23, // 5: glutton.Step.block:type_name -> glutton.Block + 25, // 6: glutton.Step.operation:type_name -> glutton.Operation + 26, // 7: glutton.Operation.requests:type_name -> glutton.Request + 2, // 8: glutton.Request.write_ram:type_name -> glutton.WriteRAMRequest + 4, // 9: glutton.Request.read_ram:type_name -> glutton.ReadRAMRequest + 6, // 10: glutton.Request.write_disk:type_name -> glutton.WriteDiskRequest + 8, // 11: glutton.Request.read_disk:type_name -> glutton.ReadDiskRequest + 17, // 12: glutton.Request.burn_cpu:type_name -> glutton.BurnCPURequest + 19, // 13: glutton.Request.ingest:type_name -> glutton.IngestRequest + 27, // 14: glutton.Request.sleep:type_name -> glutton.SleepRequest + 23, // 15: glutton.RunScriptRequest.script:type_name -> glutton.Block + 30, // 16: glutton.RunScriptResponse.result:type_name -> glutton.StepResult + 31, // 17: glutton.StepResult.stats:type_name -> glutton.Stats + 30, // 18: glutton.StepResult.children:type_name -> glutton.StepResult + 2, // 19: glutton.Glutton.WriteRAM:input_type -> glutton.WriteRAMRequest + 4, // 20: glutton.Glutton.ReadRAM:input_type -> glutton.ReadRAMRequest + 6, // 21: glutton.Glutton.WriteDisk:input_type -> glutton.WriteDiskRequest + 8, // 22: glutton.Glutton.ReadDisk:input_type -> glutton.ReadDiskRequest + 10, // 23: glutton.Glutton.OpenFD:input_type -> glutton.OpenFDRequest + 12, // 24: glutton.Glutton.Ping:input_type -> glutton.PingRequest + 14, // 25: glutton.Glutton.Gossip:input_type -> glutton.GossipRequest + 17, // 26: glutton.Glutton.BurnCPU:input_type -> glutton.BurnCPURequest + 19, // 27: glutton.Glutton.Ingest:input_type -> glutton.IngestRequest + 21, // 28: glutton.Glutton.UseCPU:input_type -> glutton.UseCPURequest + 28, // 29: glutton.Glutton.RunScript:input_type -> glutton.RunScriptRequest + 3, // 30: glutton.Glutton.WriteRAM:output_type -> glutton.WriteRAMResponse + 5, // 31: glutton.Glutton.ReadRAM:output_type -> glutton.ReadRAMResponse + 7, // 32: glutton.Glutton.WriteDisk:output_type -> glutton.WriteDiskResponse + 9, // 33: glutton.Glutton.ReadDisk:output_type -> glutton.ReadDiskResponse + 11, // 34: glutton.Glutton.OpenFD:output_type -> glutton.OpenFDResponse + 13, // 35: glutton.Glutton.Ping:output_type -> glutton.PingResponse + 15, // 36: glutton.Glutton.Gossip:output_type -> glutton.GossipResponse + 18, // 37: glutton.Glutton.BurnCPU:output_type -> glutton.BurnCPUResponse + 20, // 38: glutton.Glutton.Ingest:output_type -> glutton.IngestResponse + 22, // 39: glutton.Glutton.UseCPU:output_type -> glutton.UseCPUResponse + 29, // 40: glutton.Glutton.RunScript:output_type -> glutton.RunScriptResponse + 30, // [30:41] is the sub-list for method output_type + 19, // [19:30] is the sub-list for method input_type + 19, // [19:19] is the sub-list for extension type_name + 19, // [19:19] is the sub-list for extension extendee + 0, // [0:19] is the sub-list for field type_name } func init() { file_glutton_proto_init() } @@ -1387,13 +2125,26 @@ func file_glutton_proto_init() { return } file_glutton_proto_msgTypes[19].OneofWrappers = []any{} + file_glutton_proto_msgTypes[22].OneofWrappers = []any{ + (*Step_Block)(nil), + (*Step_Operation)(nil), + } + file_glutton_proto_msgTypes[24].OneofWrappers = []any{ + (*Request_WriteRam)(nil), + (*Request_ReadRam)(nil), + (*Request_WriteDisk)(nil), + (*Request_ReadDisk)(nil), + (*Request_BurnCpu)(nil), + (*Request_Ingest)(nil), + (*Request_Sleep)(nil), + } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_glutton_proto_rawDesc), len(file_glutton_proto_rawDesc)), NumEnums: 2, - NumMessages: 21, + NumMessages: 30, NumExtensions: 0, NumServices: 1, }, diff --git a/internal/proto/glutton/glutton.proto b/internal/proto/glutton/glutton.proto index e717abd4cd..54c534ecf0 100644 --- a/internal/proto/glutton/glutton.proto +++ b/internal/proto/glutton/glutton.proto @@ -66,6 +66,16 @@ service Glutton { // load; calling again replaces it, and num_cores=0 stops it. See // UseCPURequest for the (goroutines x duty cycle) shape. rpc UseCPU(UseCPURequest) returns (UseCPUResponse) {} + + // Runs a script inside the glutton as one request, so a caller can act + // out a whole on-sandbox activity (a build, a test run, a file churn) + // without a network round trip per operation. A script is a tree: a + // Block runs its steps in order and may loop them for a wall-clock + // budget; an Operation runs its requests at the same time. The request + // holds the connection open until the script finishes, and the response + // mirrors the tree with summed stats at every node; the requests' own + // results are not returned. + rpc RunScript(RunScriptRequest) returns (RunScriptResponse) {} } enum WriteMode { @@ -244,3 +254,105 @@ message UseCPUResponse { // num_cores unless it was capped at GOMAXPROCS. int32 num_cores = 1; } + +// A Block is a list of steps run in order. With loop_duration_ms set, the +// block cycles through its steps until the budget elapses. A budget is +// checked between passes: the pass in progress finishes, so a loop +// overruns its budget by at most one pass and no request is cut short. +// A nested block stops cycling once any ancestor's budget has elapsed, +// the same way. +message Block { + // At least one step is required. + repeated Step steps = 1; + + // Wall-clock budget to cycle through the steps for, in milliseconds; + // the pass under way when it elapses finishes. Zero runs the steps once. + int64 loop_duration_ms = 2; +} + +// A Step is either a nested Block or an Operation. +message Step { + oneof kind { + Block block = 1; + Operation operation = 2; + } +} + +// An Operation runs its requests at the same time and waits for all of +// them. The first request to fail cancels the rest. RAM requests take +// turns on the glutton's single RAM lock, so only CPU, disk, and sleep +// requests overlap in practice. +message Operation { + // At least one request is required. + repeated Request requests = 1; +} + +// A Request is one of the service's work requests. UseCPU and OpenFD are +// left out because they set a background load that outlives the script +// (BurnCPU is the CPU spike), and Gossip because it starts a loop. +message Request { + oneof kind { + WriteRAMRequest write_ram = 1; + ReadRAMRequest read_ram = 2; + WriteDiskRequest write_disk = 3; + ReadDiskRequest read_disk = 4; + BurnCPURequest burn_cpu = 5; + IngestRequest ingest = 6; + SleepRequest sleep = 7; + } +} + +message SleepRequest { + // Wall-clock time to idle, in milliseconds. The glutton stays awake and + // does nothing, which paces the requests around it. + int64 duration_ms = 1; +} + +message RunScriptRequest { + Block script = 1; +} + +message RunScriptResponse { + // The script's root block. + StepResult result = 1; +} + +// A StepResult mirrors one node of the script: a block, an operation, or +// a request. Its children are in the request's order: a block's steps or +// an operation's requests; a request has none. A node inside a looped +// block is entered once per pass, and its stats sum over every entry, so +// the response is the size of the script however long it ran. +message StepResult { + Stats stats = 1; + repeated StepResult children = 2; +} + +// Stats are a node's vital signs. Every field but elapsed_ms is the sum +// over the node's children and entries, so the root carries the totals. +message Stats { + // Wall clock spent in this node, summed over its entries. A block's is + // the sum of its steps'; an operation's is its longest request's, since + // they overlap; so a parent's elapsed is not the sum of its children's. + int64 elapsed_ms = 1; + + // Times this node was entered: a looped block's pass count, and for + // every other node the entries its enclosing loops gave it. + int64 passes = 2; + + // Requests run beneath this node. + int64 requests_run = 3; + + // Bytes WriteDisk generated and Ingest received, and bytes ReadDisk read. + int64 disk_bytes_written = 4; + int64 disk_bytes_read = 5; + + // Bytes WriteRAM was asked to write, and bytes ReadRAM walked. + int64 ram_bytes_written = 6; + int64 ram_bytes_read = 7; + + // Hash iterations BurnCPU completed. + int64 burn_iterations = 8; + + // Milliseconds slept. + int64 slept_ms = 9; +} diff --git a/internal/proto/glutton/glutton_grpc.pb.go b/internal/proto/glutton/glutton_grpc.pb.go index f6e590b4b1..5952f01960 100644 --- a/internal/proto/glutton/glutton_grpc.pb.go +++ b/internal/proto/glutton/glutton_grpc.pb.go @@ -43,6 +43,7 @@ const ( Glutton_BurnCPU_FullMethodName = "/glutton.Glutton/BurnCPU" Glutton_Ingest_FullMethodName = "/glutton.Glutton/Ingest" Glutton_UseCPU_FullMethodName = "/glutton.Glutton/UseCPU" + Glutton_RunScript_FullMethodName = "/glutton.Glutton/RunScript" ) // GluttonClient is the client API for Glutton service. @@ -88,6 +89,15 @@ type GluttonClient interface { // load; calling again replaces it, and num_cores=0 stops it. See // UseCPURequest for the (goroutines x duty cycle) shape. UseCPU(ctx context.Context, in *UseCPURequest, opts ...grpc.CallOption) (*UseCPUResponse, error) + // Runs a script inside the glutton as one request, so a caller can act + // out a whole on-sandbox activity (a build, a test run, a file churn) + // without a network round trip per operation. A script is a tree: a + // Block runs its steps in order and may loop them for a wall-clock + // budget; an Operation runs its requests at the same time. The request + // holds the connection open until the script finishes, and the response + // mirrors the tree with summed stats at every node; the requests' own + // results are not returned. + RunScript(ctx context.Context, in *RunScriptRequest, opts ...grpc.CallOption) (*RunScriptResponse, error) } type gluttonClient struct { @@ -198,6 +208,16 @@ func (c *gluttonClient) UseCPU(ctx context.Context, in *UseCPURequest, opts ...g return out, nil } +func (c *gluttonClient) RunScript(ctx context.Context, in *RunScriptRequest, opts ...grpc.CallOption) (*RunScriptResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RunScriptResponse) + err := c.cc.Invoke(ctx, Glutton_RunScript_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // GluttonServer is the server API for Glutton service. // All implementations must embed UnimplementedGluttonServer // for forward compatibility. @@ -241,6 +261,15 @@ type GluttonServer interface { // load; calling again replaces it, and num_cores=0 stops it. See // UseCPURequest for the (goroutines x duty cycle) shape. UseCPU(context.Context, *UseCPURequest) (*UseCPUResponse, error) + // Runs a script inside the glutton as one request, so a caller can act + // out a whole on-sandbox activity (a build, a test run, a file churn) + // without a network round trip per operation. A script is a tree: a + // Block runs its steps in order and may loop them for a wall-clock + // budget; an Operation runs its requests at the same time. The request + // holds the connection open until the script finishes, and the response + // mirrors the tree with summed stats at every node; the requests' own + // results are not returned. + RunScript(context.Context, *RunScriptRequest) (*RunScriptResponse, error) mustEmbedUnimplementedGluttonServer() } @@ -281,6 +310,9 @@ func (UnimplementedGluttonServer) Ingest(context.Context, *IngestRequest) (*Inge func (UnimplementedGluttonServer) UseCPU(context.Context, *UseCPURequest) (*UseCPUResponse, error) { return nil, status.Error(codes.Unimplemented, "method UseCPU not implemented") } +func (UnimplementedGluttonServer) RunScript(context.Context, *RunScriptRequest) (*RunScriptResponse, error) { + return nil, status.Error(codes.Unimplemented, "method RunScript not implemented") +} func (UnimplementedGluttonServer) mustEmbedUnimplementedGluttonServer() {} func (UnimplementedGluttonServer) testEmbeddedByValue() {} @@ -482,6 +514,24 @@ func _Glutton_UseCPU_Handler(srv interface{}, ctx context.Context, dec func(inte return interceptor(ctx, in, info, handler) } +func _Glutton_RunScript_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RunScriptRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GluttonServer).RunScript(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Glutton_RunScript_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GluttonServer).RunScript(ctx, req.(*RunScriptRequest)) + } + return interceptor(ctx, in, info, handler) +} + // Glutton_ServiceDesc is the grpc.ServiceDesc for Glutton service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -529,6 +579,10 @@ var Glutton_ServiceDesc = grpc.ServiceDesc{ MethodName: "UseCPU", Handler: _Glutton_UseCPU_Handler, }, + { + MethodName: "RunScript", + Handler: _Glutton_RunScript_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "glutton.proto", From 341037dcfa0ca1b5d5961c391fc43cdb334b3969 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Tue, 20 Oct 2026 22:52:15 +0000 Subject: [PATCH 26/63] atelet: Wait for superseded uploads to stop (#2833) When a Checkpoint with an immediate upload, or a PromoteSnapshot, makes a snapshot durable, atelet now cancels any upload of an older snapshot still in progress and waits for it to stop before returning. As during a PromoteSnapshot, no background upload starts while a Checkpoint with an immediate upload runs. Once the control plane learns that a newer snapshot is durable, atelet is no longer uploading an older snapshot in the background, so the control plane can delete its objects. Once the new snapshot is committed, the canceled upload deletes the older snapshot's local copy before the Checkpoint or PromoteSnapshot returns. Only a Run or Restore in progress, which may read any local snapshot, keeps a superseded local copy. Part of https://github.com/agent-substrate/substrate/issues/2069 and https://github.com/agent-substrate/substrate/issues/2571 - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- cmd/atelet/internal/snapshottracker/actor.go | 28 ++- .../internal/snapshottracker/background.go | 15 +- .../snapshottracker/background_test.go | 7 +- .../internal/snapshottracker/detach_test.go | 8 +- .../internal/snapshottracker/lifecycle.go | 76 ++++-- .../internal/snapshottracker/metrics.go | 7 +- .../internal/snapshottracker/queues_test.go | 3 +- .../internal/snapshottracker/report_test.go | 25 +- .../internal/snapshottracker/tracker.go | 29 ++- .../internal/snapshottracker/tracker_test.go | 16 +- cmd/atelet/internal/snapshottracker/upload.go | 7 +- .../internal/snapshottracker/upload_test.go | 230 ++++++++++++++---- cmd/atelet/lifecycle_test.go | 122 ++++++++++ cmd/atelet/main.go | 4 +- docs/metrics/registry/metrics.yaml | 15 +- 15 files changed, 458 insertions(+), 134 deletions(-) diff --git a/cmd/atelet/internal/snapshottracker/actor.go b/cmd/atelet/internal/snapshottracker/actor.go index d97a6e86d6..73e8d6e27a 100644 --- a/cmd/atelet/internal/snapshottracker/actor.go +++ b/cmd/atelet/internal/snapshottracker/actor.go @@ -187,16 +187,16 @@ type holdKind int const ( holdNone holdKind = iota holdStart // Run or Restore - holdCheckpoint // Checkpoint - holdPromote // PromoteSnapshot + holdCheckpoint // Checkpoint under UPLOAD_MODE_DEFERRED + holdDurable // Checkpoint under UPLOAD_MODE_IMMEDIATE, or PromoteSnapshot holdTerminate // Terminate ) // blocksUploads reports whether no background upload of the actor starts while -// an RPC of kind k holds it: PromoteSnapshot, which makes a snapshot durable -// itself, or Terminate, which forgets the actor. +// an RPC of kind k holds it: one that makes a snapshot durable, which would +// only cancel the upload, or Terminate, which forgets the actor. func (k holdKind) blocksUploads() bool { - return k == holdPromote || k == holdTerminate + return k == holdDurable || k == holdTerminate } // actorSnapshots is what atelet tracks of one actor's snapshots on this node. @@ -387,6 +387,24 @@ func (a *actorSnapshots) runningUpload() *SnapshotRef { return &a.op.target } +// supersededUpload returns the upload in progress when a snapshot newer than +// the one it uploads is durable, or nil. +func (a *actorSnapshots) supersededUpload() *actorOp { + up := a.runningUpload() + if up == nil || a.LatestDurable == nil || up.generation() >= a.LatestDurable.generation() { + return nil + } + return a.op +} + +// localInUse reports whether a Run or Restore of the actor in progress may use +// a superseded local snapshot. A Restore may read any local snapshot in place; +// a Checkpoint or PromoteSnapshot only reads LatestLocal, which is not +// superseded until the new snapshot is committed, and a Terminate reads none. +func (a *actorSnapshots) localInUse() bool { + return a.hold == holdStart +} + // runningDetach returns the snapshot an opDetach goroutine is currently // detaching, or nil. func (a *actorSnapshots) runningDetach() *SnapshotRef { diff --git a/cmd/atelet/internal/snapshottracker/background.go b/cmd/atelet/internal/snapshottracker/background.go index 963ca4e759..f335810dfb 100644 --- a/cmd/atelet/internal/snapshottracker/background.go +++ b/cmd/atelet/internal/snapshottracker/background.go @@ -89,8 +89,8 @@ func (k opKind) String() string { // logOp logs how an upload or detach ended. Foreground failures are returned // to the RPC caller, so only background failures are logged here: as a -// warning, or at info level once ctx has ended, as when Terminate or an atelet -// shutdown cancels the operation. +// warning, or at info level once ctx has ended, as when Terminate, an atelet +// shutdown, or a newer durable snapshot cancels the operation. func logOp(ctx context.Context, actorUID string, kind opKind, target SnapshotRef, uploaded, background bool, elapsed time.Duration, err error) { attrs := []any{ slog.String("op", kind.String()), @@ -123,7 +123,11 @@ func logOp(ctx context.Context, actorUID string, kind opKind, target SnapshotRef // - Forget cancels any operation in progress and waits for it to end before // it drops the actor's state. No operation begins during the Terminate // that calls it. -// - Checkpoint never waits for an upload. +// - CommitDurable and CommitPromoted cancel an upload of a snapshot older +// than LatestDurable and wait for it to end; a deferred Checkpoint never +// waits for an upload. +// - No upload begins during a Checkpoint under UPLOAD_MODE_IMMEDIATE or a +// PromoteSnapshot, though one already running goes on. // - Run, Restore, and Checkpoint wait for a detach to end, so it never // deletes the actor's local snapshots underneath them. type actorOp struct { @@ -260,8 +264,9 @@ func (t *Tracker) beginDueWork(actorUID string, kind opKind) func() { case opUpload: if due, ok := t.uploadDueLocked(a); ok && !due.After(now) { if a.hold.blocksUploads() { - // A PromoteSnapshot of the actor is in progress, and queues - // the actor again when it ends (see holdLocked). + // A Checkpoint under UPLOAD_MODE_IMMEDIATE or a PromoteSnapshot + // of the actor is in progress, and queues the actor again when + // it ends (see holdLocked). return nil } run := t.beginUploadLocked(t.ctx, a) diff --git a/cmd/atelet/internal/snapshottracker/background_test.go b/cmd/atelet/internal/snapshottracker/background_test.go index a9687a389c..942e300567 100644 --- a/cmd/atelet/internal/snapshottracker/background_test.go +++ b/cmd/atelet/internal/snapshottracker/background_test.go @@ -321,9 +321,8 @@ func TestTrackerTerminateHoldsOffBackgroundWork(t *testing.T) { } } -// A Checkpoint's commits and LocalSnapshotsToKeep, PromoteSnapshot's commit, -// and State neither wait for the actor's background operation in progress nor -// cancel it. +// CommitLocal, LocalSnapshotsToKeep, and State neither wait for the actor's +// background upload in progress nor cancel it. func TestCommitsDoNotWaitForInFlightOp(t *testing.T) { useActorDir(t) tr := newTracker(t) @@ -338,8 +337,6 @@ func TestCommitsDoNotWaitForInFlightOp(t *testing.T) { }{ {"CommitLocal", func() { tr.CommitLocal(trackedActorUID, testRef("s2", 2, t0)) }}, {"LocalSnapshotsToKeep", func() { tr.LocalSnapshotsToKeep(trackedActorUID) }}, - {"CommitDurable", func() { tr.CommitDurable(trackedActorUID, testRef("s3", 3, t0)) }}, - {"CommitPromoted", func() { tr.CommitPromoted(trackedActorUID, testRef("s4", 4, t0)) }}, {"State", func() { tr.State(trackedActorUID) }}, } { returnsPromptly(t, c.name+" during a background operation", c.call) diff --git a/cmd/atelet/internal/snapshottracker/detach_test.go b/cmd/atelet/internal/snapshottracker/detach_test.go index e53d88eaae..1317ffb1d9 100644 --- a/cmd/atelet/internal/snapshottracker/detach_test.go +++ b/cmd/atelet/internal/snapshottracker/detach_test.go @@ -402,7 +402,7 @@ func TestTrackerDetachTimer(t *testing.T) { tr.CommitLocal(trackedActorUID, testRef("s2", 2, tr.clock.Now())) due := detachDueOf(tr, trackedActorUID) - endCheckpoint, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s3", 3, time.Time{}).Snapshot) + endCheckpoint, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s3", 3, time.Time{}).Snapshot, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) if err != nil { t.Fatalf("beginCheckpoint: %v", err) } @@ -483,7 +483,7 @@ func TestTrackerWaitsForDetach(t *testing.T) { { name: "BeginCheckpoint", begin: func(tr *Tracker, ctx context.Context, uid string) (func(), error) { - return tr.BeginCheckpoint(ctx, uid, testRef("s2", 2, time.Time{}).Snapshot) + return tr.BeginCheckpoint(ctx, uid, testRef("s2", 2, time.Time{}).Snapshot, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) }, }, } { @@ -539,7 +539,7 @@ func TestTrackerDetachWaitChecksGeneration(t *testing.T) { stale := testRef("s0", 1, time.Time{}) returnsPromptly(t, "BeginCheckpoint of stale generation during detach", func() { - if _, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, stale.Snapshot); apierror.Code(err) != codes.Aborted { + if _, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, stale.Snapshot, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); apierror.Code(err) != codes.Aborted { t.Errorf("BeginCheckpoint = %v, want Aborted", err) } }) @@ -652,7 +652,7 @@ func TestTrackerHoldsFirstCheckpoint(t *testing.T) { store := newFakeSnapshotStore() tr := newIdleTestTracker(t, store) s1 := testRef("s1", 1, tr.clock.Now()) - end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, s1.Snapshot) + end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, s1.Snapshot, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) if err != nil { t.Fatalf("beginCheckpoint: %v", err) } diff --git a/cmd/atelet/internal/snapshottracker/lifecycle.go b/cmd/atelet/internal/snapshottracker/lifecycle.go index 69bd53ac8f..55b35b06c1 100644 --- a/cmd/atelet/internal/snapshottracker/lifecycle.go +++ b/cmd/atelet/internal/snapshottracker/lifecycle.go @@ -65,12 +65,14 @@ func (t *Tracker) CommitLocal(actorUID string, snap SnapshotRef) *ateletpb.Actor // CommitDurable records snap, just uploaded by a Checkpoint under // UPLOAD_MODE_IMMEDIATE, as the actor's latest durable snapshot, and returns // the actor's state. A commit of a snapshot already durable or older than -// LatestDurable leaves the state as it is. It drops an older LatestLocal, -// which snap supersedes, so the Checkpoint's prune deletes it even if no local -// copy of snap is kept. The Checkpoint then records its local copy with -// CommitLocal, if it manages to keep one. An older snapshot's upload in -// progress runs on, and removes that snapshot's local copy when it ends (see -// removeSupersededLocalCopy). +// LatestDurable leaves the state as it is. +// +// It drops an older LatestLocal, which snap supersedes, so the Checkpoint's +// prune deletes it even if no local copy of snap is kept. The Checkpoint then +// records its local copy with CommitLocal, if it manages to keep one. +// +// Once it records snap, it returns only after any upload of an older snapshot +// has stopped (see durableStateLocked). func (t *Tracker) CommitDurable(actorUID string, snap SnapshotRef) *ateletpb.ActorSnapshotState { t.lock(actorUID) defer t.unlock(actorUID) @@ -89,7 +91,7 @@ func (t *Tracker) CommitDurable(actorUID string, snap SnapshotRef) *ateletpb.Act } t.cancelReportLocked(a) t.durableCommittedLocked(a) - return t.stateLocked(a) + return t.durableStateLocked(a) } // LocalSnapshotsToKeep returns the actor's local snapshots atelet still needs: @@ -115,11 +117,15 @@ func (t *Tracker) LocalSnapshotsToKeep(actorUID string) []string { } // CommitPromoted records snap, a local snapshot PromoteSnapshot made durable or -// found durable already, as the actor's latest durable snapshot, keeping its -// local copy, the time its checkpoint committed, and any UploadTime on snap. -// It leaves DetachAt as it is. A promote of a snapshot already durable, or -// older than LatestDurable, keeps what was recorded. It returns the actor's -// state. +// found durable already, as the actor's latest durable snapshot, and returns +// the actor's state. A promote of a snapshot already durable, or older than +// LatestDurable, keeps what was recorded. +// +// It keeps snap's local copy, the time its checkpoint committed, and any +// UploadTime on snap. It leaves DetachAt as it is. +// +// Once it records snap, it returns only after any upload of an older snapshot +// has stopped (see durableStateLocked). func (t *Tracker) CommitPromoted(actorUID string, snap SnapshotRef) *ateletpb.ActorSnapshotState { t.lock(actorUID) defer t.unlock(actorUID) @@ -149,7 +155,7 @@ func (t *Tracker) CommitPromoted(actorUID string, snap SnapshotRef) *ateletpb.Ac t.cancelReportLocked(a) } t.durableCommittedLocked(a) - return t.stateLocked(a) + return t.durableStateLocked(a) } // BeginStart records that a Run or Restore is starting the actor on this node, @@ -189,25 +195,31 @@ func (t *Tracker) BeginStart(ctx context.Context, actorUID string) (end func(), return t.holdLocked(actorUID, a, holdStart), nil } -// BeginCheckpoint records that a Checkpoint of snap is in progress on this -// node, once any background detach in progress has finished, and returns the -// func to call when the checkpoint ends, whether or not it succeeded. It fails -// with Aborted when snap has a stale generation for the actor: below the -// actor's newest snapshot on this node, or equal to it under another -// snapshot_uuid. atelet does not detach the actor in between, so a detach -// cannot delete the local snapshot the checkpoint writes. -func (t *Tracker) BeginCheckpoint(ctx context.Context, actorUID string, snap *ateletpb.Snapshot) (end func(), err error) { +// BeginCheckpoint records that a Checkpoint of snap under mode is in progress +// on this node, once any background detach in progress has finished, and +// returns the func to call when the checkpoint ends, whether or not it +// succeeded. It fails with Aborted when snap has a stale generation for the +// actor: below the actor's newest snapshot on this node, or equal to it under +// another snapshot_uuid. atelet does not detach the actor in between, so a +// detach cannot delete the local snapshot the checkpoint writes. Under +// UPLOAD_MODE_IMMEDIATE, atelet starts no background upload in between either, +// since CommitDurable would only cancel it; one already running goes on. +func (t *Tracker) BeginCheckpoint(ctx context.Context, actorUID string, snap *ateletpb.Snapshot, mode ateletpb.UploadMode) (end func(), err error) { t.lock(actorUID) defer t.unlock(actorUID) a, err := t.waitForDetachLocked(ctx, actorUID, func(a *actorSnapshots) error { return a.checkGeneration(snap) }) if err != nil { return nil, err } + kind := holdCheckpoint + if mode == ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE { + kind = holdDurable + } // Unlike BeginStart, leave DetachAt as it is. Either it is already // zero, because the actor's Run or Restore on this node unscheduled the // detach, or this is a retry of a Checkpoint that already committed its // snapshot, whose detach stays scheduled. - return t.holdLocked(actorUID, a, holdCheckpoint), nil + return t.holdLocked(actorUID, a, kind), nil } // BeginPromote records that a PromoteSnapshot of snap is in progress on this @@ -230,7 +242,7 @@ func (t *Tracker) BeginPromote(ctx context.Context, actorUID string, snap Snapsh return false, nil, err } } - return alreadyDurable, t.holdLocked(actorUID, a, holdPromote), nil + return alreadyDurable, t.holdLocked(actorUID, a, holdDurable), nil } // BeginTerminate records that a Terminate of the actor is in progress on this @@ -328,6 +340,24 @@ func (t *Tracker) durableCommittedLocked(a *actorSnapshots) { t.requeueLocked(a) } +// durableStateLocked returns the actor's state for CommitDurable or +// CommitPromoted to return once they record a new LatestDurable. Once the +// control plane learns of it, it may delete the objects of older snapshots, so +// this first cancels any upload of an older snapshot in progress and waits for +// it to end, releasing the actor's lock meanwhile. The object-store plugin may +// still finish an object that upload had under way; deleting the actor +// collects it. +func (t *Tracker) durableStateLocked(a *actorSnapshots) *ateletpb.ActorSnapshotState { + for op := a.supersededUpload(); op != nil; op = a.supersededUpload() { + op.cancel() + t.unlock(a.actorUID) + // The upload ends by updating a, the entry it began on. + <-op.done + t.lock(a.actorUID) + } + return t.stateLocked(a) +} + // succeededLocked finishes a change that leaves the actor in a successful end // state, such as a background upload or detach: it starts the actor's backoff // over, saves the actor's state, and queues the actor for its next background diff --git a/cmd/atelet/internal/snapshottracker/metrics.go b/cmd/atelet/internal/snapshottracker/metrics.go index a2db45a833..736003fb15 100644 --- a/cmd/atelet/internal/snapshottracker/metrics.go +++ b/cmd/atelet/internal/snapshottracker/metrics.go @@ -250,9 +250,10 @@ func outcomeAttrs(ctx context.Context, target SnapshotRef, err error) []attribut } // errorType returns the error.type of a failed upload or detach: Canceled or -// DeadlineExceeded once ctx has ended (when Terminate, an atelet shutdown, or -// the background upload timeout ends it), and otherwise the code an apierror -// or a plugin's gRPC status carries, or Internal. +// DeadlineExceeded once ctx has ended (when Terminate, an atelet shutdown, a +// newer durable snapshot, or the background upload timeout ends it), and +// otherwise the code an apierror or a plugin's gRPC status carries, or +// Internal. func errorType(ctx context.Context, err error) string { if ctxErr := ctx.Err(); ctxErr != nil { return status.FromContextError(ctxErr).Code().String() diff --git a/cmd/atelet/internal/snapshottracker/queues_test.go b/cmd/atelet/internal/snapshottracker/queues_test.go index e144f64ab8..4f55d2a263 100644 --- a/cmd/atelet/internal/snapshottracker/queues_test.go +++ b/cmd/atelet/internal/snapshottracker/queues_test.go @@ -21,6 +21,7 @@ import ( "time" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/internal/proto/ateletpb" ) // Uploads and detaches have queues and workers of their own, so a due detach @@ -183,7 +184,7 @@ func TestTrackerUploadRunsDuringCheckpointHoldWhileDetachWaits(t *testing.T) { t0 := tr.clock.Now() tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) - endCheckpoint, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s2", 2, time.Time{}).Snapshot) + endCheckpoint, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s2", 2, time.Time{}).Snapshot, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) if err != nil { t.Fatalf("BeginCheckpoint: %v", err) } diff --git a/cmd/atelet/internal/snapshottracker/report_test.go b/cmd/atelet/internal/snapshottracker/report_test.go index 7fd6d29794..98a89d2e5c 100644 --- a/cmd/atelet/internal/snapshottracker/report_test.go +++ b/cmd/atelet/internal/snapshottracker/report_test.go @@ -295,9 +295,12 @@ func TestTrackerDoesNotReportWhenNothingChanged(t *testing.T) { for _, tc := range []struct { name string snap SnapshotRef + // cancels is set when snap is newer than the upload's snapshot, so + // CommitDurable cancels the upload and waits for it to stop. + cancels bool }{ - {"newer durable snapshot", committedRef("s2", 2, time.Time{})}, - {"same generation already durable", committedRef("s1", 1, time.Time{})}, + {"newer durable snapshot", committedRef("s2", 2, time.Time{}), true}, + {"same generation already durable", committedRef("s1", 1, time.Time{}), false}, } { t.Run("upload overtaken by "+tc.name, func(t *testing.T) { store := newFakeSnapshotStore() @@ -311,7 +314,9 @@ func TestTrackerDoesNotReportWhenNothingChanged(t *testing.T) { durable.UploadTime = durable.CommitTime.Add(time.Second) tr.CommitDurable(trackedActorUID, durable) - store.release <- nil + if !tc.cancels { + store.release <- nil + } waitClosed(t, done, "the upload of s1") if reportDue(tr) || tr.reporting.queue.Len() != 0 { t.Errorf("report due = %v, queued = %d after an overtaken upload, want neither", reportDue(tr), tr.reporting.queue.Len()) @@ -1135,7 +1140,17 @@ func TestTrackerDefersReportDuringHold(t *testing.T) { { name: "BeginCheckpoint", begin: func(t *testing.T, tr *Tracker) func() { - end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, committedRef("s2", 2, tr.clock.Now()).Snapshot) + end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, committedRef("s2", 2, tr.clock.Now()).Snapshot, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + if err != nil { + t.Fatalf("BeginCheckpoint: %v", err) + } + return end + }, + }, + { + name: "BeginCheckpoint immediate", + begin: func(t *testing.T, tr *Tracker) func() { + end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, committedRef("s2", 2, tr.clock.Now()).Snapshot, ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE) if err != nil { t.Fatalf("BeginCheckpoint: %v", err) } @@ -1232,7 +1247,7 @@ func TestTrackerDefersReportDuringHold(t *testing.T) { uploadNow(t, tr, store, nil) s2 := committedRef("s2", 2, tr.clock.Now()) - end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, s2.Snapshot) + end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, s2.Snapshot, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) if err != nil { t.Fatalf("BeginCheckpoint: %v", err) } diff --git a/cmd/atelet/internal/snapshottracker/tracker.go b/cmd/atelet/internal/snapshottracker/tracker.go index 88c61e0fd8..35df8fae5c 100644 --- a/cmd/atelet/internal/snapshottracker/tracker.go +++ b/cmd/atelet/internal/snapshottracker/tracker.go @@ -106,16 +106,18 @@ type Store interface { // The first table shows how each RPC on an actor treats the actor's upload or // detach in progress, and its report due. The RPC waits for the operation to // end as it begins (BeginStart, BeginCheckpoint, BeginPromote, or Detach), -// cancels it and waits for it to stop, or leaves it alone (-). Terminate -// cancels in Forget, once the workload is gone. A canceled report is neither -// sent nor retried, though an attempt under way finishes. +// cancels it and waits for it to stop, or leaves it alone (-). A Checkpoint or +// PromoteSnapshot cancels once it commits its snapshot, and Terminate cancels +// in Forget, once the workload is gone. A canceled report is neither sent nor +// retried, though an attempt under way finishes. // // RPC upload in progress detach in progress report due // Run, Restore - waits - -// Checkpoint - waits cancels +// deferred Checkpoint - waits cancels +// immediate Checkpoint cancels waits cancels // PromoteSnapshot waits if the same waits cancels if a -// snapshot local snapshot -// is kept +// snapshot, else local snapshot +// cancels is kept // DetachActor waits waits cancels unless // it fails // Terminate cancels cancels cancels @@ -127,7 +129,8 @@ type Store interface { // // running upload due detach due report due // Run, Restore begins waits waits -// Checkpoint begins waits waits +// deferred Checkpoint begins waits waits +// immediate Checkpoint waits waits waits // PromoteSnapshot waits waits waits // Terminate waits waits waits // upload or detach waits waits begins @@ -199,7 +202,7 @@ type Tracker struct { // actors maps actorUID to the entries requeueLocked keeps in memory (see // requeueLocked). requeueLocked drops any other entry, and findLocked then // reads the state from the file. Read and change it through findLocked, - // findOrAddLocked, addLocked, dropLocked, and heldLocked. + // findOrAddLocked, addLocked, dropLocked, and localInUseLocked. actors map[string]*actorSnapshots } @@ -381,14 +384,14 @@ func (t *Tracker) dropLocked(actorUID string) { delete(t.actors, actorUID) } -// heldLocked reports whether a Run, Restore, Checkpoint, or PromoteSnapshot of -// the actor is in progress. An entry with one stays in memory, so this never -// reads the snapshot state file. -func (t *Tracker) heldLocked(actorUID string) bool { +// localInUseLocked reports whether a Run or Restore of the actor in progress +// may use a superseded local snapshot (see actorSnapshots.localInUse). An +// entry with one stays in memory, so this never reads the snapshot state file. +func (t *Tracker) localInUseLocked(actorUID string) bool { t.mu.Lock() defer t.mu.Unlock() a := t.actors[actorUID] - return a != nil && a.hold != holdNone && a.hold != holdTerminate + return a != nil && a.localInUse() } // uploadDueLocked returns when the actor's background upload is due, as diff --git a/cmd/atelet/internal/snapshottracker/tracker_test.go b/cmd/atelet/internal/snapshottracker/tracker_test.go index 52ab4d4d36..4ffba6205f 100644 --- a/cmd/atelet/internal/snapshottracker/tracker_test.go +++ b/cmd/atelet/internal/snapshottracker/tracker_test.go @@ -788,14 +788,16 @@ func TestTrackerCheckGeneration(t *testing.T) { } checkBegin := func(t *testing.T, tr *Tracker, r SnapshotRef, wantCode codes.Code) { t.Helper() - end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, r.Snapshot) - if err == nil { - end() - } - if got := apierror.Code(err); got != wantCode { - t.Errorf("BeginCheckpoint(%v) code = %v, want %v", r.Snapshot, got, wantCode) + for _, mode := range []ateletpb.UploadMode{ateletpb.UploadMode_UPLOAD_MODE_DEFERRED, ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE} { + end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, r.Snapshot, mode) + if err == nil { + end() + } + if got := apierror.Code(err); got != wantCode { + t.Errorf("BeginCheckpoint(%v, %v) code = %v, want %v", r.Snapshot, mode, got, wantCode) + } } - _, end, err = tr.BeginPromote(t.Context(), trackedActorUID, r) + _, end, err := tr.BeginPromote(t.Context(), trackedActorUID, r) if err == nil { end() } diff --git a/cmd/atelet/internal/snapshottracker/upload.go b/cmd/atelet/internal/snapshottracker/upload.go index 42066c2809..efb833e5cb 100644 --- a/cmd/atelet/internal/snapshottracker/upload.go +++ b/cmd/atelet/internal/snapshottracker/upload.go @@ -183,13 +183,12 @@ func (t *Tracker) recordUploadLocked(a *actorSnapshots, target SnapshotRef, uplo // removeSupersededLocalCopyLocked deletes the local copy of target, an upload's // snapshot that a newer snapshot superseded, once no upload needs it: the copy -// was kept only for the upload. It leaves the copy while a Run, Restore, -// Checkpoint, or PromoteSnapshot of the actor is in progress, since a Restore -// or PromoteSnapshot may be reading it, and deletes it under the actor's lock +// was kept only for the upload. It leaves the copy while a Run or Restore in +// progress may use it (see localInUse), and deletes it under the actor's lock // so that none begins reading it meanwhile. Best-effort: the actor's next // checkpoint, detach, or terminate prunes a copy left behind. func (t *Tracker) removeSupersededLocalCopyLocked(actorUID string, target SnapshotRef) { - if t.heldLocked(actorUID) { + if t.localInUseLocked(actorUID) { return } if err := removeLocalCheckpoint(t.ctx, actorUID, target.uuid()); err != nil { diff --git a/cmd/atelet/internal/snapshottracker/upload_test.go b/cmd/atelet/internal/snapshottracker/upload_test.go index f5d8318e0d..fbf591486c 100644 --- a/cmd/atelet/internal/snapshottracker/upload_test.go +++ b/cmd/atelet/internal/snapshottracker/upload_test.go @@ -150,54 +150,73 @@ func TestTrackerCheckpointDuringUpload(t *testing.T) { } } -// An immediate checkpoint makes the actor's newest snapshot durable, which -// drops the older local snapshot, then records its own local copy. An older -// snapshot's upload in progress runs on: LocalSnapshotsToKeep keeps that -// snapshot, and the upload deletes it when it ends without holding the actor -// back when it fails, though the new local copy has a detach scheduled. +// An immediate checkpoint makes the actor's newest snapshot durable, drops the +// older local snapshot, cancels any older upload in progress, and waits for it +// to stop before returning. The upload then deletes the older snapshot's local +// copy, though the checkpoint holds the actor. func TestTrackerImmediateCheckpointDuringUpload(t *testing.T) { for _, tc := range []struct { name string uploadErr error }{ - {name: "older upload succeeds"}, - {name: "older upload fails", uploadErr: errors.New("boom")}, + {name: "older upload canceled", uploadErr: context.Canceled}, + {name: "older upload finished before cancel", uploadErr: nil}, + {name: "older upload failed", uploadErr: errors.New("boom")}, } { t.Run(tc.name, func(t *testing.T) { store := newFakeSnapshotStore() store.setBlocking(true) + store.setIgnoreCancel(true) tr := newTestTracker(t, store) if err := os.MkdirAll(ateletpath.LocalSnapshotDir(trackedActorUID, "s1"), 0o700); err != nil { t.Fatal(err) } t0 := tr.clock.Now() tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) - done := startUpload(t, tr, store) + startUpload(t, tr, store) s2 := testRef("s2", 2, t0.Add(time.Minute)) + endCheckpoint, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, s2.Snapshot, ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE) + if err != nil { + t.Fatalf("BeginCheckpoint: %v", err) + } s2.UploadTime = s2.CommitTime.Add(time.Second) - st := tr.CommitDurable(trackedActorUID, s2) - if st.GetLatestLocal() != nil || st.GetUploadDueTime() != nil || uuidOf(st.GetUploading()) != "s1" { - t.Errorf("state = %v, want nothing local or due, and s1 uploading", st) + stateCh := make(chan *ateletpb.ActorSnapshotState, 1) + go func() { + stateCh <- tr.CommitDurable(trackedActorUID, s2) + }() + if got := waitCanceled(t, store); got != "s1" { + t.Fatalf("canceled upload = %q, want s1", got) + } + select { + case st := <-stateCh: + t.Fatalf("CommitDurable returned %v before the older upload stopped", st) + case <-time.After(20 * time.Millisecond): } - if keep := tr.LocalSnapshotsToKeep(trackedActorUID); !slices.Equal(keep, []string{"s1"}) { - t.Errorf("LocalSnapshotsToKeep = %v, want [s1]: s1 is being uploaded", keep) + + store.release <- tc.uploadErr + var st *ateletpb.ActorSnapshotState + select { + case st = <-stateCh: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for CommitDurable to return") + } + if st.GetLatestLocal() != nil || st.GetUploadDueTime() != nil || st.GetUploading() != nil { + t.Errorf("state = %v, want nothing local, due, or uploading", st) + } + if keep := tr.LocalSnapshotsToKeep(trackedActorUID); len(keep) != 0 { + t.Errorf("LocalSnapshotsToKeep = %v, want none", keep) } if got := st.GetLatestDurable(); uuidOf(got) != "s2" || !timeOf(got.GetDurableCompleteTime()).Equal(s2.UploadTime) { t.Errorf("latest_durable = %v, want s2 uploaded at %v", got, s2.UploadTime) } - tr.CommitLocal(trackedActorUID, testRef("s2", 2, s2.CommitTime)) - - store.release <- tc.uploadErr - waitClosed(t, done, "the upload of s1") if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s1")); !os.IsNotExist(err) { - t.Errorf("s1 after its upload: stat = %v, want it deleted", err) - } - if got := uuidOf(tr.State(trackedActorUID).GetLatestDurable()); got != "s2" { - t.Errorf("latest_durable = %q after the upload of s1, want s2", got) + t.Errorf("s1 once CommitDurable returned: stat = %v, want it deleted", err) } + tr.CommitLocal(trackedActorUID, testRef("s2", 2, s2.CommitTime)) + endCheckpoint() if inMemory(tr) { - t.Error("entry stayed in memory after the older upload ended, want it dropped") + t.Error("entry stayed in memory after the checkpoint ended, want it dropped") } // The next deferred checkpoint is due a full delay after it was taken. @@ -210,33 +229,56 @@ func TestTrackerImmediateCheckpointDuringUpload(t *testing.T) { } } -// An older upload finishing after a newer snapshot became durable and another -// local checkpoint was taken does not push the new local checkpoint's upload -// due time back. -func TestTrackerLateUploadDoesNotPushBackNextUpload(t *testing.T) { +// CommitPromoted cancels an older snapshot's upload in progress and waits for +// it to stop before returning. The upload deletes the older snapshot's local +// copy before CommitPromoted returns, though the promote holds the actor. +func TestTrackerCommitPromotedCancelsOlderUpload(t *testing.T) { store := newFakeSnapshotStore() store.setBlocking(true) + store.setIgnoreCancel(true) tr := newTestTracker(t, store) + for _, name := range []string{"s1", "s2"} { + if err := os.MkdirAll(ateletpath.LocalSnapshotDir(trackedActorUID, name), 0o700); err != nil { + t.Fatal(err) + } + } t0 := tr.clock.Now() tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) - done := startUpload(t, tr, store) - + startUpload(t, tr, store) s2 := testRef("s2", 2, t0.Add(time.Minute)) + tr.CommitLocal(trackedActorUID, s2) + alreadyDurable, endPromote, err := tr.BeginPromote(t.Context(), trackedActorUID, s2) + if err != nil || alreadyDurable { + t.Fatalf("BeginPromote = (%v, %v), want (false, nil)", alreadyDurable, err) + } + defer endPromote() s2.UploadTime = s2.CommitTime.Add(time.Second) - tr.CommitDurable(trackedActorUID, s2) - t3 := t0.Add(2 * time.Minute) - tr.CommitLocal(trackedActorUID, testRef("s3", 3, t3)) - stepTo(tr, t3.Add(10*time.Minute)) - store.release <- nil - waitClosed(t, done, "the upload of s1") + stateCh := make(chan *ateletpb.ActorSnapshotState, 1) + go func() { + stateCh <- tr.CommitPromoted(trackedActorUID, s2) + }() + if got := waitCanceled(t, store); got != "s1" { + t.Fatalf("canceled upload = %q, want s1", got) + } + select { + case st := <-stateCh: + t.Fatalf("CommitPromoted returned %v before the older upload stopped", st) + case <-time.After(20 * time.Millisecond): + } - st := tr.State(trackedActorUID) - if got := uuidOf(st.GetLatestDurable()); got != "s2" { - t.Errorf("latest_durable = %q, want s2", got) + store.release <- context.Canceled + var st *ateletpb.ActorSnapshotState + select { + case st = <-stateCh: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for CommitPromoted to return") } - if got, want := timeOf(st.GetUploadDueTime()), t3.Add(testUploadDelay); !got.Equal(want) { - t.Errorf("upload_due_time = %v, want %v: an upload delay after s3 was taken", got, want) + if uuidOf(st.GetLatestDurable()) != "s2" || uuidOf(st.GetLatestLocal()) != "s2" || st.GetUploading() != nil || st.GetUploadDueTime() != nil { + t.Errorf("state = %v, want s2 durable and local with nothing uploading or due", st) + } + if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s1")); !os.IsNotExist(err) { + t.Errorf("s1 once CommitPromoted returned: stat = %v, want it deleted", err) } } @@ -540,8 +582,9 @@ func TestTrackerStartKeepsUploadBackoff(t *testing.T) { } } -// BeginStart and BeginCheckpoint wait only for a detach in progress, not for a -// background upload. +// BeginStart, and BeginCheckpoint in either upload mode, wait only for a detach +// in progress, not for a background upload, so a Checkpoint never waits for +// one before it checkpoints the worker. func TestTrackerStartAndCheckpointDoNotWaitForUpload(t *testing.T) { store := newFakeSnapshotStore() store.setBlocking(true) @@ -549,14 +592,18 @@ func TestTrackerStartAndCheckpointDoNotWaitForUpload(t *testing.T) { tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) done := startUpload(t, tr, store) + beginCheckpoint := func(mode ateletpb.UploadMode) func(context.Context, string) (func(), error) { + return func(ctx context.Context, uid string) (func(), error) { + return tr.BeginCheckpoint(ctx, uid, testRef("s2", 2, time.Time{}).Snapshot, mode) + } + } for _, tc := range []struct { name string begin func(context.Context, string) (func(), error) }{ {"BeginStart", tr.BeginStart}, - {"BeginCheckpoint", func(ctx context.Context, uid string) (func(), error) { - return tr.BeginCheckpoint(ctx, uid, testRef("s2", 2, time.Time{}).Snapshot) - }}, + {"BeginCheckpoint deferred", beginCheckpoint(ateletpb.UploadMode_UPLOAD_MODE_DEFERRED)}, + {"BeginCheckpoint immediate", beginCheckpoint(ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE)}, } { returnsPromptly(t, tc.name+" during a background upload", func() { end, err := tc.begin(t.Context(), trackedActorUID) @@ -573,7 +620,7 @@ func TestTrackerStartAndCheckpointDoNotWaitForUpload(t *testing.T) { } // A background upload runs even while the actor is held by a Run, Restore, or -// Checkpoint, since an upload keeps the local snapshot intact. +// deferred Checkpoint, since an upload keeps the local snapshot intact. func TestTrackerUploadRunsWhileActorHeld(t *testing.T) { store := newFakeSnapshotStore() store.setBlocking(true) @@ -581,7 +628,7 @@ func TestTrackerUploadRunsWhileActorHeld(t *testing.T) { t0 := tr.clock.Now() tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) - end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s2", 2, t0).Snapshot) + end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s2", 2, t0).Snapshot, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) if err != nil { t.Fatalf("BeginCheckpoint: %v", err) } @@ -906,9 +953,9 @@ func TestTrackerCommitLocalDropsFailedUploading(t *testing.T) { } } -// A superseded local copy stays on disk while a Run, Restore, Checkpoint, or -// PromoteSnapshot of the actor is in progress, since a Restore may be reading -// it in place. It is left for the actor's next checkpoint to prune. +// A superseded local copy stays on disk while a Run or Restore of the actor is +// in progress, since a Restore may be reading it in place. It is left for the +// actor's next checkpoint to prune. func TestTrackerKeepsSupersededCopyWhileHeld(t *testing.T) { t.Run("upload ends", func(t *testing.T) { store := newFakeSnapshotStore() @@ -1024,6 +1071,29 @@ func TestTrackerTerminateHoldsOffSupersededUploadingCleanup(t *testing.T) { } } +// A Run or Restore in progress may read an older local copy in place; +// Checkpoint and PromoteSnapshot do not. +func TestLocalInUse(t *testing.T) { + for _, tc := range []struct { + name string + hold holdKind + want bool + }{ + {name: "nothing in progress", hold: holdNone, want: false}, + {name: "Run or Restore", hold: holdStart, want: true}, + {name: "deferred Checkpoint", hold: holdCheckpoint, want: false}, + {name: "immediate Checkpoint or PromoteSnapshot", hold: holdDurable, want: false}, + {name: "Terminate", hold: holdTerminate, want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + a := &actorSnapshots{actorUID: trackedActorUID, hold: tc.hold} + if got := a.localInUse(); got != tc.want { + t.Errorf("localInUse() = %v, want %v", got, tc.want) + } + }) + } +} + // Uploading is saved to the snapshot state file, so an atelet restart mid-upload // resumes s1 unless a newer snapshot was already durable before the restart. func TestTrackerResumesUploadingAfterRestart(t *testing.T) { @@ -1117,6 +1187,7 @@ func TestTrackerResumesUploadingAfterRestart(t *testing.T) { } store := newFakeSnapshotStore() store.setBlocking(true) + store.setIgnoreCancel(true) ctx, cancel := context.WithCancel(t.Context()) before := New(ctx, store, newStoreTracker(t, nil).clock, WithUploadDelay(testUploadDelay), WithDetachDelay(testDetachDelay)) before.Start() @@ -1126,13 +1197,22 @@ func TestTrackerResumesUploadingAfterRestart(t *testing.T) { if got := waitStarted(t, store); got != "s1" { t.Fatalf("before started = %q, want s1", got) } - before.CommitDurable(trackedActorUID, testRef("s2", 2, t0.Add(time.Minute))) - // Capture the state file as a hard crash mid-upload leaves it. + commitDone := make(chan struct{}) + go func() { + defer close(commitDone) + before.CommitDurable(trackedActorUID, testRef("s2", 2, t0.Add(time.Minute))) + }() + waitCanceled(t, store) + // Capture the state file as a hard crash mid-upload leaves it: + // CommitDurable saved LatestDurable=s2, and s1's upload has not yet + // cleared Uploading. stateFile := ateletpath.ActorSnapshotStateFile(trackedActorUID) crashedState, err := os.ReadFile(stateFile) if err != nil { t.Fatal(err) } + store.release <- context.Canceled + waitClosed(t, commitDone, "CommitDurable") cancel() if err := before.Wait(t.Context()); err != nil { t.Fatalf("before.Wait: %v", err) @@ -1277,6 +1357,54 @@ func TestTrackerUploadWaitsForPromote(t *testing.T) { }) } +// A background upload that comes due while a Checkpoint under +// UPLOAD_MODE_IMMEDIATE holds the actor waits until the checkpoint ends, then +// skips if the checkpoint made a newer snapshot durable or runs if the +// checkpoint failed. +func TestTrackerUploadWaitsForImmediateCheckpoint(t *testing.T) { + for _, tc := range []struct { + name string + // commit is set when the checkpoint makes s2 durable before it ends. + commit bool + want []string + }{ + {name: "checkpoint succeeds", commit: true}, + {name: "checkpoint fails", want: []string{"s1"}}, + } { + t.Run(tc.name, func(t *testing.T) { + store := newFakeSnapshotStore() + tr := newIdleTestTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + + s2 := testRef("s2", 2, tr.clock.Now()) + endCheckpoint, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, s2.Snapshot, ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE) + if err != nil { + t.Fatalf("BeginCheckpoint: %v", err) + } + stepTo(tr, tr.clock.Now().Add(testUploadDelay)) + processNext(t, tr, opUpload) + if got := store.uploads(); len(got) != 0 { + t.Fatalf("uploads during the checkpoint = %v, want none", got) + } + if n := tr.queues[opUpload].Len(); n != 0 { + t.Fatalf("queued uploads during the checkpoint = %d, want none: the checkpoint's end queues it", n) + } + + if tc.commit { + s2.UploadTime = tr.clock.Now() + tr.CommitDurable(trackedActorUID, s2) + } + endCheckpoint() + if n := tr.queues[opUpload].Len(); n != 0 { + processNext(t, tr, opUpload) + } + if got := store.uploads(); !slices.Equal(got, tc.want) { + t.Errorf("uploads after the checkpoint ended = %v, want %v", got, tc.want) + } + }) + } +} + // An out-of-order CommitLocal with a lower generation than Uploading does not // replace a failed Uploading target, reset its backoff, or delete Uploading's // local copy when it finishes. diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index 322bf16051..898a16e066 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -2128,6 +2128,10 @@ type gatingTrackerStore struct { objects *recordingObjectStorage started chan struct{} release chan struct{} + // canceled, if set, receives once an upload's context ends, and that + // upload still waits for release before it returns, as an object-store + // call that is finishing would. + canceled chan struct{} } func (s *gatingTrackerStore) UploadLocalSnapshot(ctx context.Context, actorUID string, ref snapshottracker.SnapshotRef) (bool, error) { @@ -2135,6 +2139,10 @@ func (s *gatingTrackerStore) UploadLocalSnapshot(ctx context.Context, actorUID s select { case <-s.release: case <-ctx.Done(): + if s.canceled != nil { + s.canceled <- struct{}{} + <-s.release + } return false, ctx.Err() } uploaded, err := s.inner.UploadLocalSnapshot(ctx, actorUID, ref) @@ -2281,6 +2289,92 @@ func TestCheckpointKeepsSnapshotBeingUploaded(t *testing.T) { } } +// TestImmediateCheckpointCancelsBackgroundUpload suspends a resumed actor with +// an immediate upload while the background upload of its previous pause +// snapshot still runs: the checkpoint checkpoints the worker without waiting +// for that upload, then cancels it, and returns only once it has stopped, so +// the control plane can delete the older snapshot's objects. +func TestImmediateCheckpointCancelsBackgroundUpload(t *testing.T) { + a := runPluginActor(t) + clk := clocktesting.NewFakeClock(time.Now()) + store := &gatingTrackerStore{ + inner: &trackerStore{s: a.s}, + started: make(chan struct{}, 1), + release: make(chan struct{}), + canceled: make(chan struct{}, 1), + } + a.s.snapshots = snapshottracker.New(t.Context(), store, clk, + snapshottracker.WithUploadDelay(time.Second), + snapshottracker.WithDetachDelay(time.Hour)) + a.s.snapshots.Start() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if err := a.s.snapshots.Wait(ctx); err != nil { + t.Errorf("Wait: %v", err) + } + }) + // A canceled upload holds on until released; release it before Wait + // even when the test stops early. + release := sync.OnceFunc(func() { close(store.release) }) + t.Cleanup(release) + + cpResp, err := a.checkpoint(t, a.snapshot("pause-snap-1", 1), ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + if err != nil { + t.Fatalf("Checkpoint (pause-snap-1): %v", err) + } + stepToUpload(t, clk, cpResp.GetState().GetUploadDueTime().AsTime(), store.started) + a.restoreLocal(t, "pause-snap-1", 1) + a.worker.mu.Lock() + delete(a.worker.actorDirs, "CheckpointWorkload") + a.worker.mu.Unlock() + + type checkpointResult struct { + state *ateletpb.ActorSnapshotState + err error + } + resCh := make(chan checkpointResult, 1) + go func() { + resp, err := a.checkpoint(t, a.snapshot("suspend-snap-2", 2), ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE) + resCh <- checkpointResult{resp.GetState(), err} + }() + select { + case <-store.canceled: + case res := <-resCh: + t.Fatalf("immediate Checkpoint returned (%v, %v) without canceling the background upload", res.state, res.err) + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for the immediate Checkpoint to cancel the background upload") + } + a.worker.mu.Lock() + _, checkpointed := a.worker.actorDirs["CheckpointWorkload"] + a.worker.mu.Unlock() + if !checkpointed { + t.Error("background upload canceled before the worker was checkpointed, want the checkpoint not to wait for it") + } + select { + case res := <-resCh: + t.Fatalf("immediate Checkpoint returned (%v, %v) before the canceled upload stopped", res.state, res.err) + case <-time.After(50 * time.Millisecond): + } + + release() + var res checkpointResult + select { + case res = <-resCh: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for the immediate Checkpoint once the canceled upload stopped") + } + if res.err != nil { + t.Fatalf("Checkpoint (suspend-snap-2): %v", res.err) + } + if got := res.state.GetLatestDurable().GetSnapshot().GetSnapshotUuid(); got != "suspend-snap-2" { + t.Errorf("latest_durable = %q, want suspend-snap-2", got) + } + if res.state.GetUploading() != nil { + t.Errorf("uploading = %v, want no upload in progress once the checkpoint returns", res.state.GetUploading()) + } +} + // resumedWithPendingUpload runs a plugin actor, pauses it into pause-snap-1 // with a deferred upload, and resumes it from that snapshot's local copy // before the upload is due. It returns the actor, its gated tracker store, and @@ -2332,6 +2426,34 @@ func resumedWithPendingUpload(t *testing.T) (*pluginActor, *gatingTrackerStore, return a, store, stepPastDue } +// TestImmediateCheckpointHoldsOffBackgroundUpload checkpoints an actor resumed +// from its pause snapshot while that snapshot's background upload comes due: +// under UPLOAD_MODE_IMMEDIATE the upload does not start, since the checkpoint +// would only cancel it, while under UPLOAD_MODE_DEFERRED it starts. +func TestImmediateCheckpointHoldsOffBackgroundUpload(t *testing.T) { + for _, tc := range []struct { + mode ateletpb.UploadMode + snapshot string + wantStarted bool + }{ + {mode: ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE, snapshot: "suspend-snap-2"}, + {mode: ateletpb.UploadMode_UPLOAD_MODE_DEFERRED, snapshot: "pause-snap-2", wantStarted: true}, + } { + t.Run(tc.mode.String(), func(t *testing.T) { + a, _, stepPastDue := resumedWithPendingUpload(t) + var started atomic.Bool + a.worker.onCheckpoint = func() { started.Store(stepPastDue()) } + + if _, err := a.checkpoint(t, a.snapshot(tc.snapshot, 2), tc.mode); err != nil { + t.Fatalf("Checkpoint (%s): %v", tc.snapshot, err) + } + if got := started.Load(); got != tc.wantStarted { + t.Errorf("background upload of pause-snap-1 started during the checkpoint = %v, want %v", got, tc.wantStarted) + } + }) + } +} + // TestTerminateHoldsOffBackgroundUpload terminates an actor resumed from its // pause snapshot, whose background upload comes due while the worker tears the // actor down: the upload does not start, since Terminate would only cancel it. diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 1ce756b21b..73ff2dadf5 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -970,7 +970,9 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest // A detach in progress finishes first, and no detach starts until the // checkpoint returns, so it cannot delete the snapshot being written. - endCheckpoint, err := s.snapshots.BeginCheckpoint(ctx, actorUID, req.GetSnapshot()) + // Under UPLOAD_MODE_IMMEDIATE, no background upload starts either, so + // none competes with this checkpoint's upload. + endCheckpoint, err := s.snapshots.BeginCheckpoint(ctx, actorUID, req.GetSnapshot(), req.GetUploadConfig().GetMode()) if err != nil { return nil, err } diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index 4121648c17..73cc50143c 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -1070,13 +1070,14 @@ groups: note: > atelet uploads deferred snapshots in the background once the upload delay or the detach delay elapses. A probe that finds the manifest committed - already without uploading is not recorded. An upload that Terminate or an - atelet shutdown cancels records error.type Canceled, and one that runs - past its timeout (--snapshot-upload-timeout plus the time the snapshot - takes at --snapshot-upload-est-throughput-mib, doubled for each - consecutive failure to at most 8 times that) records DeadlineExceeded. - Synchronous uploads from UPLOAD_MODE_IMMEDIATE checkpoints and - PromoteSnapshot are recorded on ate.actor.checkpoint.duration instead. + already without uploading is not recorded. An upload that Terminate, an + atelet shutdown, or a newer durable snapshot cancels records error.type + Canceled, and one that runs past its timeout (--snapshot-upload-timeout + plus the time the snapshot takes at --snapshot-upload-est-throughput-mib, + doubled for each consecutive failure to at most 8 times that) records + DeadlineExceeded. Synchronous uploads from UPLOAD_MODE_IMMEDIATE + checkpoints and PromoteSnapshot are recorded on + ate.actor.checkpoint.duration instead. annotations: substrate: emitted_by: [atelet] From 04c0b0e900fbe3c3c68f61ae26a98a218e75b274 Mon Sep 17 00:00:00 2001 From: haiyanmeng Date: Tue, 20 Oct 2026 22:58:13 +0000 Subject: [PATCH 27/63] Benchmarking: measure ate-api-server cold resume capacity on the fake data plane (#2686) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds what it takes to measure ate-api-server's and Postgres's cold resume capacity on the fake data plane from the benchmark automation, plus the first tests that use it. - **boomer: record server-side latency.** `resumepaused` records `locust_server_duration_milliseconds` from the `x-server-elapsed-us` trailer that `ateinterceptors.ServerUnaryInterceptor` sets, next to the client latency. The gap between the two is network and client time. - **Scrape boomer in automation runner Jobs.** The runner container declares port 8001 and `monitoring.yaml`'s `boomer-worker` job keeps `substrate-benchmark-runner` pods, so automated runs get the same latency histograms as manual ones. - **Shape the control plane for capacity runs.** A locust test can carry a `controlPlane:` block (`benchmarking/automation/controlplane.py`): - `apiServer:` sets ate-api-server's replicas, CPU and memory (requests = limits) and its Postgres pool size. - `postgres:` loads `pg_stat_statements` with a `postgres_exporter` sidecar that Prometheus scrapes, and passes extra Postgres settings. Placement is left to the install: capacity tests install with `--cordon-control-plane`, which keeps ate-api-server and Postgres on their own tainted pools, away from the load. The block patches the installed manifests after the deploy; a TODO in `controlplane.py` records the drift guards that shim still lacks. - **Summarize ladder runs per step, on the runner.** For a ladder run, `benchmarking/locust/ladder_steps.py` reads each step's boundaries from the runner's log, queries the in-cluster Prometheus over the step's steady window, and appends a `ladder_step_NNN` row per step to `stats.jsonl`: throughput, client and server latency, CPU and memory of ate-api-server, Postgres and fake-atelet, and Postgres connections, commits and waits. It is additive, like the server telemetry harvest it follows: a failure is logged and the run's other results are kept. - **Cold resume tests.** `tests.yaml` adds two: - `ateapi_cold_resume_pause_fake_2cpu`: a 100–1500/s cold resume ladder against one 2 CPU replica. - `ateapi_cold_resume_pause_fake_smoke`: the same setup at 50 actors and 20 → 50/s, a quick end-to-end check. Each actor books 10m CPU, since a fake Worker reports its node's allocatable as its capacity and placement must never bind. The benchmarking README explains the per-step rows. **Running them.** The target cluster needs the `ate-control-plane` and `ate-postgres` node pools `--cordon-control-plane` expects, and a node with room for the runner. On current `main` the target cluster's env also needs `SNAPSHOT_LOCATION` (#2544) and `ATE_INGRESS_ALLOWED_SPIFFE_IDS` (#2390), which every install now requires. Addresses #2261 - [x] Tests pass (`go test -race ./internal/benchmarking/...`, the `benchmarking/automation` and `benchmarking/locust` Python tests, `hack/verify-all.sh`) - [x] Appropriate changes to documentation are included in the PR --- benchmarking/README.md | 30 ++ benchmarking/automation/Dockerfile | 2 +- benchmarking/automation/controlplane.py | 275 ++++++++++++++++++ .../automation/manifests/runner-job.yaml.tmpl | 5 + benchmarking/automation/test_controlplane.py | 204 +++++++++++++ benchmarking/automation/test_locust.py | 127 ++++++++ benchmarking/automation/tests.yaml | 84 ++++++ benchmarking/automation/testtypes/__init__.py | 3 +- benchmarking/automation/testtypes/locust.py | 24 +- benchmarking/locust/Dockerfile | 1 + benchmarking/locust/ladder_steps.py | 253 ++++++++++++++++ benchmarking/locust/runner.py | 14 + .../locust/unit_tests/test_ladder_steps.py | 223 ++++++++++++++ .../unit_tests/test_server_telemetry.py | 23 ++ benchmarking/monitoring.yaml | 49 +++- internal/benchmarking/boomer/metrics/prom.go | 17 +- .../benchmarking/boomer/metrics/prom_test.go | 73 +++++ .../boomer/resumepaused/resumepaused.go | 45 ++- .../boomer/resumepaused/resumepaused_test.go | 58 ++++ 19 files changed, 1488 insertions(+), 22 deletions(-) create mode 100644 benchmarking/automation/controlplane.py create mode 100644 benchmarking/automation/test_controlplane.py create mode 100644 benchmarking/automation/test_locust.py create mode 100644 benchmarking/locust/ladder_steps.py create mode 100644 benchmarking/locust/unit_tests/test_ladder_steps.py create mode 100644 internal/benchmarking/boomer/metrics/prom_test.go diff --git a/benchmarking/README.md b/benchmarking/README.md index 553597aec2..b8bc2aef47 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -588,6 +588,36 @@ warm-up until they have all joined at the spawn rate. * `CreateActor`, `ResumeActorWarmup`, `SuspendActorAsyncWarmup`: A user's warm-up of its actor, in the step that added the user. +#### Analyzing a ResumePaused Ladder Run + +`stats.csv` and the other rows of `stats.jsonl` cover a whole run, so they +blend every step of a ladder. After a ladder run the runner adds one +`stats.jsonl` row per step, with `metric` `ladder_step_001`, +`ladder_step_002`, and so on ([`locust/ladder_steps.py`](locust/ladder_steps.py)). +It reads each step's start and end from the run's log and queries the +benchmarking Prometheus over the step's steady window, from 30 s after the +step starts to 5 s before it ends (`window_start` and `window_end`). A row +holds the cold resume and pause rates, failures by operation +(`failures_`), client and server latency, the CPU and memory of +ate-api-server, Postgres and fake-atelet, and Postgres's connections, +commits and waits. + +The runner logs and skips a step with no end in the log and a step too short +to leave a window after the trim. A failed query adds no step rows at all +rather than a ladder with gaps, and leaves the run's other results as they +are. Read a step's figures against its `target_rps`: a `resume_rps` below it +means the step fell short of its rate. + +To pull the step rows out of a run's results: + +```bash +RUN=/runs//run_date=/run_ts=/run_tag= +gcloud storage cat "$RUN/stats.jsonl" | grep '"metric": "ladder_step_' +``` + +`` is the orchestrator's `--dest`, and its log has the full `RUN` path +on each `Uploaded ... -> gs://...` line. + ### Isolate Mode A real cold resume is limited by the node's restore capacity, which hides the diff --git a/benchmarking/automation/Dockerfile b/benchmarking/automation/Dockerfile index acbecda152..6a77157a72 100644 --- a/benchmarking/automation/Dockerfile +++ b/benchmarking/automation/Dockerfile @@ -62,7 +62,7 @@ RUN python3 -m venv /opt/automation/venv \ && /opt/automation/venv/bin/pip install --no-cache-dir -r /opt/automation/requirements.txt ENV PATH=/opt/automation/venv/bin:$PATH -COPY orchestrator.py util.py /opt/automation/ +COPY orchestrator.py util.py controlplane.py /opt/automation/ COPY testtypes/ /opt/automation/testtypes/ COPY manifests/runner-job.yaml.tmpl manifests/nighthawk-ingress-runner-job.yaml.tmpl /opt/automation/manifests/ diff --git a/benchmarking/automation/controlplane.py b/benchmarking/automation/controlplane.py new file mode 100644 index 0000000000..1b4003913d --- /dev/null +++ b/benchmarking/automation/controlplane.py @@ -0,0 +1,275 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shapes the control plane for a capacity run, as a test entry's +`controlPlane:` block asks: an `apiServer:` section for ate-api-server +(see API_SERVER_KNOBS) and a `postgres:` section (see POSTGRES_DEFAULTS). + +A test type validates the block with validate() and calls shape() from its +pre_test. Isolation from the load is the install's: --cordon-control-plane +in ateArgs keeps ate-api-server and Postgres on their own tainted pools. +""" + +import json +import re +from typing import Any + +from util import run, run_no_check + +NAMESPACE = "ate-system" + +# Knobs of the `apiServer:` section. None has a default here: the install +# defines ate-api-server's shape, and a run overrides only the knobs it sets. +API_SERVER_KNOBS = { + # Required, the independent variable of the run. CPU request and limit + # of each replica; Go sizes GOMAXPROCS from the limit. + "cpu", + # Memory request and limit of each replica. + "memory", + "replicas", + # pgxpool max connections per replica, set through ateapi's env + # ConfigMap (see set_pool_max_conns). + "poolMaxConns", +} + +# Knobs of the `postgres:` section. +POSTGRES_DEFAULTS = { + # Load pg_stat_statements and run postgres_exporter next to Postgres, + # for Prometheus (benchmarking/monitoring.yaml) to scrape. + "stats": True, + # Postgres settings as name=value, passed to the server as -c flags in + # the restart stats makes, over postgresql.conf. + "args": [], +} + + +def config(block: dict[str, Any]) -> dict[str, Any]: + """The `controlPlane:` block with `postgres:` merged over its defaults. + `apiServer:` holds only the knobs the block sets.""" + return { + "apiServer": dict(block.get("apiServer", {})), + "postgres": {**POSTGRES_DEFAULTS, **block.get("postgres", {})}, + } + + +def validate(block: Any) -> None: + """Raise ValueError unless block is a valid `controlPlane:` block.""" + if not isinstance(block, dict): + raise ValueError("controlPlane must be a mapping") + _check_knobs("controlPlane", block, {"apiServer", "postgres"}) + if "apiServer" not in block: + raise ValueError("needs controlPlane.apiServer") + _check_knobs("controlPlane.apiServer", block["apiServer"], API_SERVER_KNOBS) + _check_knobs("controlPlane.postgres", block.get("postgres", {}), set(POSTGRES_DEFAULTS)) + cfg = config(block) + api, pg = cfg["apiServer"], cfg["postgres"] + if "cpu" not in api or not _is_quantity(api["cpu"]): + raise ValueError("needs controlPlane.apiServer.cpu, as in 2 or 1500m") + if "memory" in api and not _is_quantity(api["memory"]): + raise ValueError("invalid controlPlane.apiServer.memory") + if "replicas" in api and not _is_positive_int(api["replicas"]): + raise ValueError("controlPlane.apiServer.replicas must be an int >= 1") + if "poolMaxConns" in api and not _is_positive_int(api["poolMaxConns"]): + raise ValueError("controlPlane.apiServer.poolMaxConns must be an int >= 1") + if not isinstance(pg["stats"], bool): + raise ValueError("controlPlane.postgres.stats must be a bool") + args = pg["args"] + if not isinstance(args, list) or not all( + isinstance(a, str) and POSTGRES_ARG_PATTERN.fullmatch(a) for a in args + ): + raise ValueError( + "controlPlane.postgres.args must be a list of name=value settings, " + "as in commit_delay=100" + ) + if args and not pg["stats"]: + raise ValueError( + "controlPlane.postgres.args needs stats: true, whose Postgres restart applies them" + ) + + +def _check_knobs(path: str, section: Any, allowed: set[str]) -> None: + if not isinstance(section, dict): + raise ValueError(f"{path} must be a mapping") + unknown = set(section) - allowed + if unknown: + raise ValueError( + f"unknown {path} knob(s) {sorted(unknown)}; allowed: {sorted(allowed)}" + ) + + +def _is_positive_int(v: Any) -> bool: + return isinstance(v, int) and not isinstance(v, bool) and v >= 1 + + +def _is_quantity(v: Any) -> bool: + """A Kubernetes quantity as tests.yaml spells it: 2, 0.5, "1500m", "4Gi".""" + if isinstance(v, bool): + return False + if isinstance(v, (int, float)): + return v > 0 + return isinstance(v, str) and bool(QUANTITY_PATTERN.fullmatch(v)) + + +# A Postgres setting as postgres.args spells it: a lowercase name, an equals +# sign, and a value with no whitespace, which the -c flag takes as is. +POSTGRES_ARG_PATTERN = re.compile(r"[a-z_]+=\S+") + + +# A non-negative Kubernetes quantity: a decimal number with a binary-SI +# suffix (Ki to Ei), a decimal-SI suffix (m to E), or a decimal exponent. +QUANTITY_PATTERN = re.compile( + r"([0-9]+(\.[0-9]*)?|\.[0-9]+)([KMGTPE]i|[mkMGTPE]|[eE][+-]?[0-9]+)?" +) + + +# TODO: guard against drift in the manifests these patches assume, so a +# change fails the run instead of quietly skewing its numbers. Tests that +# read manifests/ate-install and check the ate-api-server container, the +# Postgres base args postgres_args restates, the socket volume, and the env +# ConfigMap name and key; and a check after shape() that the running pods +# carry the requested resources and Postgres the requested settings. +def shape(cfg: dict[str, Any]) -> None: + """Shape the control plane for the run: Postgres statistics, the pool + size, then ate-api-server's resources, whose rollout picks up the pool + size. No unpatch: every test redeploys substrate.""" + api, pg = cfg["apiServer"], cfg["postgres"] + if pg["stats"]: + enable_postgres_stats(pg["args"]) + if "poolMaxConns" in api: + set_pool_max_conns(api["poolMaxConns"]) + resize_api_server(api) + # The placement of the run, for comparing runs: a step compares across + # runs only when ateapi and Postgres sat on the same nodes. + run_no_check( + ["kubectl", "-n", NAMESPACE, "get", "pods", "-o", "wide", + "-l", "app in (ate-api-server,postgres)"] + ) + + +POSTGRES_EXPORTER_IMAGE = "quay.io/prometheuscommunity/postgres-exporter:v0.17.1" + + +def postgres_args(settings: list[str]) -> list[str]: + """The postgres container's args: the base manifest's config_file, which + a strategic merge would otherwise drop, pg_stat_statements, then each of + settings as a -c flag, which overrides postgresql.conf.""" + args = [ + "-c", "config_file=/etc/postgresql/postgresql.conf", + "-c", "shared_preload_libraries=pg_stat_statements", + ] + for setting in settings: + args += ["-c", setting] + return args + + +def enable_postgres_stats(settings: list[str]) -> None: + """Load pg_stat_statements, apply settings, and add a postgres_exporter + sidecar, which reaches the server over the pod's unix socket (trusted by + pg_hba). One patch, thus one Postgres restart, before ateapi rolls.""" + patch = { + "spec": { + "template": { + "spec": { + "containers": [ + { + "name": "postgres", + # Strategic merge replaces args whole. + "args": postgres_args(settings), + }, + { + "name": "postgres-exporter", + "image": POSTGRES_EXPORTER_IMAGE, + "args": ["--collector.stat_statements"], + "env": [ + { + "name": "DATA_SOURCE_NAME", + "value": "host=/var/run/postgresql user=postgres dbname=atepg sslmode=disable", + } + ], + # ate-system admits scrapes only to named ports + # such as metrics. + "ports": [{"name": "metrics", "containerPort": 9187}], + "volumeMounts": [ + {"name": "socket", "mountPath": "/var/run/postgresql"} + ], + "resources": {"requests": {"cpu": "100m", "memory": "128Mi"}}, + }, + ] + } + } + } + } + _patch_and_wait("statefulset", "postgres", patch) + run( + ["kubectl", "-n", NAMESPACE, "exec", "postgres-0", "-c", "postgres", "--", + "psql", "-U", "postgres", "-d", "atepg", "-v", "ON_ERROR_STOP=1", + "-c", "CREATE EXTENSION IF NOT EXISTS pg_stat_statements"] + ) + + +# ate-setup's ConfigMap of ateapi environment variables. ateapi reads +# ATE_API_POSTGRES_POOL_MAX_CONNS from it at start, over any pool_max_conns in +# the DSN. +ENV_CONFIGMAP = "ate-api-server-envvars" +POOL_MAX_CONNS_KEY = "ATE_API_POSTGRES_POOL_MAX_CONNS" + + +def pool_max_conns_patch(n: int) -> dict[str, Any]: + return {"data": {POOL_MAX_CONNS_KEY: str(n)}} + + +def set_pool_max_conns(n: int) -> None: + """Set the read/write pool size ateapi reads at start; the + resize_api_server rollout that follows picks it up. The owner and watch + pools stay at 2 and 3 connections per replica.""" + run( + ["kubectl", "-n", NAMESPACE, "patch", "configmap", ENV_CONFIGMAP, + "--type", "merge", "-p", json.dumps(pool_max_conns_patch(n))] + ) + + +def api_server_patch(api: dict[str, Any]) -> dict[str, Any]: + """A strategic merge patch that sets only the knobs api holds, so the + rest of the Deployment keeps the install's values.""" + resources = {"cpu": str(api["cpu"])} + if "memory" in api: + resources["memory"] = api["memory"] + pod_spec = { + "containers": [ + { + "name": "ate-api-server", + "resources": {"requests": resources, "limits": dict(resources)}, + } + ] + } + spec: dict[str, Any] = {"template": {"spec": pod_spec}} + if "replicas" in api: + spec["replicas"] = api["replicas"] + return {"spec": spec} + + +def resize_api_server(api: dict[str, Any]) -> None: + _patch_and_wait("deployment", "ate-api-server", api_server_patch(api)) + + +def _patch_and_wait(kind: str, name: str, patch: dict[str, Any]) -> None: + run( + ["kubectl", "-n", NAMESPACE, "patch", kind, name, + "--type=strategic", "-p", json.dumps(patch)] + ) + run( + ["kubectl", "-n", NAMESPACE, "rollout", "status", f"{kind}/{name}", + "--timeout=600s"] + ) + diff --git a/benchmarking/automation/manifests/runner-job.yaml.tmpl b/benchmarking/automation/manifests/runner-job.yaml.tmpl index 5303dcb2fb..9b6f939894 100644 --- a/benchmarking/automation/manifests/runner-job.yaml.tmpl +++ b/benchmarking/automation/manifests/runner-job.yaml.tmpl @@ -175,6 +175,11 @@ spec: env: - name: OTEL_EXPORTER_OTLP_ENDPOINT value: http://opentelemetry-collector.gke-managed-otel.svc.cluster.local:4317 + # boomer's /metrics; the boomer-worker job of + # benchmarking/monitoring.yaml scrapes it. + ports: + - name: boomer-metrics + containerPort: 8001 volumeMounts: - name: servicedns-ca mountPath: /run/servicedns-ca diff --git a/benchmarking/automation/test_controlplane.py b/benchmarking/automation/test_controlplane.py new file mode 100644 index 0000000000..b243676257 --- /dev/null +++ b/benchmarking/automation/test_controlplane.py @@ -0,0 +1,204 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for controlplane.py: +python3 benchmarking/automation/test_controlplane.py""" + +import os +import unittest +from unittest import mock + +import yaml + +import controlplane + + +def stub(test, **fakes): + for name, fake in fakes.items(): + patcher = mock.patch.object(controlplane, name, fake) + patcher.start() + test.addCleanup(patcher.stop) + + +API = {"apiServer": {"cpu": 2}} + + +class ConfigTest(unittest.TestCase): + def test_defaults(self): + cfg = controlplane.config(API) + # The install defines ate-api-server's shape; only what the block + # sets is overridden. + self.assertEqual(cfg["apiServer"], {"cpu": 2}) + self.assertEqual(cfg["postgres"], {"stats": True, "args": []}) + + +class ValidateTest(unittest.TestCase): + def test_valid_knobs(self): + controlplane.validate(API) + controlplane.validate( + {"apiServer": {"cpu": "1500m", "memory": "8Gi", "replicas": 2, "poolMaxConns": 16}, + "postgres": {"stats": False}}) + controlplane.validate( + {**API, "postgres": {"args": ["commit_delay=100", "max_wal_size=16GB"]}}) + + def test_invalid(self): + def api(**knobs): + return {"apiServer": {"cpu": 2, **knobs}} + + def pg(**knobs): + return {**API, "postgres": knobs} + + bad = [ + {}, + "2", + {"apiServer": "2"}, + {"apiServer": {}}, + {**API, "ateapi": {}}, + {**API, "postgres": "on"}, + api(cpu=0), + api(cpu="two"), + api(cpu=True), + api(cpus=2), + api(replicas=0), + api(replicas=True), + api(poolMaxConns="64"), + api(memory="4 GiB"), + pg(stats="yes"), + pg(statz=True), + pg(args="commit_delay=100"), + pg(args=["commit_delay"]), + pg(args=["commit_delay = 100"]), + pg(args=["-c commit_delay=100"]), + pg(args=[100]), + # The settings ride on the restart stats makes. + pg(stats=False, args=["commit_delay=100"]), + ] + for block in bad: + with self.assertRaises(ValueError, msg=repr(block)): + controlplane.validate(block) + + def test_errors_name_the_knob(self): + with self.assertRaisesRegex(ValueError, "controlPlane.apiServer.cpu"): + controlplane.validate({"apiServer": {}}) + with self.assertRaisesRegex(ValueError, r"unknown controlPlane.postgres knob\(s\) \['statz'\]"): + controlplane.validate({**API, "postgres": {"statz": True}}) + + +class ApiServerPatchTest(unittest.TestCase): + def patch(self, **knobs): + return controlplane.api_server_patch( + controlplane.config({"apiServer": {"cpu": 2, **knobs}})["apiServer"]) + + def test_sets_the_knobs_given(self): + patch = self.patch(memory="4Gi", replicas=1) + self.assertEqual(patch["spec"]["replicas"], 1) + pod = patch["spec"]["template"]["spec"] + resources = pod["containers"][0]["resources"] + self.assertEqual(resources["requests"], {"cpu": "2", "memory": "4Gi"}) + self.assertEqual(resources["limits"], resources["requests"]) + # Placement is the install's. + self.assertNotIn("affinity", pod) + + def test_unset_knobs_keep_the_install(self): + patch = self.patch() + self.assertNotIn("replicas", patch["spec"]) + resources = patch["spec"]["template"]["spec"]["containers"][0]["resources"] + self.assertEqual(resources, {"requests": {"cpu": "2"}, "limits": {"cpu": "2"}}) + + +class ShapeTest(unittest.TestCase): + # Postgres restarts once, then ate-api-server rolls once and reads the + # pool size. + def test_order(self): + calls = [] + stub(self, **{ + name: lambda *_, name=name: calls.append(name) + for name in ("enable_postgres_stats", "set_pool_max_conns", "resize_api_server", "run_no_check") + }) + controlplane.shape(controlplane.config({"apiServer": {"cpu": 2, "poolMaxConns": 40}})) + self.assertEqual( + [c for c in calls if c != "run_no_check"], + ["enable_postgres_stats", "set_pool_max_conns", "resize_api_server"], + ) + + def test_unset_pool_size_keeps_the_install(self): + calls = [] + stub(self, + enable_postgres_stats=lambda *_: None, + set_pool_max_conns=lambda n: calls.append(n), + resize_api_server=lambda *_: None, + run_no_check=lambda *_: None) + controlplane.shape(controlplane.config(API)) + self.assertEqual(calls, []) + + def test_passes_the_settings(self): + got = [] + stub(self, + enable_postgres_stats=lambda settings: got.append(settings), + resize_api_server=lambda *_: None, + run_no_check=lambda *_: None) + controlplane.shape(controlplane.config({**API, "postgres": {"args": ["commit_delay=100"]}})) + self.assertEqual(got, [["commit_delay=100"]]) + + +class PostgresExporterTest(unittest.TestCase): + """The exporter's port is one ate-system's observability NetworkPolicy + admits; any other name leaves Prometheus timing out on the scrape.""" + + POLICIES = os.path.join( + os.path.dirname(__file__), "..", "..", "manifests", "ate-install", "ate-system-networkpolicies.yaml" + ) + + def admitted_ports(self): + for doc in yaml.safe_load_all(open(self.POLICIES)): + if doc and doc["metadata"]["name"] == "ate-system-observability": + return {p["port"] for rule in doc["spec"]["ingress"] for p in rule["ports"]} + self.fail("no ate-system-observability NetworkPolicy") + + def test_exporter_port_is_admitted(self): + patches = [] + stub(self, + _patch_and_wait=lambda kind, obj, patch: patches.append(patch), + run=lambda *_, **__: None) + controlplane.enable_postgres_stats([]) + containers = patches[0]["spec"]["template"]["spec"]["containers"] + exporter = next(c for c in containers if c["name"] == "postgres-exporter") + admitted = self.admitted_ports() + for port in exporter["ports"]: + self.assertIn(port["name"], admitted) + + +class PostgresArgsTest(unittest.TestCase): + def test_base_args_first_then_settings(self): + self.assertEqual( + controlplane.postgres_args(["commit_delay=100", "wal_buffers=64MB"]), + [ + "-c", "config_file=/etc/postgresql/postgresql.conf", + "-c", "shared_preload_libraries=pg_stat_statements", + "-c", "commit_delay=100", + "-c", "wal_buffers=64MB", + ], + ) + + +class PoolMaxConnsTest(unittest.TestCase): + def test_patch(self): + self.assertEqual( + controlplane.pool_max_conns_patch(40), + {"data": {"ATE_API_POSTGRES_POOL_MAX_CONNS": "40"}}, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarking/automation/test_locust.py b/benchmarking/automation/test_locust.py new file mode 100644 index 0000000000..0b3613ac15 --- /dev/null +++ b/benchmarking/automation/test_locust.py @@ -0,0 +1,127 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for testtypes/locust.py: +python3 benchmarking/automation/test_locust.py""" + +import os +import unittest +from unittest import mock + +import yaml + +import controlplane +import orchestrator +from testtypes import locust + + +def entry(**fields): + return { + "name": "t", + "file": "/app/tests/resumepaused.py,/app/shapes/ladder_shape.py", + "duration": "21m", + "users": 200, + **fields, + } + + +CONTROL_PLANE = {"apiServer": {"cpu": 2}} + + +class ValidateTest(unittest.TestCase): + def test_without_control_plane(self): + locust.validate(entry()) + self.assertIsNone(locust.control_plane_config(entry())) + + def test_with_control_plane(self): + test = entry(runnerCpu="8", runnerMemory="2Gi", controlPlane=CONTROL_PLANE) + locust.validate(test) + self.assertEqual(locust.control_plane_config(test), controlplane.config(CONTROL_PLANE)) + + # controlplane checks the block; its error names the test. + def test_invalid_control_plane_names_the_test(self): + with self.assertRaisesRegex(ValueError, "locust test 't': needs controlPlane.apiServer.cpu"): + locust.validate(entry(controlPlane={"apiServer": {}})) + + def test_missing_required(self): + test = entry() + del test["users"] + with self.assertRaises(ValueError): + locust.validate(test) + + +class PreTestTest(unittest.TestCase): + def shaped(self, test): + calls = [] + patcher = mock.patch.object(controlplane, "shape", calls.append) + patcher.start() + self.addCleanup(patcher.stop) + locust.pre_test(test) + return calls + + def test_without_control_plane_leaves_the_cluster(self): + self.assertEqual(self.shaped(entry()), []) + + def test_shapes_the_control_plane(self): + self.assertEqual(self.shaped(entry(controlPlane=CONTROL_PLANE)), [controlplane.config(CONTROL_PLANE)]) + + +class JobTest(unittest.TestCase): + def render(self, test): + subs = { + "JOB_NAME": "j", "NAME": "t", "TAG": "c", "IMAGE": "i", "DEST": "d", + **locust.job_subs(test), + } + tmpl = locust.job_tmpl(os.path.join(os.path.dirname(__file__), "manifests")) + text = orchestrator.render_template(tmpl, subs, test.get("flags", [])) + self.assertNotIn("${", text) + jobs = [d for d in yaml.safe_load_all(text) if d and d["kind"] == "Job"] + self.assertEqual(len(jobs), 1) + return jobs[0]["spec"]["template"]["spec"] + + def test_defaults_render(self): + test = entry() + pod = self.render(test) + self.assertNotIn("affinity", pod) + self.assertEqual( + pod["containers"][0]["resources"], + {"requests": {"cpu": "500m", "memory": "512Mi"}}, + ) + # monitoring.yaml's boomer-worker job scrapes this port. + self.assertIn( + {"name": "boomer-metrics", "containerPort": 8001}, + pod["containers"][0]["ports"], + ) + + def test_sized_runner_renders(self): + test = entry(runnerCpu="8", runnerMemory="2Gi", controlPlane=CONTROL_PLANE, + flags=["--actors", "200"]) + pod = self.render(test) + self.assertEqual( + pod["containers"][0]["resources"]["requests"], + {"cpu": "8", "memory": "2Gi"}, + ) + self.assertEqual(pod["containers"][0]["args"][-2:], ["--actors", "200"]) + + +class TestsYamlTest(unittest.TestCase): + def test_entries_validate(self): + path = os.path.join(os.path.dirname(__file__), "tests.yaml") + with open(path) as f: + tests = yaml.safe_load(f)["tests"] + orchestrator.validate_and_normalize_tests(tests) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarking/automation/tests.yaml b/benchmarking/automation/tests.yaml index efd3bcf77d..945fdda3d1 100644 --- a/benchmarking/automation/tests.yaml +++ b/benchmarking/automation/tests.yaml @@ -112,6 +112,25 @@ # seconds, or the step falls short of its rate. A resumepaused iteration is a # resume plus a pause, so size it by the p99 of both together. # +# An optional `controlPlane:` block shapes the control plane for a capacity +# run; pre_test applies it after the deploy. Its `apiServer:` section, which +# is required, sizes ate-api-server: +# cpu Required. CPU of each replica, set as both request and +# limit (e.g. 2 or "1500m"). +# memory Optional. Memory request and limit of each replica. +# replicas Optional. +# poolMaxConns Optional. pgxpool max connections per replica, set +# through ateapi's env ConfigMap. +# An optional knob left unset keeps the installed value. +# Its optional `postgres:` section: +# stats Optional (default true). Load pg_stat_statements and add +# a postgres_exporter sidecar, scraped by monitoring.yaml. +# args Optional. Postgres settings as name=value (e.g. +# commit_delay=100), applied in the restart stats makes. +# The block leaves placement to the install. A capacity run sets +# --cordon-control-plane in ateArgs, which keeps ate-api-server and Postgres +# off the nodes that run the load, on the tainted pools that flag expects. +# # ---- type: nighthawk-ingress -------------------------------------------------------- # # Runs the Nighthawk adaptive router-capacity benchmark. `duration` is the @@ -662,3 +681,68 @@ tests: nighthawk-ingress: envoyCpu: 16 tailLatencySloMs: 25 + # ate-api-server and Postgres cold resume ceiling on the fake data plane. + # Each user owns one actor: its first iteration creates the actor and + # makes it cold, in the step that adds the user, and every later one + # resumes it and pauses it again. fake-atelet answers each call, restore + # and checkpoint alike, after 150 ms on every atelet node, so a cycle + # spends 300 ms in the data plane, after minting the actor's certificate + # on each restore as ateom does. The 200 fake Workers hold 1000 actors + # each, and at 10m CPU an actor books next to nothing of a Worker's CPU, + # which a fake Worker reports as its node's allocatable, so placement + # never binds on any node size. resumepaused sends the actors no + # requests, which the fake data plane needs. + - name: ateapi_cold_resume_pause_fake_2cpu + type: locust + description: "ate-api-server cold resume on the fake data plane, ate-api-server at 2 CPUs: rate ladder 100-1500 cold resumes/s" + targetCluster: dev + file: /app/tests/resumepaused.py,/app/shapes/ladder_shape.py + duration: 23m + users: 1000 + workerCount: 200 + actorCPU: "10m" + runnerCpu: "8" + runnerMemory: "2Gi" + isolate: + delay: 150ms + ateArgs: + - "--cordon-control-plane" + controlPlane: + apiServer: + replicas: 1 + cpu: 2 + memory: 4Gi + poolMaxConns: 64 + flags: + - "--ladder-spawn-rate" + - "1000" + - "--ladder" + - "1000:3m@100,1000:3m@200,1000:3m@400,1000:3m@600,1000:3m@800,1000:3m@1000,1000:3m@1500" + # Smoke for the cold resume setup above: the same fake data plane and + # ate-api-server shape at a load far below its ceiling, to check a change + # end to end in minutes. 50 users, each a cycle of about 300 ms, can make + # some 150 cold resumes/s, so both steps reach their rate. + - name: ateapi_cold_resume_pause_fake_smoke + type: locust + description: "ate-api-server cold resume smoke on the fake data plane, ate-api-server at 2 CPUs: 50 actors, 20 then 50 cold resumes/s" + targetCluster: dev + file: /app/tests/resumepaused.py,/app/shapes/ladder_shape.py + duration: 6m + users: 50 + workerCount: 2 + actorCPU: "10m" + isolate: + delay: 150ms + ateArgs: + - "--cordon-control-plane" + controlPlane: + apiServer: + replicas: 1 + cpu: 2 + memory: 4Gi + poolMaxConns: 64 + flags: + - "--ladder-spawn-rate" + - "50" + - "--ladder" + - "50:2m@20,50:2m@50" diff --git a/benchmarking/automation/testtypes/__init__.py b/benchmarking/automation/testtypes/__init__.py index 2bd024b20b..5890ef6046 100644 --- a/benchmarking/automation/testtypes/__init__.py +++ b/benchmarking/automation/testtypes/__init__.py @@ -22,8 +22,7 @@ build_image(commit) builds + pushes the runner image; called lazily, once per (target cluster, type). pre_test(test) after deploy_substrate, before deploy_workloads — - the type's chance to shape the cluster (nighthawk-ingress - pins the router; locust needs nothing). + the type's chance to shape the cluster. job_tmpl(manifests_dir) path of the type's own Job template. job_subs(test) the type's ${...} substitutions for it. The test diff --git a/benchmarking/automation/testtypes/locust.py b/benchmarking/automation/testtypes/locust.py index fd68897207..bf2aaa2540 100644 --- a/benchmarking/automation/testtypes/locust.py +++ b/benchmarking/automation/testtypes/locust.py @@ -16,21 +16,39 @@ The runner Job wraps benchmarking/locust/runner.py, which drives locust (and boomer for glutton tests) and uploads its own results. + +An optional `controlPlane:` block shapes ate-api-server and Postgres for a +capacity run (see controlplane.py); without it pre_test leaves the cluster +as installed. """ import os from typing import Any +import controlplane from util import build_and_push TEST_TYPE = "locust" +def control_plane_config(test: dict[str, Any]) -> dict[str, Any] | None: + """The test's `controlPlane:` block merged over its defaults, or None.""" + if "controlPlane" not in test: + return None + return controlplane.config(test["controlPlane"]) + + def validate(test: dict[str, Any]) -> None: name = test.get("name") for field in ("file", "duration", "users"): if field not in test: raise ValueError(f"locust test {name!r} missing {field!r}") + if "controlPlane" not in test: + return + try: + controlplane.validate(test["controlPlane"]) + except ValueError as e: + raise ValueError(f"locust test {name!r}: {e}") from None def build_image(commit: str) -> str: @@ -43,7 +61,11 @@ def build_image(commit: str) -> str: def pre_test(test: dict[str, Any]) -> None: - """Nothing to shape on the cluster before a locust run.""" + """With a `controlPlane:` block, shape the control plane for the run.""" + cfg = control_plane_config(test) + if cfg is None: + return + controlplane.shape(cfg) def job_tmpl(manifests_dir: str) -> str: diff --git a/benchmarking/locust/Dockerfile b/benchmarking/locust/Dockerfile index a1c85050db..c2ecb40d8c 100644 --- a/benchmarking/locust/Dockerfile +++ b/benchmarking/locust/Dockerfile @@ -59,6 +59,7 @@ COPY benchmarking/locust/runner.py /app/runner.py COPY benchmarking/locust/cluster_facts.py /app/cluster_facts.py COPY benchmarking/locust/server_telemetry.py /app/server_telemetry.py COPY benchmarking/locust/phase_breakdown.py /app/phase_breakdown.py +COPY benchmarking/locust/ladder_steps.py /app/ladder_steps.py COPY benchmarking/analysis/phase_report.py /app/phase_report.py ENV PYTHONPATH=/app:/app/deps diff --git a/benchmarking/locust/ladder_steps.py b/benchmarking/locust/ladder_steps.py new file mode 100644 index 0000000000..654a0cd9c3 --- /dev/null +++ b/benchmarking/locust/ladder_steps.py @@ -0,0 +1,253 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Per-step rows for stats.jsonl from a ladder run (shapes/ladder_shape.py). + +stats.jsonl's other rows cover the whole run, so they blend every step of a +ladder. This reads each step's start and end from the runner's log and +queries Prometheus over the step's steady window: from SETTLE seconds after +the step starts, as boomer reads a new target rate within 10 s, to TAIL +seconds before it ends. Each step becomes one `ladder_step_NNN` row. + +Like the server telemetry, this is additive: runner.py logs a failure here +and keeps the locust measurements. +""" + +import json +import re +import urllib.parse +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable, TextIO + +# Seconds trimmed from each end of a step. +SETTLE = 30 +TAIL = 5 + +STEP_RE = re.compile( + r"\[locust\] \[(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d),\d+\].*" + r"Ladder step (\d+)/(\d+): users=(\d+).*target_rps=([\d.]+|unchanged)" +) +DONE_RE = re.compile(r"\[locust\] \[(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d),\d+\].*Ladder complete") + + +def _ts(s: str) -> int: + return int(datetime.strptime(s, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc).timestamp()) + + +def steps(lines: Iterable[str]) -> list[dict[str, Any]]: + """The ladder's steps from runner log lines, each with its start and end + as Unix seconds. A step ends where the next starts; the last ends at + "Ladder complete", or is None if the log has no such line. A step logged + with target_rps=unchanged keeps the previous step's rate, None before + any step sets one.""" + out: list[dict[str, Any]] = [] + done = None + for line in lines: + if m := STEP_RE.search(line): + rps = out[-1]["target_rps"] if out else None + if m[5] != "unchanged": + rps = float(m[5]) + out.append({"step": int(m[2]), "users": int(m[4]), + "target_rps": rps, "start": _ts(m[1])}) + elif m := DONE_RE.search(line): + done = _ts(m[1]) + for i, s in enumerate(out): + s["end"] = out[i + 1]["start"] if i + 1 < len(out) else done + return out + + +# Seconds a query may take, so an unreachable Prometheus fails the summary +# instead of hanging the runner. +QUERY_TIMEOUT_S = 30 + + +def _get_json(url: str) -> Any: + with urllib.request.urlopen(url, timeout=QUERY_TIMEOUT_S) as r: + return json.load(r) + + +# Not server_telemetry.py's helper, which turns a failed query into an empty +# result: a step's row must fail instead of carrying a gap. +def query(prom: str, q: str, t: int) -> list[dict[str, Any]]: + url = f"{prom.rstrip('/')}/api/v1/query?" + urllib.parse.urlencode({"query": q, "time": t}) + data = _get_json(url) + if data.get("status") != "success": + raise RuntimeError(f"{q}: {data}") + return data["data"]["result"] + + +def scalar(prom: str, q: str, t: int) -> float | None: + """The query's first value, or None if it has none or it is NaN.""" + r = query(prom, q, t) + if not r: + return None + v = float(r[0]["value"][1]) + return None if v != v else v + + +def by_label(prom: str, q: str, t: int, label: str) -> dict[str, float]: + return {r["metric"].get(label, ""): float(r["value"][1]) for r in query(prom, q, t)} + + +# TODO: separate the test's metrics from the harness. summarize() and +# memory() hardcode the cold resume test's queries (its request names, +# containers and database), so each new ladder test means another copy. +# The test should declare the metrics it reports, leaving this module the +# step windows and the queries' execution. +def summarize(prom: str, s: dict[str, Any]) -> dict[str, Any]: + """One step's throughput, latency, CPU and Postgres figures over its + steady window.""" + a, b = s["start"] + SETTLE, s["end"] - TAIL + secs = b - a + d = f"{secs}s" + req = 'locust_requests_total{name="%s",status="%s"}' + hist = 'histogram_quantile(%s, sum by (le) (increase(%s_bucket{name="%s"%s}[%s])))' + mean = ('sum(increase(%s_sum{name="%s"%s}[%s]))' + '/sum(increase(%s_count{name="%s"%s}[%s]))') + cpu = 'sum(increase(container_cpu_usage_seconds_total{namespace="ate-system",container="%s"}[%s]))/%d' + thr = ('sum(increase(container_cpu_cfs_throttled_periods_total{namespace="ate-system",container="ate-api-server"}[%s]))' + '/sum(increase(container_cpu_cfs_periods_total{namespace="ate-system",container="ate-api-server"}[%s]))') + pg_conns = 'avg_over_time(sum(pg_stat_activity_count{datname="atepg"%s})[%s:15s])' + client, server = "locust_request_duration_milliseconds", "locust_server_duration_milliseconds" + ok = ',status="success"' + + def q(expr: str) -> float | None: + return scalar(prom, expr, b) + + def latency(metric: str, name: str, extra: str, quantile: float) -> float | None: + return q(hist % (quantile, metric, name, extra, d)) + + def mean_ms(metric: str, name: str, extra: str) -> float | None: + return q(mean % (metric, name, extra, d, metric, name, extra, d)) + + per_replica = by_label( + prom, + f'sum by (pod) (increase(container_cpu_usage_seconds_total{{namespace="ate-system",container="ate-api-server"}}[{d}]))/{secs}', + b, "pod") + out = { + "step": s["step"], + "users": s["users"], + "target_rps": s["target_rps"], + "window": [a, b], + "resume_rps": q(f"sum(increase({req % ('ResumeActorCold', 'success')}[{d}]))/{secs}"), + "pause_rps": q(f"sum(increase({req % ('SuspendActorAsync', 'success')}[{d}]))/{secs}"), + "failures": by_label(prom, f'sum by (name) (increase(locust_requests_total{{status="failure"}}[{d}]))', b, "name"), + "resume_client_p50_ms": latency(client, "ResumeActorCold", ok, 0.5), + "resume_client_p99_ms": latency(client, "ResumeActorCold", ok, 0.99), + "resume_server_p50_ms": latency(server, "ResumeActorCold", "", 0.5), + "resume_server_p90_ms": latency(server, "ResumeActorCold", "", 0.9), + "resume_server_p95_ms": latency(server, "ResumeActorCold", "", 0.95), + "resume_server_p99_ms": latency(server, "ResumeActorCold", "", 0.99), + "pause_client_p50_ms": latency(client, "SuspendActorAsync", ok, 0.5), + "pause_client_p99_ms": latency(client, "SuspendActorAsync", ok, 0.99), + "resume_client_mean_ms": mean_ms(client, "ResumeActorCold", ok), + "resume_server_mean_ms": mean_ms(server, "ResumeActorCold", ""), + "pause_client_mean_ms": mean_ms(client, "SuspendActorAsync", ok), + "pause_server_mean_ms": mean_ms(server, "SuspendActorAsync", ""), + "ateapi_cores": q(cpu % ("ate-api-server", d, secs)), + # Each replica's cores, busiest first. + "ateapi_cores_per_replica": sorted(per_replica.values(), reverse=True), + "ateapi_throttled_ratio": q(thr % (d, d)), + "postgres_cores": q(cpu % ("postgres", d, secs)), + # On the fake data plane this is fake-atelet, which runs in atelet's + # DaemonSet under atelet's container name. + "atelet_cores": q(cpu % ("atelet", d, secs)), + "pg_active_conns": q(pg_conns % (',state="active"', d)), + "pg_conns": q(pg_conns % ("", d)), + "pg_commits_per_s": q(f'sum(increase(pg_stat_database_xact_commit{{datname="atepg"}}[{d}]))/{secs}'), + "pg_lock_wait_conns": q(pg_conns % (',state="active",wait_event_type="Lock"', d)), + "pg_walwrite_wait_conns": q(pg_conns % (',state="active",wait_event="WALWrite"', d)), + } + out.update(memory(prom, d, secs, b)) + return out + + +def memory(prom: str, d: str, secs: int, b: int) -> dict[str, Any]: + """Memory over the step's window: peak working set and RSS in MiB, the + share of time stalled on memory (PSI), and Postgres's buffer cache hits.""" + ws = 'container_memory_working_set_bytes{namespace="ate-system",container="%s"}' + rss = 'container_memory_rss{namespace="ate-system",container="%s"}' + psi = 'rate(container_pressure_memory_waiting_seconds_total{namespace="ate-system",container="%s"}[%s])' + blks = 'sum(increase(pg_stat_database_blks_%s{datname="atepg"}[%s]))' + + def mib(q: str) -> float | None: + v = scalar(prom, q, b) + return None if v is None else v / (1024 * 1024) + + hit, read = scalar(prom, blks % ("hit", d), b), scalar(prom, blks % ("read", d), b) + return { + # The largest replica's peak, and all replicas' mean together. + "ateapi_mem_ws_max_mib": mib(f'max(max_over_time({ws % "ate-api-server"}[{d}]))'), + "ateapi_mem_ws_total_mib": mib(f'sum(avg_over_time({ws % "ate-api-server"}[{d}]))'), + "ateapi_mem_rss_max_mib": mib(f'max(max_over_time({rss % "ate-api-server"}[{d}]))'), + "ateapi_mem_psi": scalar(prom, f'max({psi % ("ate-api-server", d)})', b), + "postgres_mem_ws_mib": mib(f'max(max_over_time({ws % "postgres"}[{d}]))'), + "postgres_mem_rss_mib": mib(f'max(max_over_time({rss % "postgres"}[{d}]))'), + "postgres_mem_psi": scalar(prom, f'max({psi % ("postgres", d)})', b), + "pg_cache_hit_ratio": None if hit is None or read is None or hit + read == 0 else hit / (hit + read), + "pg_blks_read_per_s": None if read is None else read / secs, + } + + +def stats_record(summary: dict[str, Any], timestamp: str, tag: str, + test_name: str) -> dict[str, Any]: + """A step's summary as a stats.jsonl row, its measurements flattened to + scalars like every other row's.""" + nested = ("step", "window", "failures", "ateapi_cores_per_replica") + m = {k: v for k, v in summary.items() if k not in nested} + m["window_start"], m["window_end"] = summary["window"] + m["ateapi_cores_busiest_replica"] = max(summary["ateapi_cores_per_replica"], default=None) + for name, n in summary["failures"].items(): + m[f"failures_{name}"] = n + return { + "timestamp": timestamp, + "tag": tag, + "test_name": test_name, + "metric": f"ladder_step_{summary['step']:03d}", + "measurements": m, + } + + +def append_ladder_steps(jsonl_path: Path, log_path: Path, prom_url: str, + timestamp: str, tag: str, test_name: str, + logs: TextIO) -> int: + """Append a row per measurable ladder step in the runner log at + log_path to jsonl_path, and return how many. A run that is no ladder + has none. Every step is queried before any row is written, so a failed + query leaves no partial ladder.""" + with open(log_path, errors="replace") as f: + ladder = steps(f) + rows = [] + for s in ladder: + if s["end"] is None: + _log(logs, f"Ladder step {s['step']} has no end in the log; skipped") + continue + if s["end"] - s["start"] <= SETTLE + TAIL: + _log(logs, f"Ladder step {s['step']} lasts {s['end'] - s['start']}s, too short " + f"for a steady window after trimming {SETTLE + TAIL}s; skipped") + continue + rows.append(stats_record(summarize(prom_url, s), timestamp, tag, test_name)) + if rows: + with open(jsonl_path, "a", encoding="utf-8") as f: + for row in rows: + f.write(json.dumps(row) + "\n") + return len(rows) + + +def _log(logs: TextIO, msg: str) -> None: + print(msg, flush=True) + logs.write(msg + "\n") + logs.flush() diff --git a/benchmarking/locust/runner.py b/benchmarking/locust/runner.py index be7503321e..03bf485e89 100644 --- a/benchmarking/locust/runner.py +++ b/benchmarking/locust/runner.py @@ -54,6 +54,7 @@ from common.boomer_config import build_config_json from server_telemetry import extract_and_record_server_telemetry from phase_breakdown import append_phase_breakdown +from ladder_steps import append_ladder_steps # Path inside the locust image to the boomer-worker binary baked in by # benchmarking/locust/Dockerfile. @@ -658,6 +659,19 @@ def main() -> None: except Exception as e: tee(logs, f"Warning: Failed to harvest server telemetry: {e}") + # Per-step rows for a ladder run, from the same Prometheus. Placed + # after the server telemetry, whose wait lets the last step's samples + # be scraped. + try: + n = append_ladder_steps( + jsonl_path, logs_path, args.prometheus_url, data_ts, + args.tag, args.name, logs, + ) + if n: + tee(logs, f"Appended {n} ladder step rows to {jsonl_path}") + except Exception as e: + tee(logs, f"Warning: Failed to summarize ladder steps: {e}") + # The phase breakdown is additive too. It reads the node logs now, # while the pods still exist: the orchestrator deletes them as soon # as this process exits. The window reaches back to this run's start diff --git a/benchmarking/locust/unit_tests/test_ladder_steps.py b/benchmarking/locust/unit_tests/test_ladder_steps.py new file mode 100644 index 0000000000..89d6c3c9db --- /dev/null +++ b/benchmarking/locust/unit_tests/test_ladder_steps.py @@ -0,0 +1,223 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for ladder_steps.py. + +Run via: python3 benchmarking/locust/unit_tests/test_ladder_steps.py +""" + +import io +import json +import sys +import tempfile +import unittest +import urllib.parse +from pathlib import Path +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import ladder_steps + +LOG = """\ +[locust] [2026-10-12 00:19:07,123] host/INFO/ladder_shape: Ladder step 1/2: users=100 trace_probability=unchanged target_rps=500.0 +[boomer] {"msg":"dynconfig applied"} +[locust] [2026-10-12 00:22:07,456] host/INFO/ladder_shape: Ladder step 2/2: users=100 trace_probability=unchanged target_rps=1000.0 +[locust] [2026-10-12 00:25:07,789] host/INFO/ladder_shape: Ladder complete after 360s +""" +T0 = 1791764347 # 2026-10-12 00:19:07 UTC + + +class StepsTest(unittest.TestCase): + def test_parses_steps_and_their_ends(self): + self.assertEqual(ladder_steps.steps(LOG.splitlines()), [ + {"step": 1, "users": 100, "target_rps": 500.0, "start": T0, "end": T0 + 180}, + {"step": 2, "users": 100, "target_rps": 1000.0, "start": T0 + 180, "end": T0 + 360}, + ]) + + def test_unfinished_ladder_leaves_the_last_end_open(self): + got = ladder_steps.steps(LOG.splitlines()[:3]) + self.assertEqual([s["end"] for s in got], [T0 + 180, None]) + + # A step that changes only the users still bounds its neighbors. + def test_unchanged_rate_keeps_the_previous_one(self): + log = ( + "[locust] [2026-10-12 00:19:07,1] x: Ladder step 1/3: users=100 trace_probability=unchanged target_rps=unchanged\n" + "[locust] [2026-10-12 00:22:07,1] x: Ladder step 2/3: users=100 trace_probability=unchanged target_rps=500.0\n" + "[locust] [2026-10-12 00:25:07,1] x: Ladder step 3/3: users=200 trace_probability=unchanged target_rps=unchanged\n" + "[locust] [2026-10-12 00:28:07,1] x: Ladder complete after 540s\n" + ) + got = ladder_steps.steps(log.splitlines()) + self.assertEqual([(s["step"], s["target_rps"]) for s in got], [(1, None), (2, 500.0), (3, 500.0)]) + self.assertEqual([s["end"] for s in got], [T0 + 180, T0 + 360, T0 + 540]) + + +def prometheus(answers): + """A fake Prometheus API: the first matching answer for each query, by + substring; a query no answer matches gets an empty result.""" + def get_json(url): + q = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)["query"][0] + for needle, result in answers: + if needle in q: + return {"status": "success", "data": {"result": result}} + return {"status": "success", "data": {"result": []}} + return get_json + + +def value(v, **labels): + return [{"metric": labels, "value": [0, str(v)]}] + + +class SummarizeTest(unittest.TestCase): + STEP = {"step": 1, "users": 100, "target_rps": 500.0, "start": T0, "end": T0 + 180} + + def summarize(self, answers): + with mock.patch.object(ladder_steps, "_get_json", prometheus(answers)): + return ladder_steps.summarize("http://prom", self.STEP) + + def test_window_trims_both_ends(self): + got = self.summarize([]) + self.assertEqual(got["window"], [T0 + ladder_steps.SETTLE, T0 + 180 - ladder_steps.TAIL]) + + def test_missing_series_are_none_and_nan_is_none(self): + got = self.summarize([('status="success"', value("NaN"))]) + self.assertIsNone(got["resume_rps"]) + self.assertIsNone(got["postgres_cores"]) + self.assertIsNone(got["pg_cache_hit_ratio"]) + self.assertEqual(got["ateapi_cores_per_replica"], []) + + def test_per_replica_cores_busiest_first(self): + answers = [("sum by (pod)", [ + {"metric": {"pod": "a"}, "value": [0, "290"]}, + {"metric": {"pod": "b"}, "value": [0, "435"]}, + ])] + got = self.summarize(answers) + self.assertEqual(got["ateapi_cores_per_replica"], [435.0, 290.0]) + + def test_atelet_cores_come_from_the_atelet_container(self): + got = self.summarize([('container="atelet"', value(0.25))]) + self.assertEqual(got["atelet_cores"], 0.25) + + def test_cache_hit_ratio(self): + got = self.summarize([("blks_hit", value(990)), ("blks_read", value(10))]) + self.assertAlmostEqual(got["pg_cache_hit_ratio"], 0.99) + self.assertAlmostEqual(got["pg_blks_read_per_s"], 10 / 145) + + def test_walwrite_waiters_come_from_their_own_query(self): + got = self.summarize([('wait_event="WALWrite"', value(42)), ('wait_event_type="Lock"', value(3))]) + self.assertEqual(got["pg_walwrite_wait_conns"], 42.0) + self.assertEqual(got["pg_lock_wait_conns"], 3.0) + + def test_query_trims_the_url_and_times_out(self): + opened = [] + + def urlopen(url, timeout=None): + opened.append((url, timeout)) + return mock.MagicMock(__enter__=lambda self: self) + + with mock.patch.object(ladder_steps.urllib.request, "urlopen", urlopen), \ + mock.patch.object(ladder_steps.json, "load", lambda r: {"status": "success", "data": {"result": []}}): + ladder_steps.query("http://prom/", "up", T0) + url, timeout = opened[0] + self.assertTrue(url.startswith("http://prom/api/v1/query?"), url) + self.assertEqual(timeout, ladder_steps.QUERY_TIMEOUT_S) + + def test_failed_query_raises(self): + with mock.patch.object(ladder_steps, "_get_json", lambda url: {"status": "error"}): + with self.assertRaises(RuntimeError): + ladder_steps.summarize("http://prom", self.STEP) + + +class StatsRecordTest(unittest.TestCase): + def test_flattens_the_summary(self): + summary = {"step": 2, "users": 100, "target_rps": 500.0, "window": [10, 20], + "failures": {"ResumeActorCold": 3.0}, "ateapi_cores_per_replica": [1.5, 0.5], + "resume_rps": 499.0} + self.assertEqual(ladder_steps.stats_record(summary, "ts", "c", "t"), { + "timestamp": "ts", "tag": "c", "test_name": "t", "metric": "ladder_step_002", + "measurements": {"users": 100, "target_rps": 500.0, "resume_rps": 499.0, + "window_start": 10, "window_end": 20, + "ateapi_cores_busiest_replica": 1.5, + "failures_ResumeActorCold": 3.0}, + }) + + def test_no_replicas_reported(self): + summary = {"step": 1, "window": [10, 20], "failures": {}, "ateapi_cores_per_replica": []} + m = ladder_steps.stats_record(summary, "ts", "c", "t")["measurements"] + self.assertIsNone(m["ateapi_cores_busiest_replica"]) + + +class AppendLadderStepsTest(unittest.TestCase): + def append(self, log, get_json=None, existing=""): + with tempfile.TemporaryDirectory() as d: + log_path, jsonl_path = Path(d, "logs.txt"), Path(d, "stats.jsonl") + log_path.write_text(log) + if existing: + jsonl_path.write_text(existing) + logs = io.StringIO() + with mock.patch.object(ladder_steps, "_get_json", get_json or prometheus([])), \ + mock.patch("builtins.print"): + n = ladder_steps.append_ladder_steps( + jsonl_path, log_path, "http://prom", "ts", "c", "t", logs) + rows = jsonl_path.read_text().splitlines() if jsonl_path.exists() else None + return n, rows, logs.getvalue() + + def test_appends_a_row_per_finished_step(self): + n, rows, _ = self.append(LOG, existing='{"metric": "locust"}\n') + self.assertEqual(n, 2) + self.assertEqual(json.loads(rows[0]), {"metric": "locust"}) + self.assertEqual([json.loads(r)["metric"] for r in rows[1:]], + ["ladder_step_001", "ladder_step_002"]) + + def test_skips_a_step_too_short_to_measure(self): + short = LOG.replace("00:22:07,456", "00:19:37,456") # step 1 lasts 30s + n, rows, logged = self.append(short) + self.assertEqual(n, 1) + self.assertEqual(json.loads(rows[0])["metric"], "ladder_step_002") + self.assertIn("too short", logged) + + def test_unfinished_step_is_skipped(self): + n, _, logged = self.append("\n".join(LOG.splitlines()[:3])) + self.assertEqual(n, 1) + self.assertIn("no end", logged) + + # A run without the ladder shape leaves stats.jsonl alone, and creates + # none when the run produced none. + def test_no_ladder_writes_nothing(self): + self.assertEqual(self.append("[locust] no ladder here\n"), (0, None, "")) + + def test_failed_query_writes_no_partial_ladder(self): + calls = [] + + def get_json(url): + calls.append(url) + # Step 1 takes some 35 queries, so this fails within step 2. + if len(calls) > 40: + return {"status": "error"} + return {"status": "success", "data": {"result": []}} + + with tempfile.TemporaryDirectory() as d: + log_path, jsonl_path = Path(d, "logs.txt"), Path(d, "stats.jsonl") + log_path.write_text(LOG) + jsonl_path.write_text('{"metric": "locust"}\n') + with mock.patch.object(ladder_steps, "_get_json", get_json): + with self.assertRaises(RuntimeError): + ladder_steps.append_ladder_steps( + jsonl_path, log_path, "http://prom", "ts", "c", "t", io.StringIO()) + self.assertGreater(len(calls), 40) + self.assertEqual(jsonl_path.read_text(), '{"metric": "locust"}\n') + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarking/locust/unit_tests/test_server_telemetry.py b/benchmarking/locust/unit_tests/test_server_telemetry.py index 47ed4539d4..9e4525f21c 100644 --- a/benchmarking/locust/unit_tests/test_server_telemetry.py +++ b/benchmarking/locust/unit_tests/test_server_telemetry.py @@ -785,6 +785,29 @@ def test_telemetry_survives_a_missing_stats_csv(self, _run, _up, telemetry): self.assertEqual(telemetry.call_args.kwargs["lag_s"], 70) self.assertNotIn("worker_pod_count", telemetry.call_args.kwargs) + # Ladder rows follow the telemetry harvest, whose wait covers their + # last samples, and a failure there still uploads the run. + @mock.patch.object(runner, "upload") + @mock.patch.object(runner, "run_test", return_value=1) + def test_ladder_steps_follow_telemetry_and_may_fail(self, _run, upload): + calls = [] + telemetry = mock.Mock(side_effect=lambda **_: calls.append("telemetry")) + + def ladder(jsonl_path, log_path, prom_url, *_): + calls.append(("ladder", log_path.name, prom_url)) + raise RuntimeError("prometheus down") + + out = io.StringIO() + with mock.patch.object(runner, "extract_and_record_server_telemetry", telemetry), \ + mock.patch.object(runner, "append_ladder_steps", ladder), \ + mock.patch.object(sys, "argv", ARGV + ["--no-cluster-facts", "--allow-empty-stats", + "--prometheus-url", "http://x:9090"]), \ + contextlib.redirect_stdout(out): + runner.main() + self.assertEqual(calls, ["telemetry", ("ladder", "unit-run_logs.txt", "http://x:9090")]) + self.assertIn("Failed to summarize ladder steps: prometheus down", out.getvalue()) + self.assertTrue(upload.called) + if __name__ == "__main__": unittest.main() diff --git a/benchmarking/monitoring.yaml b/benchmarking/monitoring.yaml index 772f85050e..abb9b8b228 100644 --- a/benchmarking/monitoring.yaml +++ b/benchmarking/monitoring.yaml @@ -81,6 +81,30 @@ subjects: name: prometheus namespace: benchmarking --- +# The postgres-exporter job discovers pods in ate-system. A Role there would +# go with the namespace, which every automated test deletes and recreates. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus-benchmarking-pods +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus-benchmarking-pods +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-benchmarking-pods +subjects: +- kind: ServiceAccount + name: prometheus + namespace: benchmarking +--- apiVersion: v1 kind: ConfigMap metadata: @@ -137,7 +161,8 @@ data: target_label: pod # The locust pod annotates port 8000 for the Python locust process. The - # boomer-worker sidecar exposes its Prometheus metrics on port 8001. + # boomer-worker sidecar exposes its Prometheus metrics on port 8001, and + # so does boomer inside an automation runner Job. - job_name: 'boomer-worker' kubernetes_sd_configs: - role: pod @@ -149,7 +174,27 @@ data: - __meta_kubernetes_pod_label_app - __meta_kubernetes_pod_container_port_number action: keep - regex: locust;8001 + regex: (locust|substrate-benchmark-runner);8001 + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod + + # The postgres_exporter sidecar that an automation test with a + # `controlPlane:` block adds to Postgres (benchmarking/automation/ + # controlplane.py): connections by state, commits, cache hits, and + # pg_stat_statements. + - job_name: 'postgres-exporter' + kubernetes_sd_configs: + - role: pod + namespaces: + names: + - ate-system + relabel_configs: + - source_labels: + - __meta_kubernetes_pod_label_app + - __meta_kubernetes_pod_container_port_number + action: keep + regex: postgres;9187 - source_labels: [__meta_kubernetes_pod_name] action: replace target_label: pod diff --git a/internal/benchmarking/boomer/metrics/prom.go b/internal/benchmarking/boomer/metrics/prom.go index 34b811a8f2..d290a3661a 100644 --- a/internal/benchmarking/boomer/metrics/prom.go +++ b/internal/benchmarking/boomer/metrics/prom.go @@ -47,6 +47,15 @@ var ( []string{"method", "name", "status", "user_class"}, ) + serverDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "locust_server_duration_milliseconds", + Help: "Server-side latency in milliseconds, read from the response trailer or header, by method/name/user_class.", + Buckets: prometheus.ExponentialBuckets(0.1, 2, 20), + }, + []string{"method", "name", "user_class"}, + ) + activeUsers = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "locust_users", @@ -57,7 +66,7 @@ var ( ) func init() { - prometheus.MustRegister(requestsTotal, requestDuration, activeUsers) + prometheus.MustRegister(requestsTotal, requestDuration, serverDuration, activeUsers) } // Serve starts a /metrics HTTP server on addr. Returns when the server stops @@ -113,6 +122,12 @@ func RecordPromFailure(method, name, userClass string, latency time.Duration) { requestDuration.WithLabelValues(method, name, "failure", userClass).Observe(ms) } +// RecordServerLatency reports the server's own elapsed time for a request, +// to Prometheus only: locust has one latency per request, the client's. +func RecordServerLatency(method, name, userClass string, latency time.Duration) { + serverDuration.WithLabelValues(method, name, userClass).Observe(float64(latency) / float64(time.Millisecond)) +} + // UpdateUsers shifts the active-users gauge for a class by delta (positive on // user start, negative on stop). func UpdateUsers(userClass string, delta float64) { diff --git a/internal/benchmarking/boomer/metrics/prom_test.go b/internal/benchmarking/boomer/metrics/prom_test.go new file mode 100644 index 0000000000..9ecb884b5e --- /dev/null +++ b/internal/benchmarking/boomer/metrics/prom_test.go @@ -0,0 +1,73 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// The server histogram resolves a fast RPC's sub-millisecond time, +// which a first bucket at 1 ms would hide. +func TestRecordServerLatencyKeepsSubMillisecond(t *testing.T) { + serverDuration.Reset() + RecordServerLatency("grpc", "SubMs", "TestUser", 300*time.Microsecond) + + got := collectLines(t, "locust_server_duration_milliseconds") + for _, line := range []string{ + `locust_server_duration_milliseconds_bucket{method="grpc",name="SubMs",user_class="TestUser",le="0.2"} 0`, + `locust_server_duration_milliseconds_bucket{method="grpc",name="SubMs",user_class="TestUser",le="0.4"} 1`, + `locust_server_duration_milliseconds_sum{method="grpc",name="SubMs",user_class="TestUser"} 0.3`, + } { + if !strings.Contains(got, line) { + t.Errorf("missing line %q in:\n%s", line, got) + } + } +} + +func TestRecordServerLatency(t *testing.T) { + serverDuration.Reset() + RecordServerLatency("grpc", "Srv", "TestUser", 1500*time.Microsecond) + RecordServerLatency("grpc", "Srv", "TestUser", 2500*time.Microsecond) + + got := collectLines(t, "locust_server_duration_milliseconds") + if !strings.Contains(got, `locust_server_duration_milliseconds_sum{method="grpc",name="Srv",user_class="TestUser"} 4`) { + t.Errorf("sum is not 4 ms:\n%s", got) + } + if !strings.Contains(got, `locust_server_duration_milliseconds_count{method="grpc",name="Srv",user_class="TestUser"} 2`) { + t.Errorf("count is not 2:\n%s", got) + } +} + +// collectLines scrapes the default registry and returns the lines of the +// named metric family. +func collectLines(t *testing.T, name string) string { + t.Helper() + rec := httptest.NewRecorder() + promhttp.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + var b strings.Builder + for _, line := range strings.Split(rec.Body.String(), "\n") { + if strings.Contains(line, name) { + b.WriteString(line) + b.WriteString("\n") + } + } + return b.String() +} diff --git a/internal/benchmarking/boomer/resumepaused/resumepaused.go b/internal/benchmarking/boomer/resumepaused/resumepaused.go index e6364311db..4d737c0f0a 100644 --- a/internal/benchmarking/boomer/resumepaused/resumepaused.go +++ b/internal/benchmarking/boomer/resumepaused/resumepaused.go @@ -36,6 +36,7 @@ import ( "time" "github.com/google/uuid" + "go.ate.dev/substrate/internal/ateinterceptors" "go.ate.dev/substrate/internal/benchmarking/boomer/boomerutil" "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" bmetrics "go.ate.dev/substrate/internal/benchmarking/boomer/metrics" @@ -43,7 +44,9 @@ import ( "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel" "golang.org/x/time/rate" + "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" ) @@ -216,13 +219,13 @@ const ( // create creates one actor from the template. func (r *runtime) create(ctx context.Context, name string) error { - return r.call(ctx, "CreateActor", func(ctx context.Context) error { + return r.call(ctx, "CreateActor", func(ctx context.Context, trailer grpc.CallOption) error { _, err := r.cfg.APIStub.CreateActor(ctx, &ateapipb.CreateActorRequest{ Actor: &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: r.cfg.Atespace, Name: name}, ActorTemplate: &ateapipb.ObjectRef{Atespace: templateAtespace, Name: templateName}, }, - }) + }, trailer) return err }) } @@ -262,11 +265,11 @@ func (r *runtime) cycle(name string) { // suspends. An actor already suspended fails it with FailedPrecondition; the // actor is cold, which is all recovery wants, so that counts as a success. func (r *runtime) recover(ctx context.Context, name string) { - _ = r.call(ctx, "SuspendActorAsyncRecovery", func(ctx context.Context) error { + _ = r.call(ctx, "SuspendActorAsyncRecovery", func(ctx context.Context, trailer grpc.CallOption) error { _, err := r.cfg.APIStub.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ Actor: r.ref(name), Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, - }) + }, trailer) if status.Code(err) == codes.FailedPrecondition { return nil } @@ -306,10 +309,10 @@ var errAlreadyRunning = errors.New("actor was already running: not a cold resume // resume calls ResumeActor. A response that reports no resume happened fails // the sample with errAlreadyRunning. func (r *runtime) resume(ctx context.Context, metricName, name string) error { - return r.call(ctx, metricName, func(ctx context.Context) error { + return r.call(ctx, metricName, func(ctx context.Context, trailer grpc.CallOption) error { resp, err := r.cfg.APIStub.ResumeActor(ctx, &ateapipb.ResumeActorRequest{ Actor: r.ref(name), - }) + }, trailer) if err == nil && !resp.GetResumed() { return errAlreadyRunning } @@ -320,11 +323,11 @@ func (r *runtime) resume(ctx context.Context, metricName, name string) error { // pause suspends an actor asynchronously, keeping its snapshot on the node. // suffix keeps warm-up calls apart from the measured ones. func (r *runtime) pause(ctx context.Context, suffix, name string) error { - return r.call(ctx, "SuspendActorAsync"+suffix, func(ctx context.Context) error { + return r.call(ctx, "SuspendActorAsync"+suffix, func(ctx context.Context, trailer grpc.CallOption) error { _, err := r.cfg.APIStub.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ Actor: r.ref(name), Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, - }) + }, trailer) return err }) } @@ -405,10 +408,10 @@ func (r *runtime) delete(ctx context.Context, name string) error { return ctx.Err() } } - err = r.call(ctx, "DeleteActor", func(ctx context.Context) error { + err = r.call(ctx, "DeleteActor", func(ctx context.Context, trailer grpc.CallOption) error { _, err := r.cfg.APIStub.DeleteActor(ctx, &ateapipb.DeleteActorRequest{ Actor: r.ref(name), - }) + }, trailer) return err }) if c := status.Code(err); c != codes.Aborted && c != codes.Unavailable { @@ -423,12 +426,12 @@ func (r *runtime) ref(name string) *ateapipb.ObjectRef { } func (r *runtime) ensureAtespace(ctx context.Context) error { - return r.call(ctx, "CreateAtespace", func(ctx context.Context) error { + return r.call(ctx, "CreateAtespace", func(ctx context.Context, trailer grpc.CallOption) error { _, err := r.cfg.APIStub.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{ Atespace: &ateapipb.Atespace{ Metadata: &ateapipb.ResourceMetadata{Name: r.cfg.Atespace}, }, - }) + }, trailer) if status.Code(err) == codes.AlreadyExists { return nil } @@ -436,17 +439,29 @@ func (r *runtime) ensureAtespace(ctx context.Context) error { }) } -// call times one unary RPC and reports its latency to locust and Prometheus. -func (r *runtime) call(ctx context.Context, name string, do func(context.Context) error) error { +// call times one unary RPC and reports it. The client latency goes to locust +// and Prometheus; for a successful call, the server's own elapsed time, from +// the trailer that ateinterceptors.ServerUnaryInterceptor sets, goes to +// Prometheus as a second histogram, so the gap between the two is network +// and client time. Failures stay out of it: the server histogram has no +// status, and a call cut off by its deadline brings back no trailer. do +// passes trailer to its RPC so the trailer is captured. +func (r *runtime) call(ctx context.Context, name string, do func(ctx context.Context, trailer grpc.CallOption) error) error { ctx, span := r.cfg.Tracer.Start(ctx, name) defer span.End() start := time.Now() - err := do(ctx) + var tr metadata.MD + err := do(ctx, grpc.Trailer(&tr)) latency := time.Since(start) boomerutil.LogSampledTrace(span, name, latency, boomerutil.SourceClient, err) r.record(name, latency, err) + if err == nil { + if serverLatency, source := boomerutil.ElapsedFromMD(tr, ateinterceptors.ServerElapsedTrailer, 0); source == boomerutil.SourceServer { + bmetrics.RecordServerLatency("grpc", name, userClass, serverLatency) + } + } return err } diff --git a/internal/benchmarking/boomer/resumepaused/resumepaused_test.go b/internal/benchmarking/boomer/resumepaused/resumepaused_test.go index aa6eaa57f9..6fd4342719 100644 --- a/internal/benchmarking/boomer/resumepaused/resumepaused_test.go +++ b/internal/benchmarking/boomer/resumepaused/resumepaused_test.go @@ -28,12 +28,14 @@ import ( "time" "github.com/prometheus/client_golang/prometheus/promhttp" + "go.ate.dev/substrate/internal/ateinterceptors" "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" ) @@ -58,6 +60,9 @@ type fakeControlClient struct { // deleteAborts is how many DeleteActor calls fail with Aborted, as when // another operation holds the actor lease, before deletes succeed. deleteAborts int + // serverElapsedUS, when set, is returned in the ServerElapsedTrailer of + // every ResumeActor and SuspendActor. + serverElapsedUS string // resumeEntered and resumeGate, when set, hold every ResumeActor: it // signals resumeEntered and then waits for resumeGate to close or its // context to end. @@ -69,6 +74,19 @@ func newFake() *fakeControlClient { return &fakeControlClient{states: map[string]ateapipb.ActorState{}} } +// setTrailer fills the trailer a call asked for, as ate-api-server's +// ateinterceptors.ServerUnaryInterceptor does. +func (f *fakeControlClient) setTrailer(opts []grpc.CallOption) { + if f.serverElapsedUS == "" { + return + } + for _, o := range opts { + if t, ok := o.(grpc.TrailerCallOption); ok { + *t.TrailerAddr = metadata.Pairs(ateinterceptors.ServerElapsedTrailer, f.serverElapsedUS) + } + } +} + func (f *fakeControlClient) record(rpc, actor string) { f.calls = append(f.calls, rpc+" "+actor) } @@ -130,6 +148,7 @@ func (f *fakeControlClient) ResumeActor(ctx context.Context, in *ateapipb.Resume } f.mu.Lock() defer f.mu.Unlock() + f.setTrailer(opts) name := in.GetActor().GetName() f.record("ResumeActor", name) if f.resumeErr != nil { @@ -153,6 +172,7 @@ func (f *fakeControlClient) ResumeActor(ctx context.Context, in *ateapipb.Resume func (f *fakeControlClient) SuspendActor(ctx context.Context, in *ateapipb.SuspendActorRequest, opts ...grpc.CallOption) (*ateapipb.SuspendActorResponse, error) { f.mu.Lock() defer f.mu.Unlock() + f.setTrailer(opts) name := in.GetActor().GetName() f.record("SuspendActor(async)", name) if in.GetOptions().GetMode() != ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS { @@ -307,6 +327,39 @@ func TestFirstIterationMakesTheUsersActorCold(t *testing.T) { } } +// Each cycle records ate-api-server's own elapsed time beside the client +// latency, so the gap between them is network and client time. +func TestIterateRecordsServerLatency(t *testing.T) { + fake := newFake() + r := newTestRuntime(fake) + r.iterate() + fake.serverElapsedUS = "1500" + resumes, pauses := serverLatencySum(t, "ResumeActorCold"), serverLatencySum(t, "SuspendActorAsync") + r.iterate() + if got := serverLatencySum(t, "ResumeActorCold") - resumes; got != 1.5 { + t.Errorf("ResumeActorCold server latency sum grew by %v ms, want 1.5", got) + } + if got := serverLatencySum(t, "SuspendActorAsync") - pauses; got != 1.5 { + t.Errorf("SuspendActorAsync server latency sum grew by %v ms, want 1.5", got) + } +} + +// A failed call stays out of the server histogram, which has no status: its +// client sample counts as a failure, so the two would describe different +// requests. +func TestIterateLeavesFailuresOutOfServerLatency(t *testing.T) { + fake := newFake() + r := newTestRuntime(fake) + r.iterate() + fake.serverElapsedUS = "1500" + fake.resumeErr = status.Error(codes.Aborted, "lease held") + before := serverLatencySum(t, "ResumeActorCold") + r.iterate() + if got := serverLatencySum(t, "ResumeActorCold") - before; got != 0 { + t.Errorf("ResumeActorCold server latency sum grew by %v ms after a failed resume, want 0", got) + } +} + func TestIterateResumesThenPausesTheUsersActor(t *testing.T) { fake := newFake() r := newTestRuntime(fake) @@ -738,6 +791,11 @@ func requestCount(t *testing.T, name, result string) float64 { return scrape(t, `locust_requests_total{method="grpc",name="`+name+`",status="`+result+`",user_class="ResumePausedUser"} `) } +func serverLatencySum(t *testing.T, name string) float64 { + t.Helper() + return scrape(t, `locust_server_duration_milliseconds_sum{method="grpc",name="`+name+`",user_class="ResumePausedUser"} `) +} + func scrape(t *testing.T, prefix string) float64 { t.Helper() rec := httptest.NewRecorder() From abacb2ac495baa0fffa8230744f9344ebd0e4dbb Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Wed, 21 Oct 2026 00:00:34 +0000 Subject: [PATCH 28/63] Update Version Skew doc / Upgrade doc for SandboxConfig design (#2973) The version skew policy said each release ships its SandboxConfig under its own name and keeps the earlier ones. That is not how it works: it now holds a list of versions with a defaultVersion, and every runtime bump has replaced the list, so the upgrade's `deploy sandboxconfig` removes the old version from the cluster. atelet restores a snapshot on the version it was taken with, so a release that keeps the previous version lets actors suspended on it keep their memory across the upgrade. Write that down as the rule for every sandbox class: a bump adds the new version as the default, keeps the previous one unless it has a known vulnerability, and leaves its assets in place. Step 4d and the step 4 rollback in the runbook now say what suspended actors see, removing the TODOs. Why is this needed? (Or link the issue. Open one first for features, API, install or CI changes.) Fixes #1686 AI use: No, written by human. - [ ] Tests pass - [ ] Docs updated --- docs/upgrade.md | 9 +++++---- docs/version-skew-policy.md | 6 ++++-- manifests/ate-install/sandboxconfig-gvisor.yaml | 4 ++++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index f687e61f52..f440bc5540 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -218,12 +218,11 @@ kubectl -n ate-system rollout status deploy/atenet-egress go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/new.yaml ``` -This applies the SandboxConfig admission policy and the default SandboxConfig. It does not roll workers. - - +This applies the SandboxConfig admission policy and the default SandboxConfig. It does not roll workers. If the release changes a sandbox version, new actors start on the new version, and actors suspended on the previous one still restore on it. - **Done:** the command exits 0. - **Stuck:** the command fails. [Roll back step 4](#roll-back-step-4). +- **Actors:** a suspended actor whose sandbox version the release drops resumes without its memory, keeping its files. #### 4e. The rest of the system bundle @@ -276,7 +275,9 @@ go run ./cmd/ate-setup deploy apiserver --config ~/ate-upgrade/old.yaml go install ./cmd/kubectl-ate ``` -`deploy ate-controller` restores the old CRDs, RBAC and `ate-controller`. No pool rolls. Fields the new release added to WorkerPools, SandboxConfigs or CSIDriverConfigs are dropped from those objects. Remove them from your YAML files too, or `kubectl apply` rejects the files. +`deploy sandboxconfig` removes a sandbox version new in this release, so actors suspended on it resume without their memory. `deploy ate-controller` restores the old CRDs, RBAC and `ate-controller`. No pool rolls. + +Fields that the new release added to WorkerPools, SandboxConfigs or CSIDriverConfigs are dropped from those objects. Remove them from your YAML files too, or `kubectl apply` rejects the files. The database schema stays new; the old API server drops fields it does not know when it rewrites a record. Step 4e is not undone either: the NetworkPolicies and the bundled PostgreSQL stay as the new release left them. diff --git a/docs/version-skew-policy.md b/docs/version-skew-policy.md index da8436baf0..68451ab490 100644 --- a/docs/version-skew-policy.md +++ b/docs/version-skew-policy.md @@ -79,9 +79,11 @@ Between compatible releases, a CRD change only adds optional fields. The CRDs se ### SandboxConfig -Each release ships its SandboxConfig under its own name and keeps the earlier ones. The upgrade adds the N+1 one, and WorkerPools without a `configRef` switch to it. That does not change the pod template, so workers do not roll. +Each sandbox class has a default SandboxConfig, such as `gvisor-default`, that lists the sandbox versions a cluster may run. A release that changes a sandbox version adds the new one, points `defaultVersion` at it, and keeps the previous release's version unless that version has a known vulnerability. -An actor resolves its pool's SandboxConfig each time it starts. If the runtime its snapshot needs is gone, it cold boots, keeping its files and losing its memory. Rollback deletes the N+1 SandboxConfig. +A kept version's assets must stay fetchable, so a release does not overwrite them. The upgrade applies these lists. That does not change the pod template, so workers do not roll. + +An actor that starts without memory state, new or cold booting, gets the default version. An actor restoring memory state gets the version its snapshot was taken with, and stays on it until it next starts without memory state. If that version is gone, it cold boots, keeping its files and losing its memory. Rollback applies the N lists again, which removes the N+1 versions. diff --git a/manifests/ate-install/sandboxconfig-gvisor.yaml b/manifests/ate-install/sandboxconfig-gvisor.yaml index ab8fd3742c..908877e29c 100644 --- a/manifests/ate-install/sandboxconfig-gvisor.yaml +++ b/manifests/ate-install/sandboxconfig-gvisor.yaml @@ -19,6 +19,10 @@ # it) matching the worker node's architecture and extracts it locally. To pin a # different release, add a version below and point defaultVersion at it, or # create another SandboxConfig and reference it from the WorkerPool. +# +# A bump adds a version and points defaultVersion at it. Keep the +# previous version unless it has a known vulnerability: an actor restores its +# memory only on the version its snapshot was taken with. apiVersion: ate.dev/v1alpha1 kind: SandboxConfig metadata: From 196d3175caa25e3f79531c78313479aa65379698 Mon Sep 17 00:00:00 2001 From: shrutiyam-glitch Date: Wed, 21 Oct 2026 00:16:30 +0000 Subject: [PATCH 29/63] ateapi: Remove sandbox_class from Worker spec (#2922) Follow up after #2879 Removes `sandbox_class` from the `Worker` spec (`ateapipb.Worker`) and reads the worker's sandbox class from `Worker.status.default_runtime.sandbox_class`. Each worker already reports its `default_runtime` (including `sandbox_class`) to `WorkerService.RegisterWorker` via `atelet` alongside its capacity before it can take any actors. Carrying `sandbox_class` on the `Worker` spec required `ate-controller` to copy it from the `WorkerPool` on `CreateWorker` and duplicated what the worker itself reports in `WorkerStatus`. Using `status.default_runtime.sandbox_class` as the single source of truth removes that duplication across `ate-controller`, `ate-api-server`, `fake-workersync`, and `kubectl-ate`. #### Behavior changes - **Class is fixed by the first registration.** `RegisterWorker` records the class from the first report and refuses a later report with a different `default_runtime.sandbox_class` (restorable runtimes must match it too). The pool's declared class no longer cross-checks the worker's report: the pod's shape comes from the pool, and the report is the pod's own authenticated statement. The proto comments on `WorkerStatus.default_runtime` and `RegisterWorkerRequest.default_runtime` document this. - **Before a worker registers it has no class.** `kubectl ate get/top workers` show an empty CLASS for it and `--sandbox-class` filters it out;`ate.workerpool.workers` counts it under `ate.sandbox.class=unknown` (registry note updated). The scheduler never placed on such a worker, and still doesn't:`Applies` rejects an empty class, which also covers the "no default runtime" guard #2879 added, so that guard is folded into the class check. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- cmd/ateapi/internal/apivalidation/worker.go | 4 - .../internal/apivalidation/worker_test.go | 12 --- .../apivalidation/zz_generated.validation.go | 39 -------- cmd/ateapi/internal/controlapi/crash.go | 23 +++-- cmd/ateapi/internal/controlapi/crash_test.go | 2 +- .../controlapi/functionaltest/actor_test.go | 1 - .../controlapi/functionaltest/common_test.go | 1 - .../functionaltest/worker_contention_test.go | 2 +- .../controlapi/functionaltest/worker_test.go | 2 - .../internal/controlapi/hosting_test.go | 9 +- cmd/ateapi/internal/controlapi/metrics.go | 10 +- .../internal/controlapi/metrics_test.go | 17 ++-- .../controlapi/mint_after_release_test.go | 7 +- cmd/ateapi/internal/controlapi/worker_test.go | 10 +- .../internal/controlapi/workflow_delete.go | 2 +- .../workflow_reconcile_assignments_test.go | 1 - .../internal/controlapi/workflow_resume.go | 2 +- .../controlapi/workflow_resume_test.go | 23 +---- .../internal/controlapi/workflow_revert.go | 2 +- .../internal/controlapi/workflow_suspend.go | 2 +- .../controlapi/workflow_worker_delete.go | 3 +- .../controlapi/workflow_worker_delete_test.go | 1 + cmd/ateapi/internal/scheduling/scheduling.go | 18 ++-- .../internal/scheduling/scheduling_test.go | 7 +- cmd/ateapi/internal/workerservice/register.go | 25 +++-- .../internal/workerservice/register_test.go | 95 +++++++++---------- .../internal/workerservice/service_test.go | 10 +- .../internal/workersync/syncer.go | 18 +--- .../workersync/syncer_restart_test.go | 11 +-- .../internal/workersync/syncer_test.go | 95 ++++++------------- .../isolate/fake-workersync/controller.go | 51 +++++----- .../fake-workersync/controller_test.go | 23 ++--- cmd/kubectl-ate/internal/cmd/worker.go | 4 +- cmd/kubectl-ate/internal/cmd/worker_test.go | 48 ++++++---- .../internal/printer/printer_test.go | 1 - docs/metrics/registry/metrics.yaml | 11 +-- internal/ateattr/ateattr_test.go | 4 +- pkg/proto/ateapipb/ateapi.pb.go | 40 +++----- pkg/proto/ateapipb/ateapi.proto | 21 ++-- 39 files changed, 258 insertions(+), 399 deletions(-) diff --git a/cmd/ateapi/internal/apivalidation/worker.go b/cmd/ateapi/internal/apivalidation/worker.go index 6f53f82eb5..6570748c27 100644 --- a/cmd/ateapi/internal/apivalidation/worker.go +++ b/cmd/ateapi/internal/apivalidation/worker.go @@ -126,10 +126,6 @@ func ValidateWorkerUpdate(ctx context.Context, fldPath *field.Path, newVal, oldV return errs } -func ValidateCustom_Worker_SandboxClass(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { - return resources.ValidateSandboxClass(fldPath, *value) -} - func ValidateCustom_SandboxRuntime_SandboxClass(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { return resources.ValidateSandboxClass(fldPath, *value) } diff --git a/cmd/ateapi/internal/apivalidation/worker_test.go b/cmd/ateapi/internal/apivalidation/worker_test.go index a0b60ea8ea..9136e5d307 100644 --- a/cmd/ateapi/internal/apivalidation/worker_test.go +++ b/cmd/ateapi/internal/apivalidation/worker_test.go @@ -159,17 +159,6 @@ func TestValidateCreateWorkerRequest(t *testing.T) { w.WorkerSpiffeId = "spiffe://cluster.local/" + strings.Repeat("a", 2048) })), want: field.ErrorList{field.TooLong(field.NewPath("worker", "worker_spiffe_id"), nil, 2048).WithOrigin("maxLength")}, - }, { - name: "sandbox_class too long", - req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { w.SandboxClass = strings.Repeat("x", 64) })), - want: field.ErrorList{ - field.TooLong(field.NewPath("worker", "sandbox_class"), nil, 63).WithOrigin("maxLength"), - field.NotSupported[string](field.NewPath("worker", "sandbox_class"), nil, nil), - }, - }, { - name: "unsupported sandbox_class", - req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { w.SandboxClass = "kata" })), - want: field.ErrorList{field.NotSupported[string](field.NewPath("worker", "sandbox_class"), nil, nil)}, }, { name: "valid labels", req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { @@ -802,7 +791,6 @@ func validWorker(name string, mods ...func(*ateapipb.Worker)) *ateapipb.Worker { WorkerPodUid: name, NodeName: "node-1", Ips: []string{"10.1.2.3"}, - SandboxClass: "gvisor", } for _, m := range mods { m(w) diff --git a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go index e10bfb8a0f..8b32b13940 100644 --- a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go +++ b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go @@ -10096,45 +10096,6 @@ func Validate_Worker( errs = append(errs, fn(fldPath.Child("worker_spiffe_id"), &obj.WorkerSpiffeId, oldVal, oldObj != nil)...) } - { // field ateapipb.Worker.SandboxClass - fn := func( - fldPath *field.Path, - obj, oldObj *string, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.Immutable(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - // custom validation - if e := ValidateCustom_Worker_SandboxClass(ctx, op, fldPath, obj, oldObj); len(e) != 0 { - errs = append(errs, e...) - } - if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 63); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.Worker) *string { - return &oldObj.SandboxClass - }) - errs = append(errs, fn(fldPath.Child("sandbox_class"), &obj.SandboxClass, oldVal, oldObj != nil)...) - } - { // field ateapipb.Worker.Labels fn := func( fldPath *field.Path, diff --git a/cmd/ateapi/internal/controlapi/crash.go b/cmd/ateapi/internal/controlapi/crash.go index 9aeee6aea0..45a8682708 100644 --- a/cmd/ateapi/internal/controlapi/crash.go +++ b/cmd/ateapi/internal/controlapi/crash.go @@ -139,13 +139,13 @@ func crashActor(ctx context.Context, st crashActorStore, actorRef resources.Acto // instead leaves the actor (and its assignment) intact so the caller retries // crashActor, which re-attempts the release. releaseWorker is idempotent, so // a retry after a release that already succeeded is a no-op. - sandboxClass, _, err := releaseWorker(ctx, st, actor) - if err != nil { + if _, err := releaseWorker(ctx, st, actor); err != nil { return fmt.Errorf("while releasing worker to crash actor: %w", err) } // Snapshot crash attributes before pod and pool pointers are cleared below; // the counter itself is emitted only after the transition commits. + sandboxClass := actor.GetStatus().GetWorkerAssignment().GetSandboxRuntime().GetSandboxClass() crashAttrs := ateattr.ActorMetricAttributes(actor, sandboxClass, opName) _, err = st.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { @@ -212,37 +212,36 @@ type crashActorStore interface { // releaseWorker clears the worker's assignment if it still points at the given // actor. A missing worker or an already-cleared assignment is not an error. -// It returns the worker's sandboxClass if found, and the worker as it stands -// after the release, which callers holding a cache of workers hand to it. -func releaseWorker(ctx context.Context, st crashActorStore, actor *ateapipb.Actor) (string, *ateapipb.Worker, error) { +// It returns the worker as it stands after the release, which callers holding +// a cache of workers hand to it. +func releaseWorker(ctx context.Context, st crashActorStore, actor *ateapipb.Actor) (*ateapipb.Worker, error) { assignment := actor.GetStatus().GetWorkerAssignment() if assignment == nil { slog.WarnContext(ctx, "Actor's worker assignment is already cleared") - return "", nil, nil + return nil, nil } workerName := assignment.GetWorker().GetName() - worker, err := st.GetWorker(ctx, workerName) + _, err := st.GetWorker(ctx, workerName) if errors.Is(err, store.ErrNotFound) { // No need to release if the worker is not found. slog.WarnContext(ctx, "Worker already gone while crashing actor, skipping release", slog.String("worker", workerName)) - return "", nil, nil + return nil, nil } if err != nil { - return "", nil, fmt.Errorf("while getting worker to release: %w", err) + return nil, fmt.Errorf("while getting worker to release: %w", err) } - sandboxClass := worker.GetSandboxClass() // Release only this actor's assignment; the worker may be hosting others, // and they are unaffected by this one crashing. A worker that is no longer // hosting it has already been released. released, err := st.ReleaseActorFromWorker(ctx, workerName, actor.GetMetadata().GetUid()) if err != nil { - return sandboxClass, nil, fmt.Errorf("while releasing worker: %w", err) + return nil, fmt.Errorf("while releasing worker: %w", err) } if released == nil { slog.WarnContext(ctx, "Worker is not hosting this Actor, skipping release", slog.String("worker", workerName)) } - return sandboxClass, released, nil + return released, nil } diff --git a/cmd/ateapi/internal/controlapi/crash_test.go b/cmd/ateapi/internal/controlapi/crash_test.go index 8e39921a7e..41defff235 100644 --- a/cmd/ateapi/internal/controlapi/crash_test.go +++ b/cmd/ateapi/internal/controlapi/crash_test.go @@ -567,7 +567,6 @@ func TestCrashActor_Metrics(t *testing.T) { WorkerPool: "pool-1", WorkerPod: "pod-1", WorkerPodUid: "pod-uid-1", - SandboxClass: "gvisor", Status: &ateapipb.WorkerStatus{}, } if _, err := st.CreateWorker(ctx, worker); err != nil { @@ -589,6 +588,7 @@ func TestCrashActor_Metrics(t *testing.T) { WorkerPool: "pool-1", WorkerPod: "pod-1", WorkerPodUid: "pod-uid-1", + SandboxRuntime: &ateapipb.SandboxRuntime{SandboxClass: "gvisor"}, }, }, } diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index a8b013e724..a458a616c3 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -3247,7 +3247,6 @@ func TestResumeActor(t *testing.T) { WorkerPodUid: podUID, Ips: []string{"127.0.0.1"}, NodeName: "node1", - SandboxClass: "gvisor", Labels: map[string]string{poolLabelKey: ns}, Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index faa27b108b..1c00dcc2a3 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -693,7 +693,6 @@ func createWorkerPod(t *testing.T, tc *testContext, ns string, name string, node WorkerPodUid: string(createdPod.UID), Ips: []string{"127.0.0.1"}, NodeName: nodeName, - SandboxClass: string(pool.Spec.DefaultSandboxClass()), Labels: pool.GetLabels(), // Capacity is not settable here: a Worker gets it from its own // worker's report, which the reportWorkerCapacity below stands in diff --git a/cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go b/cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go index 9ac8e83c88..0d62b5256f 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go @@ -188,7 +188,7 @@ func reportWorkerCapacity(t *testing.T, tc *testContext, name string, actors int CpuMillicores: int64(actors) * 1000, MemoryBytes: int64(actors) * 512 << 20, } if toUpdate.Status.DefaultRuntime == nil { - toUpdate.Status.DefaultRuntime = defaultWorkerRuntime(toUpdate.GetSandboxClass()) + toUpdate.Status.DefaultRuntime = defaultWorkerRuntime("gvisor") } return nil }); err != nil { diff --git a/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go b/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go index 87573a4ea7..1ed2caa9c5 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go @@ -49,7 +49,6 @@ func newTestWorker(ns string) *ateapipb.Worker { WorkerPodUid: testWorkerName, NodeName: "node1", Ips: []string{"10.1.2.3"}, - SandboxClass: "gvisor", } } @@ -164,7 +163,6 @@ func TestListWorkers(t *testing.T) { WorkerPodUid: podUID, NodeName: "node1", Ips: []string{"127.0.0.1"}, - SandboxClass: "gvisor", Labels: map[string]string{"foo": "bar"}, Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, diff --git a/cmd/ateapi/internal/controlapi/hosting_test.go b/cmd/ateapi/internal/controlapi/hosting_test.go index 54449ad009..43b61303c4 100644 --- a/cmd/ateapi/internal/controlapi/hosting_test.go +++ b/cmd/ateapi/internal/controlapi/hosting_test.go @@ -59,8 +59,11 @@ func TestHostingNodeAfterWorkerRelease(t *testing.T) { WorkerPod: wa.GetWorkerPod(), WorkerPodUid: wa.GetWorkerPodUid(), NodeName: node, - SandboxClass: "gvisor", - Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}}, + Status: &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + DefaultRuntime: &ateapipb.SandboxRuntime{SandboxClass: "gvisor"}, + }, }); err != nil { t.Fatalf("CreateWorker: %v", err) } @@ -165,7 +168,6 @@ func TestHostingNodeAfterResumeOntoNewWorker(t *testing.T) { WorkerPod: "stale-pod", WorkerPodUid: testWorkerUID("stale-pod"), NodeName: "node1", - SandboxClass: "microvm", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, @@ -179,7 +181,6 @@ func TestHostingNodeAfterResumeOntoNewWorker(t *testing.T) { WorkerPod: "free-pod", WorkerPodUid: testWorkerUID("free-pod"), NodeName: "node2", - SandboxClass: "gvisor", Labels: map[string]string{"pool": "b"}, Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, diff --git a/cmd/ateapi/internal/controlapi/metrics.go b/cmd/ateapi/internal/controlapi/metrics.go index f523743627..23afb92bc5 100644 --- a/cmd/ateapi/internal/controlapi/metrics.go +++ b/cmd/ateapi/internal/controlapi/metrics.go @@ -98,11 +98,11 @@ func RegisterWorkerCount(meter metric.Meter, workers func() ([]*ateapipb.Worker, } for _, w := range ws { state := workerState(w) - // CreateWorker accepts an empty class, as the field is optional. - // Report it as unknown, not as the pool's class: the scheduler puts - // no actor on a worker that offers no class, thus the pool's series - // must not count it as free capacity. - class := ateattr.NormalizeSandboxClass(w.GetSandboxClass()) + // A Worker has no default_runtime until its worker registers with + // WorkerService. Report it as unknown, not as the pool's class: the + // scheduler puts no actor on a worker that offers no class, thus + // the pool's series must not count it as free capacity. + class := ateattr.NormalizeSandboxClass(w.GetStatus().GetDefaultRuntime().GetSandboxClass()) tally[key{w.GetWorkerNamespace(), w.GetWorkerPool(), state, class}]++ } for k, n := range tally { diff --git a/cmd/ateapi/internal/controlapi/metrics_test.go b/cmd/ateapi/internal/controlapi/metrics_test.go index a2f271b574..d88bdef9a8 100644 --- a/cmd/ateapi/internal/controlapi/metrics_test.go +++ b/cmd/ateapi/internal/controlapi/metrics_test.go @@ -76,16 +76,19 @@ func mustMetric(t *testing.T, reader *sdkmetric.ManualReader, name string) metri // worker builds an ACTIVE worker holding allocated of capacity actor slots. func worker(namespace, pool, class string, allocated, capacity int32) *ateapipb.Worker { - return &ateapipb.Worker{ + w := &ateapipb.Worker{ WorkerNamespace: namespace, WorkerPool: pool, - SandboxClass: class, Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: capacity}, Allocated: &ateapipb.WorkerResources{Actors: allocated}, }, } + if class != "" { + w.Status.DefaultRuntime = &ateapipb.SandboxRuntime{SandboxClass: class} + } + return w } type series struct{ namespace, pool, state, class string } @@ -445,11 +448,11 @@ func TestWorkerCountSeedsZeroForKnownPools(t *testing.T) { } } -// TestWorkerCountEmptyClassWorker covers the workers that CreateWorker accepted -// with no sandbox class. They report as unknown, not as an empty-string class, -// and not as capacity of the pool they name: the scheduler puts no actor on -// them, thus the pool's own series stays at 0 and an idle==0 alert on it still -// fires. +// TestWorkerCountEmptyClassWorker covers the workers that have not yet +// reported default_runtime via RegisterWorker. They report as unknown, not as +// an empty-string class, and not as capacity of the pool they name: the +// scheduler puts no actor on them, thus the pool's own series stays at 0 and +// an idle==0 alert on it still fires. func TestWorkerCountEmptyClassWorker(t *testing.T) { const ( gvisor = string(atev1alpha1.SandboxClassGvisor) diff --git a/cmd/ateapi/internal/controlapi/mint_after_release_test.go b/cmd/ateapi/internal/controlapi/mint_after_release_test.go index 6ca42106cd..95b1cfbe87 100644 --- a/cmd/ateapi/internal/controlapi/mint_after_release_test.go +++ b/cmd/ateapi/internal/controlapi/mint_after_release_test.go @@ -70,8 +70,11 @@ func TestMintWorkerActorCertificateAfterWorkerRelease(t *testing.T) { WorkerPod: w.pod, WorkerPodUid: w.podUID, NodeName: node, - SandboxClass: "gvisor", - Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}}, + Status: &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + DefaultRuntime: &ateapipb.SandboxRuntime{SandboxClass: "gvisor"}, + }, }); err != nil { t.Fatalf("CreateWorker(%s): %v", w.name, err) } diff --git a/cmd/ateapi/internal/controlapi/worker_test.go b/cmd/ateapi/internal/controlapi/worker_test.go index 8ec1a6dcd3..3a0a48f871 100644 --- a/cmd/ateapi/internal/controlapi/worker_test.go +++ b/cmd/ateapi/internal/controlapi/worker_test.go @@ -47,7 +47,6 @@ func validWorker(name string, mods ...func(*ateapipb.Worker)) *ateapipb.Worker { WorkerPodUid: name, NodeName: "node-1", Ips: []string{"10.1.2.3"}, - SandboxClass: "gvisor", } for _, m := range mods { m(w) @@ -83,7 +82,12 @@ func seedAPIWorker(t *testing.T, ctx context.Context, persistence store.Interfac t.Helper() worker = proto.Clone(worker).(*ateapipb.Worker) if worker.GetStatus() == nil { - worker.Status = &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE} + // A seeded Worker stands in for one that registered: ACTIVE, with the + // gvisor default runtime its class is read from. + worker.Status = &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + DefaultRuntime: testSandboxRuntime(), + } } created, err := persistence.CreateWorker(ctx, worker) if err != nil { @@ -460,7 +464,6 @@ func TestUpdateWorker_Errors(t *testing.T) { {"ips family added", func(w *ateapipb.Worker) { w.Ips = append(w.Ips, "fd00::1") }, codes.InvalidArgument}, {"worker_pod changed", func(w *ateapipb.Worker) { w.WorkerPod = "worker-pod-2" }, codes.InvalidArgument}, {"node_name changed", func(w *ateapipb.Worker) { w.NodeName = "node-2" }, codes.InvalidArgument}, - {"sandbox_class changed", func(w *ateapipb.Worker) { w.SandboxClass = "microvm" }, codes.InvalidArgument}, // capacity is deliberately absent here: it may change (the pool's actor // ceiling moves, a pod can be resized, a Worker may report its own). // TestUpdateWorker_CapacityChanges covers that. @@ -468,7 +471,6 @@ func TestUpdateWorker_Errors(t *testing.T) { // And immutable fields dropped, which a replacement update reads as a // request to clear them. Rejected rather than silently applied. {"ips omitted", func(w *ateapipb.Worker) { w.Ips = nil }, codes.InvalidArgument}, - {"sandbox_class omitted", func(w *ateapipb.Worker) { w.SandboxClass = "" }, codes.InvalidArgument}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { diff --git a/cmd/ateapi/internal/controlapi/workflow_delete.go b/cmd/ateapi/internal/controlapi/workflow_delete.go index 7529bd8853..96d2f28ec3 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete.go @@ -302,7 +302,7 @@ func (w *ActorWorkflow) ensureWorkerReleased(ctx context.Context, actorRef resou } if latestActor.GetStatus().GetWorkerAssignment() != nil { - _, _, err := releaseWorker(ctx, w.store, latestActor) + _, err := releaseWorker(ctx, w.store, latestActor) if err != nil { return nil, err } diff --git a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go index 00e1a61149..5e77a2a434 100644 --- a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go @@ -575,7 +575,6 @@ func TestAssignWorkerAttempt_StampsWorkerEpoch(t *testing.T) { WorkerPod: "pod-1", WorkerPodUid: name, NodeName: "node-1", - SandboxClass: "gvisor", Epoch: 1, Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index f667e0717e..58d805d2ab 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -612,7 +612,7 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour } poolNamespace = bound.GetWorkerNamespace() pool = bound.GetWorkerPool() - class = bound.GetSandboxClass() + class = bound.GetStatus().GetDefaultRuntime().GetSandboxClass() outcome = ateattr.SchedulerOutcomeAssigned logActorStateChanged(ctx, storedActor, ateattr.OperationResume) return storedActor, bound, nil diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index 2122bb0b07..b16af3144a 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -206,7 +206,6 @@ func TestAssignWorkerAttempt_SkipsWorkerAssignedInOtherAtespace(t *testing.T) { WorkerPool: "pool", WorkerPod: "pod-1", WorkerPodUid: testWorkerUID("pod-1"), - SandboxClass: "gvisor", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, @@ -269,7 +268,6 @@ func TestAssignWorkerAttempt_ReleasesIneligibleStaleWorker(t *testing.T) { WorkerPod: "stale-pod", WorkerPodUid: testWorkerUID("stale-pod"), NodeName: "node-1", - SandboxClass: "gvisor", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, @@ -283,7 +281,6 @@ func TestAssignWorkerAttempt_ReleasesIneligibleStaleWorker(t *testing.T) { WorkerPod: "free-pod", WorkerPodUid: testWorkerUID("free-pod"), NodeName: "node-1", - SandboxClass: "gvisor", Labels: map[string]string{"pool": "b"}, Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, @@ -345,7 +342,6 @@ func TestAssignWorkerAttempt_RetryAfterConflictPicksFreshWorker(t *testing.T) { WorkerPod: "contested-pod", WorkerPodUid: testWorkerUID("contested-pod"), NodeName: "node-1", - SandboxClass: "gvisor", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, @@ -359,7 +355,6 @@ func TestAssignWorkerAttempt_RetryAfterConflictPicksFreshWorker(t *testing.T) { WorkerPod: "fallback-pod", WorkerPodUid: testWorkerUID("fallback-pod"), NodeName: "node-1", - SandboxClass: "gvisor", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, @@ -455,7 +450,6 @@ func seedAssignFixture(t *testing.T, ctx context.Context, persistence store.Inte WorkerPod: "pod-1", WorkerPodUid: testWorkerUID("pod-1"), NodeName: "node-1", - SandboxClass: "gvisor", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, @@ -770,15 +764,14 @@ func TestValidateAssignedWorker(t *testing.T) { Capacity: &ateapipb.WorkerResources{Actors: 1}, DefaultRuntime: testSandboxRuntime(), } - drainingStatus := &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_DRAINING, Capacity: &ateapipb.WorkerResources{Actors: 1}} - unavailableStatus := &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE, Capacity: &ateapipb.WorkerResources{Actors: 1}} + drainingStatus := &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_DRAINING, Capacity: &ateapipb.WorkerResources{Actors: 1}, DefaultRuntime: testSandboxRuntime()} + unavailableStatus := &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE, Capacity: &ateapipb.WorkerResources{Actors: 1}, DefaultRuntime: testSandboxRuntime()} tests := []struct { name string // workerStatus is the stored worker's status, nil for a worker that // is gone. workerStatus *ateapipb.WorkerStatus - sandboxClass string tmplSelector map[string]string assignment *ateapipb.ActorAssignment // wantCode is codes.OK when validateAssignedWorker must return nil. @@ -801,7 +794,6 @@ func TestValidateAssignedWorker(t *testing.T) { { name: "crashes actor and leaves worker untouched when worker is draining", workerStatus: drainingStatus, - sandboxClass: "gvisor", assignment: ownAssignment, wantCode: codes.Aborted, wantActorState: ateapipb.ActorState_ACTOR_STATE_CRASHED, @@ -811,7 +803,6 @@ func TestValidateAssignedWorker(t *testing.T) { { name: "leaves actor and worker untouched when worker is unavailable", workerStatus: unavailableStatus, - sandboxClass: "gvisor", assignment: ownAssignment, wantCode: codes.Unavailable, wantActorState: ateapipb.ActorState_ACTOR_STATE_RESUMING, @@ -820,7 +811,6 @@ func TestValidateAssignedWorker(t *testing.T) { { name: "crashes actor and leaves worker untouched when assigned to another actor", workerStatus: activeStatus, - sandboxClass: "gvisor", assignment: otherAssignment, wantCode: codes.Aborted, wantActorState: ateapipb.ActorState_ACTOR_STATE_CRASHED, @@ -830,7 +820,6 @@ func TestValidateAssignedWorker(t *testing.T) { { name: "crashes actor and leaves worker untouched when assigned to previous incarnation of same actor", workerStatus: activeStatus, - sandboxClass: "gvisor", assignment: staleIncarnationAssignment, wantCode: codes.Aborted, wantActorState: ateapipb.ActorState_ACTOR_STATE_CRASHED, @@ -840,7 +829,6 @@ func TestValidateAssignedWorker(t *testing.T) { { name: "crashes actor and leaves worker untouched when assignment is cleared", workerStatus: activeStatus, - sandboxClass: "gvisor", assignment: nil, wantCode: codes.Aborted, wantActorState: ateapipb.ActorState_ACTOR_STATE_CRASHED, @@ -850,7 +838,6 @@ func TestValidateAssignedWorker(t *testing.T) { { name: "passes for own eligible worker", workerStatus: activeStatus, - sandboxClass: "gvisor", assignment: ownAssignment, wantCode: codes.OK, wantActorState: ateapipb.ActorState_ACTOR_STATE_RESUMING, @@ -859,7 +846,6 @@ func TestValidateAssignedWorker(t *testing.T) { { name: "releases own ineligible worker and crashes actor", workerStatus: activeStatus, - sandboxClass: "gvisor", tmplSelector: map[string]string{"pool": "other"}, assignment: ownAssignment, wantCode: codes.Aborted, @@ -883,7 +869,6 @@ func TestValidateAssignedWorker(t *testing.T) { WorkerPool: "pool", WorkerPod: "pod-1", WorkerPodUid: testWorkerUID("pod-1"), - SandboxClass: tt.sandboxClass, Status: tt.workerStatus, }); err != nil { t.Fatalf("CreateWorker: %v", err) @@ -2199,7 +2184,6 @@ func contendedWorker(podName string) *ateapipb.Worker { WorkerPod: podName, WorkerPodUid: testWorkerUID(podName), NodeName: "node-1", - SandboxClass: "gvisor", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, @@ -2505,7 +2489,6 @@ func seedRuntimeWorker(t *testing.T, ctx context.Context, st store.Interface, po WorkerPool: "pool", WorkerPod: pod, WorkerPodUid: testWorkerUID(pod), - SandboxClass: "gvisor", NodeName: node, Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, @@ -2539,7 +2522,6 @@ func seedMicroVMWorker(t *testing.T, ctx context.Context, st store.Interface, po WorkerPool: "vm-pool", WorkerPod: pod, WorkerPodUid: testWorkerUID(pod), - SandboxClass: "microvm", NodeName: node, Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, @@ -3004,7 +2986,6 @@ func seedVolumeAssignFixture(t *testing.T, ctx context.Context, persistence stor WorkerPod: "pod-1", WorkerPodUid: testWorkerUID("pod-1"), NodeName: node, - SandboxClass: "gvisor", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}, diff --git a/cmd/ateapi/internal/controlapi/workflow_revert.go b/cmd/ateapi/internal/controlapi/workflow_revert.go index 2d6f447797..e851e3328d 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert.go @@ -202,7 +202,7 @@ func (w *ActorWorkflow) ensureWorkerDiscarded(ctx context.Context, actorRef reso } } if actor.GetStatus().GetWorkerAssignment() != nil { - if _, _, err := releaseWorker(ctx, w.store, actor); err != nil { + if _, err := releaseWorker(ctx, w.store, actor); err != nil { return fmt.Errorf("while releasing worker: %w", err) } } diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend.go b/cmd/ateapi/internal/controlapi/workflow_suspend.go index 52d42c15eb..0b6a25f7bd 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend.go @@ -381,7 +381,7 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r // 1. Free the worker (if it hasn't been freed yet) if latestActor.GetStatus().GetWorkerAssignment() != nil { t = time.Now() - _, _, err := releaseWorker(ctx, w.store, latestActor) + _, err := releaseWorker(ctx, w.store, latestActor) dReleaseWorker = time.Since(t) if err != nil { return nil, err diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go index b26f9698b2..41f3d761d0 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go @@ -225,7 +225,8 @@ func (w *WorkerWorkflow) crashBoundActor(ctx context.Context, worker *ateapipb.W wasAlreadyCrashed := actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_CRASHED // Snapshot crash attributes before pod and pool pointers are cleared on actor. - crashAttrs := ateattr.ActorMetricAttributes(actor, worker.GetSandboxClass(), opName) + sandboxClass := actor.GetStatus().GetWorkerAssignment().GetSandboxRuntime().GetSandboxClass() + crashAttrs := ateattr.ActorMetricAttributes(actor, sandboxClass, opName) slog.LogAttrs(ctx, slog.LevelInfo, logMsg, append(ateattr.ActorLogAttrs(resources.ActorAttributionFromActor(actor)), diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go index efeaaa3bfa..f8580b2e50 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go @@ -61,6 +61,7 @@ func seedAPIActor(t *testing.T, ctx context.Context, persistence store.Interface WorkerPod: "worker-pod-1", WorkerPodUid: apiWorkerName, WorkerPodIps: []string{"10.1.2.3"}, + SandboxRuntime: &ateapipb.SandboxRuntime{SandboxClass: "gvisor"}, }, }, } diff --git a/cmd/ateapi/internal/scheduling/scheduling.go b/cmd/ateapi/internal/scheduling/scheduling.go index f7f4f91dd3..e48596d814 100644 --- a/cmd/ateapi/internal/scheduling/scheduling.go +++ b/cmd/ateapi/internal/scheduling/scheduling.go @@ -213,10 +213,16 @@ func utilization(capacity, used *ateapipb.WorkerResources) float64 { } func (s *scheduler) Applies(worker *ateapipb.Worker, constraints Constraints) bool { - if worker.GetSandboxClass() == "" { + // The class is the one on the default runtime the worker registered with + // WorkerService. Until that report lands the Worker has no runtime and so + // no class, and nothing is placed on it (see ServiceImpl.CreateWorker): + // the assignment records the runtime, and atelet requires it on Run and + // Restore. + workerClass := worker.GetStatus().GetDefaultRuntime().GetSandboxClass() + if workerClass == "" { return false } - if constraints.SandboxClass != "" && worker.GetSandboxClass() != constraints.SandboxClass { + if constraints.SandboxClass != "" && workerClass != constraints.SandboxClass { return false } @@ -224,14 +230,6 @@ func (s *scheduler) Applies(worker *ateapipb.Worker, constraints Constraints) bo return false } - // The assignment records this runtime and atelet requires it on - // Run/Restore, so a worker that has not registered one cannot host - // anything yet (see ServiceImpl.CreateWorker: "Nothing is placed on it - // in the meantime"). - if worker.GetStatus().GetDefaultRuntime() == nil { - return false - } - if len(constraints.RequiredNodes) > 0 && !slices.Contains(constraints.RequiredNodes, worker.GetNodeName()) { return false } diff --git a/cmd/ateapi/internal/scheduling/scheduling_test.go b/cmd/ateapi/internal/scheduling/scheduling_test.go index 36aa42547a..94a4b852ed 100644 --- a/cmd/ateapi/internal/scheduling/scheduling_test.go +++ b/cmd/ateapi/internal/scheduling/scheduling_test.go @@ -512,10 +512,9 @@ func worker(pod, class, node string, lbls map[string]string, opts ...func(*ateap defRuntime = sandboxRuntime(class, "amd64") } w := &ateapipb.Worker{ - WorkerPod: pod, - SandboxClass: class, - NodeName: node, - Labels: lbls, + WorkerPod: pod, + NodeName: node, + Labels: lbls, // A stored Worker always carries a ceiling; CreateWorker reifies one. Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, diff --git a/cmd/ateapi/internal/workerservice/register.go b/cmd/ateapi/internal/workerservice/register.go index 625e8c7e6b..c59827f758 100644 --- a/cmd/ateapi/internal/workerservice/register.go +++ b/cmd/ateapi/internal/workerservice/register.go @@ -69,11 +69,11 @@ func (s *Server) RegisterWorker(ctx context.Context, req *ateapipb.RegisterWorke slog.String("caller_pod", caller.PodName)) return nil, apierror.NotFound("Worker %s not found", name) } - if err := validateRuntimeClasses(worker.GetSandboxClass(), req); err != nil { + status := worker.GetStatus() + if err := validateRuntimeClasses(status.GetDefaultRuntime().GetSandboxClass(), req); err != nil { return nil, err } - status := worker.GetStatus() if proto.Equal(status.GetCapacity(), reported) && proto.Equal(status.GetDefaultRuntime(), defaultRuntime) && sameRuntimes(status.GetRestorableRuntimes(), restorable) { @@ -107,21 +107,18 @@ func (s *Server) RegisterWorker(ctx context.Context, req *ateapipb.RegisterWorke return &ateapipb.RegisterWorkerResponse{Worker: updated}, nil } -// validateRuntimeClasses rejects a runtime of a class other than the Worker's -// own. The class is fixed when the pool creates the Worker and names the -// sandbox its worker runs, so a runtime of another class contradicts the -// recorded Worker state. A Worker recorded without a class has nothing for a -// report to contradict. -func validateRuntimeClasses(class string, req *ateapipb.RegisterWorkerRequest) error { - if class == "" { - return nil - } - if got := req.GetDefaultRuntime().GetSandboxClass(); got != class { - return apierror.FailedPrecondition("default_runtime.sandbox_class %q does not match Worker's sandbox_class %q", got, class) +// validateRuntimeClasses rejects a runtime whose sandbox class disagrees with +// default_runtime or with a class already recorded on the Worker. A worker pod +// runs one sandbox class for its lifetime, so every runtime it reports must +// share that class. +func validateRuntimeClasses(existingClass string, req *ateapipb.RegisterWorkerRequest) error { + class := req.GetDefaultRuntime().GetSandboxClass() + if existingClass != "" && class != existingClass { + return apierror.FailedPrecondition("default_runtime.sandbox_class %q does not match Worker's recorded default_runtime.sandbox_class %q", class, existingClass) } for i, rt := range req.GetRestorableRuntimes() { if got := rt.GetSandboxClass(); got != class { - return apierror.FailedPrecondition("restorable_runtimes[%d].sandbox_class %q does not match Worker's sandbox_class %q", i, got, class) + return apierror.FailedPrecondition("restorable_runtimes[%d].sandbox_class %q does not match default_runtime.sandbox_class %q", i, got, class) } } return nil diff --git a/cmd/ateapi/internal/workerservice/register_test.go b/cmd/ateapi/internal/workerservice/register_test.go index c1cdaf9cc4..8721420492 100644 --- a/cmd/ateapi/internal/workerservice/register_test.go +++ b/cmd/ateapi/internal/workerservice/register_test.go @@ -103,80 +103,77 @@ func TestRegisterWorker_Runtimes(t *testing.T) { tests := []struct { name string - // workerClass is the class the pool created the Worker with; "" is - // a Worker recorded without one. - workerClass string - def *ateapipb.SandboxRuntime - restorable []*ateapipb.SandboxRuntime - wantCode codes.Code + // unreported seeds a Worker with no prior DefaultRuntime instead of + // seeded(). + unreported bool + def *ateapipb.SandboxRuntime + restorable []*ateapipb.SandboxRuntime + wantCode codes.Code // wantWrite is whether the report changes the record. wantWrite bool }{{ - name: "identical report writes nothing", - workerClass: "gvisor", - def: gvisorAMD64, - restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, + name: "identical report writes nothing", + def: gvisorAMD64, + restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, }, { - name: "changed default runtime writes", - workerClass: "gvisor", - def: gvisorARM64, - restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, - wantWrite: true, + name: "changed default runtime writes", + def: gvisorARM64, + restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, + wantWrite: true, }, { - name: "changed restorable runtime writes", - workerClass: "gvisor", - def: gvisorAMD64, - restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorARM64}, - wantWrite: true, + name: "changed restorable runtime writes", + def: gvisorAMD64, + restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorARM64}, + wantWrite: true, }, { // A report is compared as sent: the list is atomic, so a new order // is a new report. - name: "reordered restorable runtimes write", - workerClass: "gvisor", - def: gvisorAMD64, - restorable: []*ateapipb.SandboxRuntime{gvisorV9, gvisorV8}, - wantWrite: true, + name: "reordered restorable runtimes write", + def: gvisorAMD64, + restorable: []*ateapipb.SandboxRuntime{gvisorV9, gvisorV8}, + wantWrite: true, }, { // Runtimes are replaced like capacity: a report that lists no // restorable runtimes is a Worker that can restore from none but its // default, as when a version is disabled, and the record must not // keep advertising them. - name: "dropped restorable runtimes write", - workerClass: "gvisor", - def: gvisorAMD64, - wantWrite: true, + name: "dropped restorable runtimes write", + def: gvisorAMD64, + wantWrite: true, }, { - // The class is fixed by the pool that created the Worker. A runtime - // of another class is one it cannot run, so the report is refused - // rather than recorded for the scheduler to act on. - name: "default runtime of another class is refused", - workerClass: "gvisor", - def: microvm, - restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, - wantCode: codes.FailedPrecondition, + // Once a Worker has registered a sandbox class, a runtime of another + // class contradicts its recorded state and is refused. + name: "default runtime of another class is refused", + def: microvm, + restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, + wantCode: codes.FailedPrecondition, }, { - name: "restorable runtime of another class is refused", - workerClass: "gvisor", - def: gvisorAMD64, - restorable: []*ateapipb.SandboxRuntime{gvisorV8, microvm}, - wantCode: codes.FailedPrecondition, + name: "restorable runtime of another class is refused", + def: gvisorAMD64, + restorable: []*ateapipb.SandboxRuntime{gvisorV8, microvm}, + wantCode: codes.FailedPrecondition, }, { - // A Worker recorded without a class has nothing for a report to - // contradict. - name: "unclassed Worker takes a default runtime of any class", + // A Worker that has not yet registered a default runtime accepts any + // class on its first report, provided restorable_runtimes match it. + name: "unreported Worker takes a default runtime of any class", + unreported: true, def: microvm, - restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, wantWrite: true, }, { - name: "unclassed Worker takes restorable runtimes of mixed classes", + name: "unreported Worker refuses restorable runtimes of mixed classes", + unreported: true, def: gvisorAMD64, restorable: []*ateapipb.SandboxRuntime{gvisorV8, microvm}, - wantWrite: true, + wantCode: codes.FailedPrecondition, }} for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { name := fmt.Sprintf("1a7e4c83-6d20-4f95-b3c8-%012d", i) - before := seedWorker(t, st, name, testNode, tt.workerClass, seeded()) + initialStatus := seeded() + if tt.unreported { + initialStatus = &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE} + } + before := seedWorker(t, st, name, testNode, initialStatus) got, err := s.RegisterWorker(authed, &ateapipb.RegisterWorkerRequest{ Worker: &ateapipb.ObjectRef{Name: name}, diff --git a/cmd/ateapi/internal/workerservice/service_test.go b/cmd/ateapi/internal/workerservice/service_test.go index 56d72a0a3c..c804bc0f80 100644 --- a/cmd/ateapi/internal/workerservice/service_test.go +++ b/cmd/ateapi/internal/workerservice/service_test.go @@ -85,15 +85,14 @@ func (f *fakeControl) CrashActorReportedByWorker(_ context.Context, actorRef res // this seeds. func seedReportedWorker(t *testing.T, st store.Interface, nodeName string, capacity *ateapipb.WorkerResources) *ateapipb.Worker { t.Helper() - return seedWorker(t, st, testWorkerName, nodeName, testDefaultRuntime.GetSandboxClass(), &ateapipb.WorkerStatus{ + return seedWorker(t, st, testWorkerName, nodeName, &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: capacity, DefaultRuntime: testDefaultRuntime, }) } -// seedWorker registers a Worker named name, of sandbox class class, on -// nodeName, with status already recorded. name doubles as the pod UID, so it -// must be UUID-shaped. class "" is a Worker recorded without one. -func seedWorker(t *testing.T, st store.Interface, name, nodeName, class string, status *ateapipb.WorkerStatus) *ateapipb.Worker { +// seedWorker registers a Worker named name on nodeName, with status already +// recorded. name doubles as the pod UID, so it must be UUID-shaped. +func seedWorker(t *testing.T, st store.Interface, name, nodeName string, status *ateapipb.WorkerStatus) *ateapipb.Worker { t.Helper() created, err := st.CreateWorker(context.Background(), &ateapipb.Worker{ Metadata: &ateapipb.ResourceMetadata{Name: name}, @@ -103,7 +102,6 @@ func seedWorker(t *testing.T, st store.Interface, name, nodeName, class string, WorkerPodUid: name, NodeName: nodeName, Ips: []string{"10.1.2.3"}, - SandboxClass: class, Status: status, }) if err != nil { diff --git a/cmd/atecontroller/internal/workersync/syncer.go b/cmd/atecontroller/internal/workersync/syncer.go index 7f485bc26b..d6ecb1661e 100644 --- a/cmd/atecontroller/internal/workersync/syncer.go +++ b/cmd/atecontroller/internal/workersync/syncer.go @@ -346,13 +346,11 @@ func (s *WorkerPoolSyncer) createOrUpdateWorker(ctx context.Context, key workerK WorkerPodUid: string(pod.UID), NodeName: pod.Spec.NodeName, WorkerSpiffeId: workerSPIFFEID(pod), - SandboxClass: string(pool.Spec.DefaultSandboxClass()), Labels: pool.GetLabels(), Epoch: podEpoch(pod), - // Capacity is the Worker's to report, not the syncer's to infer - // from the pod: it is what the worker can actually supply. Until - // that report lands, CreateWorker's reified ceiling holds the - // Worker to a single Actor. + // Capacity and runtimes are the Worker's to report, not the + // syncer's to infer from the pod: they are what the worker can + // actually supply. } // status is output-only: CreateWorker registers the Worker UNAVAILABLE. // @@ -397,16 +395,6 @@ func (s *WorkerPoolSyncer) createOrUpdateWorker(ctx context.Context, key workerK w.Labels = pool.GetLabels() changed = true } - if w.GetSandboxClass() != string(pool.Spec.DefaultSandboxClass()) { - // Expected mid-rollout: sandboxClass drives the worker pod's shape, so - // editing it on the pool replaces every pod rather than reclassifying - // any. This pod predates that edit and is on its way out; its successor - // registers under a new key with the new class. Writing the pool's value - // back would be rejected, and would misreport this pod's shape to the - // scheduler if it were not. - slog.DebugContext(ctx, "Syncer: registered worker sandbox class predates its pool", - append(key.logAttrs(), slog.String("registered", w.GetSandboxClass()), slog.String("pool", string(pool.Spec.DefaultSandboxClass())))...) - } if ips := podIPs(pod); len(ips) > 0 && !slices.Equal(w.GetIps(), ips) { // TODO: I don't think this is possible, but handling this case so we can // log it just in case we can reproduce it. It is logged rather than diff --git a/cmd/atecontroller/internal/workersync/syncer_restart_test.go b/cmd/atecontroller/internal/workersync/syncer_restart_test.go index ec540dd996..6a788cc33d 100644 --- a/cmd/atecontroller/internal/workersync/syncer_restart_test.go +++ b/cmd/atecontroller/internal/workersync/syncer_restart_test.go @@ -50,7 +50,7 @@ func TestSyncer_RegistersEpoch(t *testing.T) { ns, podName, poolName := "ns-syncer-epoch", "worker-epoch-1", "pool1" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) key := seedPod(t, pods, withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 2)) mustReconcile(t, ctx, s, key) @@ -73,7 +73,7 @@ func TestSyncer_RaisesEpochOnWorkerRestart(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 0) key := seedPod(t, pods, pod) mustReconcile(t, ctx, s, key) @@ -173,7 +173,7 @@ func TestSyncer_NoPlacementUnderAnEarlierEpoch(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) key := seedPod(t, pods, withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 0)) mustReconcile(t, ctx, s, key) if state := api.get(testPodUID).GetStatus().GetState(); state != ateapipb.WorkerState_WORKER_STATE_ACTIVE { @@ -209,7 +209,7 @@ func TestSyncer_NotReadyPodRegistersAtItsEpoch(t *testing.T) { ns, poolName := "ns-syncer-notready-restart", "pool1" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := withWorkerRestarts(workerPod(ns, "worker-1", poolName, testPodUID, "10.0.0.1"), 3) pod.Status.Conditions = nil key := seedPod(t, pods, pod) @@ -229,9 +229,8 @@ func TestSyncer_EpochNeverLowered(t *testing.T) { ns, podName, poolName := "ns-syncer-lower", "worker-lower-1", "pool1" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) registered := registeredWorker(ns, poolName, podName, testPodUID, "10.0.0.1") - registered.SandboxClass = "gvisor" registered.Epoch = 5 api.put(registered) key := seedPod(t, pods, withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.1"), 0)) diff --git a/cmd/atecontroller/internal/workersync/syncer_test.go b/cmd/atecontroller/internal/workersync/syncer_test.go index 7b1623ecd2..437e37ae69 100644 --- a/cmd/atecontroller/internal/workersync/syncer_test.go +++ b/cmd/atecontroller/internal/workersync/syncer_test.go @@ -83,10 +83,10 @@ func workerPod(ns, name, poolName, uid, ip string) *corev1.Pod { return pod } -func workerPool(ns, name, sandboxClass string, labels map[string]string) *atev1alpha1.WorkerPool { +func workerPool(ns, name string, labels map[string]string) *atev1alpha1.WorkerPool { return &atev1alpha1.WorkerPool{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}, - Spec: atev1alpha1.WorkerPoolSpec{SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClass(sandboxClass)}}}, + Spec: atev1alpha1.WorkerPoolSpec{SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClassGvisor}}}, } } @@ -208,7 +208,7 @@ func TestSyncer_Lifecycle(t *testing.T) { ns, podName, poolName := "ns-syncer-lifecycle", "worker-unit-1", "pool1" api := newFakeControl() - fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, "gvisor", map[string]string{"foo": "bar"})) + fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, map[string]string{"foo": "bar"})) // A pod with no IP is not registered: there is nothing to route to yet. pod := workerPod(ns, podName, poolName, testPodUID, "") @@ -237,9 +237,6 @@ func TestSyncer_Lifecycle(t *testing.T) { if want := []string{"127.0.0.1"}; !slices.Equal(got.GetIps(), want) { t.Errorf("worker ips = %q, want %q", got.GetIps(), want) } - if got.GetSandboxClass() != "gvisor" { - t.Errorf("worker sandbox class = %q, want gvisor", got.GetSandboxClass()) - } if !maps.Equal(got.GetLabels(), map[string]string{"foo": "bar"}) { t.Errorf("worker labels = %v, want map[foo:bar]", got.GetLabels()) } @@ -250,8 +247,8 @@ func TestSyncer_Lifecycle(t *testing.T) { waitForWorker(t, ctx, api, testPodUID, func(w *ateapipb.Worker) bool { return w == nil }) } -// A pool that sets neither a sandbox class nor labels registers a worker -// carrying neither, rather than one carrying some default. +// A pool that sets no labels registers a worker carrying none, rather than one +// carrying some default. func TestSyncer_OmittedFields(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -259,7 +256,7 @@ func TestSyncer_OmittedFields(t *testing.T) { ns, podName, poolName := "ns-syncer-omitted", "worker-unit-1", "pool1" api := newFakeControl() - fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, "", nil)) + fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "127.0.0.1") if _, err := fakeK8s.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil { @@ -267,9 +264,6 @@ func TestSyncer_OmittedFields(t *testing.T) { } got := waitForWorker(t, ctx, api, testPodUID, func(w *ateapipb.Worker) bool { return w != nil }) - if got.GetSandboxClass() != "" { - t.Errorf("worker sandbox class = %q, want it empty", got.GetSandboxClass()) - } if len(got.GetLabels()) != 0 { t.Errorf("worker labels = %v, want none", got.GetLabels()) } @@ -286,7 +280,7 @@ func TestSyncer_DoesNotInferCapacityFromThePod(t *testing.T) { ns, podName, poolName := "ns-syncer-capacity", "worker-unit-1", "pool1" api := newFakeControl() - fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, "gvisor", nil)) + fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "127.0.0.1") pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{ @@ -442,7 +436,7 @@ func TestSyncer_ReconcileOrphanedWorkers(t *testing.T) { api.put(registeredWorker(ns, poolName, "worker-live", testPodUID, "10.0.0.9")) api.put(registeredWorker(ns, poolName, "worker-orphan", otherPodUID, "10.0.0.10")) - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) // The cache holds only the live pod, so it is an authoritative snapshot. seedPod(t, pods, workerPod(ns, "worker-live", poolName, testPodUID, "10.0.0.9")) @@ -558,7 +552,7 @@ func TestSyncer_CreateWorker_DualStack(t *testing.T) { ns, podName, poolName := "ns-syncer-dual-stack", "worker-unit-dual-stack", "pool-dual-stack" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "fd00::1") pod.Status.PodIPs = append(pod.Status.PodIPs, corev1.PodIP{IP: "10.0.0.1"}) key := seedPod(t, pods, pod) @@ -575,7 +569,7 @@ func TestSyncer_CreateWorker_SPIFFEID(t *testing.T) { ctx := context.Background() ns, podName, poolName := "ns-syncer-sa", "worker-unit-sa", "pool-sa" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "10.0.0.1") pod.Spec.ServiceAccountName = "substrate-worker" key := seedPod(t, pods, pod) @@ -597,7 +591,7 @@ func TestSyncer_UpdateWorker_RetryOnVersionConflict(t *testing.T) { ns, podName, poolName := "ns-syncer-conflict", "worker-unit-conflict", "pool-conflict" api := newFakeControl() - s, pods, poolIndexer := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", map[string]string{"foo": "bar"})) + s, pods, poolIndexer := setupReconcileTest(t, api, workerPool(ns, poolName, map[string]string{"foo": "bar"})) key := seedPod(t, pods, workerPod(ns, podName, poolName, testPodUID, "10.0.0.1")) mustReconcile(t, ctx, s, key) @@ -607,7 +601,7 @@ func TestSyncer_UpdateWorker_RetryOnVersionConflict(t *testing.T) { // Change a mutable worker field on the pool, so the next reconcile has an // update to make. - if err := poolIndexer.Update(workerPool(ns, poolName, "gvisor", map[string]string{"foo": "baz"})); err != nil { + if err := poolIndexer.Update(workerPool(ns, poolName, map[string]string{"foo": "baz"})); err != nil { t.Fatalf("updating pool: %v", err) } @@ -645,44 +639,13 @@ func TestSyncer_UpdateWorker_RetryOnVersionConflict(t *testing.T) { } } -// Editing a pool's sandboxClass rolls its pods rather than reclassifying them, -// so a pod that outlives the edit keeps the class it was built with. Writing the -// pool's new value back would be rejected as an immutable-field change, and -// would misreport this pod's shape to the scheduler if it were not. -func TestSyncer_SandboxClassDriftLeavesWorkerAlone(t *testing.T) { - ctx := context.Background() - - ns, podName, poolName := "ns-syncer-class", "worker-unit-class", "pool-class" - - api := newFakeControl() - s, pods, poolIndexer := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) - key := seedPod(t, pods, workerPod(ns, podName, poolName, testPodUID, "10.0.0.1")) - - mustReconcile(t, ctx, s, key) - registered := api.get(testPodUID) - - if err := poolIndexer.Update(workerPool(ns, poolName, "microvm", nil)); err != nil { - t.Fatalf("updating pool: %v", err) - } - mustReconcile(t, ctx, s, key) - - got := api.get(testPodUID) - if got.GetSandboxClass() != "gvisor" { - t.Errorf("worker sandbox class = %q, want it left at gvisor", got.GetSandboxClass()) - } - if got.GetMetadata().GetVersion() != registered.GetMetadata().GetVersion() { - t.Errorf("worker version = %d, want %d: the drift must not provoke a write", - got.GetMetadata().GetVersion(), registered.GetMetadata().GetVersion()) - } -} - // TestSyncer_MissingWorkerPool verifies that a pod whose WorkerPool is not in // the lister gets no Worker, and that the Worker comes once the pool does. func TestSyncer_MissingWorkerPool(t *testing.T) { ctx := context.Background() ns, podName, poolName := "ns-syncer-missingpool", "worker-orphan-1", "pool-orphan" - pool := workerPool(ns, poolName, "gvisor", nil) + pool := workerPool(ns, poolName, map[string]string{"foo": "bar"}) api := newFakeControl() // The pod is there; its pool is not yet. @@ -703,8 +666,8 @@ func TestSyncer_MissingWorkerPool(t *testing.T) { t.Fatalf("queued workers = %d after the pool appeared, want 1", got) } mustReconcile(t, ctx, s, key) - if got := api.get(testPodUID).GetSandboxClass(); got != "gvisor" { - t.Errorf("worker sandbox class = %q, want gvisor", got) + if got := api.get(testPodUID).GetLabels(); !maps.Equal(got, map[string]string{"foo": "bar"}) { + t.Errorf("worker labels = %v, want map[foo:bar]", got) } } @@ -930,7 +893,7 @@ func TestSyncer_OrphanedPodsOfDeletedPool_ViaInformer(t *testing.T) { ns, podName, poolName := "ns-syncer-orphaned", "worker-orphaned-1", "pool1" api := newFakeControl() - fakeK8s, poolClient := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, "gvisor", nil)) + fakeK8s, poolClient := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "10.0.0.7") if _, err := fakeK8s.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil { @@ -966,7 +929,7 @@ func TestSyncer_StartWaitsForWorkerPoolCache(t *testing.T) { //nolint:staticcheck // NewSimpleClientset is what the informer machinery takes. fakeK8s := fake.NewSimpleClientset(workerPod(ns, podName, poolName, testPodUID, "10.0.0.8")) workerFactory, workerInformer := WorkerPodInformer(fakeK8s) - _, poolFactory, workerPoolInformer := newWorkerPoolInformer(workerPool(ns, poolName, "", poolLabels)) + _, poolFactory, workerPoolInformer := newWorkerPoolInformer(workerPool(ns, poolName, poolLabels)) NewWorkerPoolSyncer(api, fakeK8s.CoreV1(), fakeK8s.AppsV1(), workerInformer, workerPoolInformer, 2).Start(ctx) workerFactory.Start(ctx.Done()) @@ -1000,7 +963,7 @@ func TestSyncer_StartWaitsForWorkerPoolCache(t *testing.T) { func TestEnqueueWorkerPool(t *testing.T) { api := newFakeControl() s, pods, _ := setupReconcileTest(t, api) - pool := workerPool("ns-pool-update", "pool-a", "gvisor", nil) + pool := workerPool("ns-pool-update", "pool-a", nil) first := seedPod(t, pods, workerPod(pool.Namespace, "worker-1", pool.Name, testPodUID, "10.0.0.1")) second := seedPod(t, pods, workerPod(pool.Namespace, "worker-2", pool.Name, otherPodUID, "10.0.0.2")) @@ -1030,7 +993,7 @@ func TestEnqueueWorkerPool(t *testing.T) { func TestEnqueueWorkerPool_DeletedFinalStateUnknown(t *testing.T) { api := newFakeControl() s, pods, _ := setupReconcileTest(t, api) - pool := workerPool("ns-pool-tombstone", "pool-a", "gvisor", nil) + pool := workerPool("ns-pool-tombstone", "pool-a", nil) seedPod(t, pods, workerPod(pool.Namespace, "worker-1", pool.Name, testPodUID, "10.0.0.1")) s.enqueueWorkerPool(cache.DeletedFinalStateUnknown{Key: pool.Namespace + "/" + pool.Name, Obj: pool}) @@ -1055,7 +1018,7 @@ func TestSyncer_SoftDelete_ViaInformer(t *testing.T) { ns, podName, poolName := "ns-syncer-softdelete", "worker-soft-1", "pool1" api := newFakeControl() - fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, "gvisor", nil)) + fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "10.0.0.6") if _, err := fakeK8s.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil { @@ -1088,7 +1051,7 @@ func TestSyncer_TerminalPod_DeletesPod(t *testing.T) { ctx := context.Background() api := newFakeControl() api.put(registeredWorker(ns, poolName, podName, testPodUID, ip)) - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, ip) pod.Status.Phase = phase @@ -1160,7 +1123,7 @@ func TestSyncer_TerminalPod_ViaInformer(t *testing.T) { ns, podName, poolName := "ns-syncer-terminal", "worker-terminal-1", "pool1" api := newFakeControl() - fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, "gvisor", nil)) + fakeK8s, _ := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "10.0.0.21") if _, err := fakeK8s.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil { @@ -1191,7 +1154,7 @@ func TestSyncer_PodRecreatedWithNewUID(t *testing.T) { ns, podName, poolName := "ns-syncer-recreate", "worker-recreate-1", "pool1" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) oldKey := seedPod(t, pods, workerPod(ns, podName, poolName, testPodUID, "10.0.0.7")) mustReconcile(t, ctx, s, oldKey) @@ -1235,7 +1198,7 @@ func TestSyncer_DeletePodWithoutIP(t *testing.T) { ns, podName, poolName := "ns-syncer-neverip", "worker-noip-1", "pool1" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "") key := seedPod(t, pods, pod) @@ -1274,7 +1237,7 @@ func TestSyncer_PodWithIPButNotReadyIsUnavailable(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, "worker-notready-1", poolName, testPodUID, "10.0.0.9") tc.mutate(pod) key := seedPod(t, pods, pod) @@ -1298,7 +1261,7 @@ func TestSyncer_DeleteNeverReadyPod(t *testing.T) { ns, podName, poolName := "ns-syncer-neverready", "worker-neverready-1", "pool1" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "10.0.0.9") pod.Status.Conditions = nil key := seedPod(t, pods, pod) @@ -1323,7 +1286,7 @@ func TestSyncer_RegisteredPodWithoutIPIsUnavailable(t *testing.T) { ns, podName, poolName := "ns-syncer-lostip", "worker-lostip-1", "pool1" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) key := seedPod(t, pods, withWorkerRestarts(workerPod(ns, podName, poolName, testPodUID, "10.0.0.9"), 0)) mustReconcile(t, ctx, s, key) @@ -1352,7 +1315,7 @@ func TestSyncer_ReadinessFlapDoesNotDeregister(t *testing.T) { ns, podName, poolName := "ns-syncer-flap", "worker-flap-1", "pool1" api := newFakeControl() - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) pod := workerPod(ns, podName, poolName, testPodUID, "10.0.0.9") key := seedPod(t, pods, pod) @@ -1408,7 +1371,7 @@ func TestSyncer_ReconcileErrorsAreClassified(t *testing.T) { api.setCreateHook(func(*ateapipb.Worker) error { return status.Error(tc.code, "rejected") }) - s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, nil)) key := seedPod(t, pods, workerPod(ns, podName, poolName, testPodUID, "10.0.0.9")) s.queue.Add(key) diff --git a/cmd/benchmarking/isolate/fake-workersync/controller.go b/cmd/benchmarking/isolate/fake-workersync/controller.go index 23e76e6ed0..29fa2d0f3f 100644 --- a/cmd/benchmarking/isolate/fake-workersync/controller.go +++ b/cmd/benchmarking/isolate/fake-workersync/controller.go @@ -96,9 +96,8 @@ type fakeWorker struct { // from before its create, so a create the server commits but the caller // sees fail is still retired, or deleted by --cleanup. created bool - // labels and sandboxClass are what the Worker is registered with. - labels map[string]string - sandboxClass string + // labels is synced onto the Worker record. + labels map[string]string // tmpl is the pool template the Worker was made from; nil for a Worker // adopted after a restart until reconcile matches it to its pool. tmpl *template @@ -201,16 +200,15 @@ func (c *controller) adopt(ctx context.Context) error { } name := w.GetMetadata().GetName() c.workers[name] = &fakeWorker{ - name: name, - pod: pod, - namespace: w.GetWorkerNamespace(), - pool: w.GetWorkerPool(), - index: index, - node: w.GetNodeName(), - labels: w.GetLabels(), - sandboxClass: w.GetSandboxClass(), - capacity: w.GetStatus().GetCapacity(), - reported: w.GetStatus().GetCapacity() != nil, + name: name, + pod: pod, + namespace: w.GetWorkerNamespace(), + pool: w.GetWorkerPool(), + index: index, + node: w.GetNodeName(), + labels: w.GetLabels(), + capacity: w.GetStatus().GetCapacity(), + reported: w.GetStatus().GetCapacity() != nil, // One never marked available goes back through create, which // marks it. created: w.GetStatus().GetState() != ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE, @@ -361,14 +359,13 @@ func (c *controller) ensure(ctx context.Context, pod string, d desiredWorker, w return errors.New("no fake-atelet node to place fake Workers on") } w = &fakeWorker{ - name: c.newUID(), - pod: pod, - namespace: d.pool.Namespace, - pool: d.pool.Name, - index: d.index, - node: fakeworker.Node(d.index, nodes), - labels: maps.Clone(d.pool.GetLabels()), - sandboxClass: string(d.pool.Spec.DefaultSandboxClass()), + name: c.newUID(), + pod: pod, + namespace: d.pool.Namespace, + pool: d.pool.Name, + index: d.index, + node: fakeworker.Node(d.index, nodes), + labels: maps.Clone(d.pool.GetLabels()), } c.mu.Lock() c.workers[w.name] = w @@ -456,7 +453,6 @@ func (c *controller) create(ctx context.Context, w *fakeWorker, wp *atev1alpha1. WorkerPodUid: name, NodeName: w.node, Ips: []string{fakeworker.IP(w.index)}, - SandboxClass: string(wp.Spec.DefaultSandboxClass()), Labels: maps.Clone(wp.GetLabels()), }}) if err != nil && status.Code(err) != codes.AlreadyExists { @@ -641,14 +637,11 @@ func (t template) capacity(allocatable corev1.ResourceList) *ateapipb.WorkerReso // fits reports whether w was made from wp's current template, so a real pool // would keep its pod. A Worker adopted after a restart has only its reported -// capacity to go by, so it fits when that is what wp would give it on its -// node now; one whose node's allocatable changed while fake-workersync was -// down is replaced too. A held pool's Workers fit, since they are left as -// they are. +// capacity to go by, so it fits when that capacity is what wp would give it on +// its node now; one whose node's allocatable changed while fake-workersync was +// down is replaced too. A held pool's Workers fit, since they are left as they +// are. func fits(w *fakeWorker, wp *atev1alpha1.WorkerPool, allocatable map[string]corev1.ResourceList) bool { - if w.sandboxClass != string(wp.Spec.DefaultSandboxClass()) { - return false - } t, err := templateOf(wp) if err != nil { return true diff --git a/cmd/benchmarking/isolate/fake-workersync/controller_test.go b/cmd/benchmarking/isolate/fake-workersync/controller_test.go index f8c74bc81f..4e6001c44e 100644 --- a/cmd/benchmarking/isolate/fake-workersync/controller_test.go +++ b/cmd/benchmarking/isolate/fake-workersync/controller_test.go @@ -435,8 +435,8 @@ func TestReconcileHonorsReplicasAndLimits(t *testing.T) { t.Errorf("index 1 placed on %q, want node-b (round robin)", w.GetNodeName()) case w.GetWorkerNamespace() != "benchmark-workloads" || w.GetWorkerPool() != "bench": t.Errorf("WorkerNamespace/WorkerPool = %q/%q", w.GetWorkerNamespace(), w.GetWorkerPool()) - case w.GetSandboxClass() != "gvisor": - t.Errorf("SandboxClass = %q", w.GetSandboxClass()) + case w.GetStatus().GetDefaultRuntime().GetSandboxClass() != "gvisor": + t.Errorf("DefaultRuntime.SandboxClass = %q", w.GetStatus().GetDefaultRuntime().GetSandboxClass()) case w.GetLabels()["workload"] != "bench": t.Errorf("Labels = %v, want the pool's, which template selectors match", w.GetLabels()) case w.GetWorkerPodUid() != w.GetMetadata().GetName() || w.GetEpoch() != 0: @@ -949,10 +949,11 @@ func TestPoolLabelChangeUpdatesWorkers(t *testing.T) { } } -// sandbox_class is immutable on a Worker, so a sandboxClass edit replaces the -// pool's Workers the way it replaces real worker pods: each pod gets a new -// Worker, under a new name and pod UID, at once; an idle old Worker is deleted -// at once, a held one once its Actors leave. +// A Worker's sandbox class is fixed on its first RegisterWorker report, so a +// sandboxClass edit replaces the pool's Workers the way it replaces real +// worker pods: each pod gets a new Worker, under a new name and pod UID, at +// once; an idle old Worker is deleted at once, a held one once its Actors +// leave. func TestPoolSandboxClassChangeReplacesWorkers(t *testing.T) { c, ctl, _, cl := newTestController(pool("bench", 2, nil)) if err := reconcile(t, c); err != nil { @@ -971,15 +972,15 @@ func TestPoolSandboxClassChangeReplacesWorkers(t *testing.T) { } for _, pod := range []string{idle, busy} { w := ctl.worker(pod) - if w.GetSandboxClass() != "microvm" || w.GetStatus().GetState() != ateapipb.WorkerState_WORKER_STATE_ACTIVE { - t.Errorf("Worker for %s = %q/%v, want microvm and active", pod, w.GetSandboxClass(), w.GetStatus().GetState()) + if w.GetStatus().GetDefaultRuntime().GetSandboxClass() != "microvm" || w.GetStatus().GetState() != ateapipb.WorkerState_WORKER_STATE_ACTIVE { + t.Errorf("Worker for %s = %q/%v, want microvm and active", pod, w.GetStatus().GetDefaultRuntime().GetSandboxClass(), w.GetStatus().GetState()) } if w.GetWorkerPodUid() != w.GetMetadata().GetName() { t.Errorf("Worker %s has pod UID %s, want its name", w.GetMetadata().GetName(), w.GetWorkerPodUid()) } } - if w := ctl.draining(busy); w.GetMetadata().GetName() != oldBusy || w.GetSandboxClass() != "gvisor" { - t.Errorf("draining Worker for %s = %s/%q, want the old gvisor one, %s", busy, w.GetMetadata().GetName(), w.GetSandboxClass(), oldBusy) + if w := ctl.draining(busy); w.GetMetadata().GetName() != oldBusy || w.GetStatus().GetDefaultRuntime().GetSandboxClass() != "gvisor" { + t.Errorf("draining Worker for %s = %s/%q, want the old gvisor one, %s", busy, w.GetMetadata().GetName(), w.GetStatus().GetDefaultRuntime().GetSandboxClass(), oldBusy) } if got := cl.statuses["bench"]; got.Replicas != 2 || got.ReadyReplicas != 2 { t.Errorf("status = %+v, want 2 replicas, 2 ready: the draining Worker is no replica", got) @@ -1013,7 +1014,7 @@ func TestFailedDrainOfAReplacedWorkerIsNoReplica(t *testing.T) { } var replacement *ateapipb.Worker for _, w := range ctl.all() { - if w.GetSandboxClass() == "microvm" { + if w.GetStatus().GetDefaultRuntime().GetSandboxClass() == "microvm" { replacement = w } } diff --git a/cmd/kubectl-ate/internal/cmd/worker.go b/cmd/kubectl-ate/internal/cmd/worker.go index 518ab4bb38..69f4076973 100644 --- a/cmd/kubectl-ate/internal/cmd/worker.go +++ b/cmd/kubectl-ate/internal/cmd/worker.go @@ -154,7 +154,7 @@ func filterWorkers(ctx context.Context, actors ActorLister, workers []*ateapipb. if labelSel != nil && !labelSel.Matches(labels.Set(w.GetLabels())) { continue } - if sandboxClass != "" && w.GetSandboxClass() != sandboxClass { + if sandboxClass != "" && w.GetStatus().GetDefaultRuntime().GetSandboxClass() != sandboxClass { continue } filtered = append(filtered, w) @@ -332,7 +332,7 @@ func (r *TopWorkersRunner) Run(ctx context.Context) error { items = append(items, &printer.WorkerTopItem{ Pod: podName, Pool: pool, - Class: w.GetSandboxClass(), + Class: w.GetStatus().GetDefaultRuntime().GetSandboxClass(), Status: printer.WorkerOccupancy(w), CPU: cpuStr, Memory: memStr, diff --git a/cmd/kubectl-ate/internal/cmd/worker_test.go b/cmd/kubectl-ate/internal/cmd/worker_test.go index 0e8a927730..90ad3b2f6c 100644 --- a/cmd/kubectl-ate/internal/cmd/worker_test.go +++ b/cmd/kubectl-ate/internal/cmd/worker_test.go @@ -125,12 +125,12 @@ func TestGetWorkersRunner_Filters(t *testing.T) { WorkerNamespace: "ns-1", WorkerPool: "counter", WorkerPod: "pod-1", - SandboxClass: "microvm", Labels: map[string]string{"ate.dev/worker-pool": "counter"}, Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - Capacity: &ateapipb.WorkerResources{Actors: 1}, - Allocated: &ateapipb.WorkerResources{Actors: 1}, + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + Allocated: &ateapipb.WorkerResources{Actors: 1}, + DefaultRuntime: &ateapipb.SandboxRuntime{SandboxClass: "microvm"}, }, }, { @@ -138,21 +138,23 @@ func TestGetWorkersRunner_Filters(t *testing.T) { WorkerNamespace: "ns-1", WorkerPool: "other", WorkerPod: "pod-2", - SandboxClass: "gvisor", Labels: map[string]string{"ate.dev/worker-pool": "other"}, - Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE}, + Status: &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + DefaultRuntime: &ateapipb.SandboxRuntime{SandboxClass: "gvisor"}, + }, }, { Metadata: &ateapipb.ResourceMetadata{Name: "worker-3", CreateTime: timestamppb.New(now.Add(-72 * time.Hour))}, WorkerNamespace: "ns-2", WorkerPool: "counter", WorkerPod: "pod-3", - SandboxClass: "gvisor", Labels: map[string]string{"ate.dev/worker-pool": "counter"}, Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - Capacity: &ateapipb.WorkerResources{Actors: 1}, - Allocated: &ateapipb.WorkerResources{Actors: 1}, + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + Allocated: &ateapipb.WorkerResources{Actors: 1}, + DefaultRuntime: &ateapipb.SandboxRuntime{SandboxClass: "gvisor"}, }, }, } @@ -327,16 +329,21 @@ func TestTopWorkersRunner_Success(t *testing.T) { WorkerNamespace: "ate-demo-counter", WorkerPool: "counter", WorkerPod: "counter-worker-pool-7b9f8-x123", - SandboxClass: "gvisor", Labels: map[string]string{"ate.dev/worker-pool": "counter"}, - Status: &ateapipb.WorkerStatus{Capacity: &ateapipb.WorkerResources{Actors: 1}, Allocated: &ateapipb.WorkerResources{Actors: 1}}, + Status: &ateapipb.WorkerStatus{ + Capacity: &ateapipb.WorkerResources{Actors: 1}, + Allocated: &ateapipb.WorkerResources{Actors: 1}, + DefaultRuntime: &ateapipb.SandboxRuntime{SandboxClass: "gvisor"}, + }, }, { WorkerNamespace: "ate-demo-counter", WorkerPool: "counter", WorkerPod: "counter-worker-pool-7b9f8-y456", - SandboxClass: "microvm", Labels: map[string]string{"ate.dev/worker-pool": "counter"}, + Status: &ateapipb.WorkerStatus{ + DefaultRuntime: &ateapipb.SandboxRuntime{SandboxClass: "microvm"}, + }, }, } @@ -400,7 +407,9 @@ func TestTopWorkersRunner_FilterNamespace(t *testing.T) { WorkerNamespace: "ns-1", WorkerPool: "pool-1", WorkerPod: "pod-1", - SandboxClass: "gvisor", + Status: &ateapipb.WorkerStatus{ + DefaultRuntime: &ateapipb.SandboxRuntime{SandboxClass: "gvisor"}, + }, }, { WorkerNamespace: "ns-2", @@ -437,8 +446,11 @@ func TestTopWorkersRunner_FilterAtespace(t *testing.T) { WorkerNamespace: "ns-1", WorkerPool: "pool-1", WorkerPod: "pod-1", - SandboxClass: "microvm", - Status: &ateapipb.WorkerStatus{Capacity: &ateapipb.WorkerResources{Actors: 1}, Allocated: &ateapipb.WorkerResources{Actors: 1}}, + Status: &ateapipb.WorkerStatus{ + Capacity: &ateapipb.WorkerResources{Actors: 1}, + Allocated: &ateapipb.WorkerResources{Actors: 1}, + DefaultRuntime: &ateapipb.SandboxRuntime{SandboxClass: "microvm"}, + }, }, { Metadata: &ateapipb.ResourceMetadata{Name: "worker-2"}, @@ -480,8 +492,10 @@ func TestTopWorkersRunner_FilterSelector(t *testing.T) { WorkerNamespace: "ns-1", WorkerPool: "counter", WorkerPod: "pod-1", - SandboxClass: "gvisor", Labels: map[string]string{"ate.dev/worker-pool": "counter"}, + Status: &ateapipb.WorkerStatus{ + DefaultRuntime: &ateapipb.SandboxRuntime{SandboxClass: "gvisor"}, + }, }, { WorkerNamespace: "ns-1", diff --git a/cmd/kubectl-ate/internal/printer/printer_test.go b/cmd/kubectl-ate/internal/printer/printer_test.go index d3f6e9ab88..26494e717b 100644 --- a/cmd/kubectl-ate/internal/printer/printer_test.go +++ b/cmd/kubectl-ate/internal/printer/printer_test.go @@ -281,7 +281,6 @@ func TestPrintWorkersTo_Table(t *testing.T) { WorkerNamespace: "default", WorkerPool: "pool-1", WorkerPod: "pod-1", - SandboxClass: "gvisor", Status: &ateapipb.WorkerStatus{ State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 4, CpuMillicores: 2000, MemoryBytes: 4 << 30}, diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index 73cc50143c..782323761a 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -267,9 +267,9 @@ groups: brief: > The source has no class, or a class that is not in this list. The default is not gvisor. Thus a bad record stays visible. - ateapi reports it for a Worker that CreateWorker accepted with - no class. atelet reports it for a snapshot manifest with no - class. + ateapi reports it for a Worker that has not reported a + default_runtime class. atelet reports it for a snapshot + manifest with no class. - id: ate.sandbox.oom_kills stability: development type: int @@ -802,9 +802,8 @@ groups: Substrate can also lose the data that it did not write. Some moves into this state are not a loss of data. They are careful responses to a control plane problem. - ate.sandbox.class is unknown when ateapi could not read the class of the - worker: the worker record is already gone, or its assignment is already - clear. + ate.sandbox.class is unknown when the actor's worker assignment is already + clear or carries no sandbox runtime. annotations: substrate: emitted_by: [ateapi] diff --git a/internal/ateattr/ateattr_test.go b/internal/ateattr/ateattr_test.go index 6810a486ed..4720bc213b 100644 --- a/internal/ateattr/ateattr_test.go +++ b/internal/ateattr/ateattr_test.go @@ -467,8 +467,8 @@ func TestActorMetricAttributes(t *testing.T) { assertAttrs(t, got, want) }) - // releaseWorker gives an empty class when the worker record is gone. - // CreateWorker can also store one. Empty is not a permitted value. + // The class is empty when the actor's worker assignment is already clear or + // carries no sandbox runtime. Empty is not a permitted value. t.Run("empty sandbox class is normalized to unknown", func(t *testing.T) { got := toMap(ActorMetricAttributes(actor, "", OperationResume)) want := map[attribute.Key]any{ diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index 413d0e17ff..d77641e198 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -7265,13 +7265,6 @@ type Worker struct { // +k8s:customValidation # a SPIFFE ID // +k8s:immutable WorkerSpiffeId string `protobuf:"bytes,8,opt,name=worker_spiffe_id,json=workerSpiffeId,proto3" json:"worker_spiffe_id,omitempty"` - // sandbox_class mirrors the WorkerPool's default sandbox class - // - // +k8s:optional - // +k8s:maxLength=63 - // +k8s:customValidation # gvisor or microvm - // +k8s:immutable - SandboxClass string `protobuf:"bytes,9,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` // labels mirror the WorkerPool object's Kubernetes labels, which selectors // match against. // Kubernetes does not cap an object's label count, so the bound below is @@ -7281,7 +7274,7 @@ type Worker struct { // +k8s:maxProperties=64 // +k8s:eachKey=+k8s:format=k8s-label-key // +k8s:eachVal=+k8s:format=k8s-label-value - Labels map[string]string `protobuf:"bytes,10,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Labels map[string]string `protobuf:"bytes,9,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // epoch counts the runs of the worker pod's worker container: 1 for its first // run and one more for each restart. 0 means it has not been reported. // @@ -7294,14 +7287,14 @@ type Worker struct { // +k8s:minimum=0 // +k8s:update=NoUnset // +k8s:customValidation - Epoch int64 `protobuf:"varint,11,opt,name=epoch,proto3" json:"epoch,omitempty"` + Epoch int64 `protobuf:"varint,10,opt,name=epoch,proto3" json:"epoch,omitempty"` // Output-only server-managed state. Absent from Create/Update request // payloads; whatever a request carries here is ignored. A client moves // state only through DrainWorker, MarkWorkerAvailable and // MarkWorkerUnavailable, and the assignments are the scheduler's. // // +k8s:optional - Status *WorkerStatus `protobuf:"bytes,12,opt,name=status,proto3" json:"status,omitempty"` + Status *WorkerStatus `protobuf:"bytes,11,opt,name=status,proto3" json:"status,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -7392,13 +7385,6 @@ func (x *Worker) GetWorkerSpiffeId() string { return "" } -func (x *Worker) GetSandboxClass() string { - if x != nil { - return x.SandboxClass - } - return "" -} - func (x *Worker) GetLabels() map[string]string { if x != nil { return x.Labels @@ -7452,7 +7438,8 @@ type WorkerStatus struct { ObservedEpoch int64 `protobuf:"varint,4,opt,name=observed_epoch,json=observedEpoch,proto3" json:"observed_epoch,omitempty"` // Runtime that new and cold-booted actors start under, as reported by the // Worker at registration. Snapshots matching default_runtime can also - // warm-boot on this worker. + // warm-boot on this worker. Its sandbox_class is the Worker's class, fixed + // by the first RegisterWorker call. // // +k8s:optional DefaultRuntime *SandboxRuntime `protobuf:"bytes,5,opt,name=default_runtime,json=defaultRuntime,proto3" json:"default_runtime,omitempty"` @@ -7935,7 +7922,9 @@ type RegisterWorkerRequest struct { Capacity *WorkerResources `protobuf:"bytes,2,opt,name=capacity,proto3" json:"capacity,omitempty"` // Runtime that new and cold-booted actors start under. // Snapshots matching default_runtime can also warm-boot on this worker. - // Like capacity, this replaces what is recorded on the Worker. + // The first registration fixes the Worker's sandbox_class; later reports + // replace version and compatibility attributes, but a different + // sandbox_class is refused. // // +k8s:required DefaultRuntime *SandboxRuntime `protobuf:"bytes,3,opt,name=default_runtime,json=defaultRuntime,proto3" json:"default_runtime,omitempty"` @@ -9945,7 +9934,7 @@ const file_ateapi_proto_rawDesc = "" + "\aoptions\x18\x02 \x01(\v2\x13.ateapi.ListOptionsR\aoptions\"c\n" + "\x12ListActorsResponse\x12%\n" + "\x06actors\x18\x01 \x03(\v2\r.ateapi.ActorR\x06actors\x12&\n" + - "\x0fnext_page_token\x18\x02 \x01(\tR\rnextPageToken\"\x91\x04\n" + + "\x0fnext_page_token\x18\x02 \x01(\tR\rnextPageToken\"\xec\x03\n" + "\x06Worker\x124\n" + "\bmetadata\x18\x01 \x01(\v2\x18.ateapi.ResourceMetadataR\bmetadata\x12)\n" + "\x10worker_namespace\x18\x02 \x01(\tR\x0fworkerNamespace\x12\x1f\n" + @@ -9956,12 +9945,11 @@ const file_ateapi_proto_rawDesc = "" + "\x0eworker_pod_uid\x18\x05 \x01(\tR\fworkerPodUid\x12\x1b\n" + "\tnode_name\x18\x06 \x01(\tR\bnodeName\x12\x10\n" + "\x03ips\x18\a \x03(\tR\x03ips\x12(\n" + - "\x10worker_spiffe_id\x18\b \x01(\tR\x0eworkerSpiffeId\x12#\n" + - "\rsandbox_class\x18\t \x01(\tR\fsandboxClass\x122\n" + - "\x06labels\x18\n" + - " \x03(\v2\x1a.ateapi.Worker.LabelsEntryR\x06labels\x12\x14\n" + - "\x05epoch\x18\v \x01(\x03R\x05epoch\x12,\n" + - "\x06status\x18\f \x01(\v2\x14.ateapi.WorkerStatusR\x06status\x1a9\n" + + "\x10worker_spiffe_id\x18\b \x01(\tR\x0eworkerSpiffeId\x122\n" + + "\x06labels\x18\t \x03(\v2\x1a.ateapi.Worker.LabelsEntryR\x06labels\x12\x14\n" + + "\x05epoch\x18\n" + + " \x01(\x03R\x05epoch\x12,\n" + + "\x06status\x18\v \x01(\v2\x14.ateapi.WorkerStatusR\x06status\x1a9\n" + "\vLabelsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01:\x0f\x82\xb5\x18\v\x18\x02\"\aWorkers\"\xd6\x02\n" + diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index 9445beeec0..284bc084d1 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -2314,14 +2314,6 @@ message Worker { // +k8s:immutable string worker_spiffe_id = 8; - // sandbox_class mirrors the WorkerPool's default sandbox class - // - // +k8s:optional - // +k8s:maxLength=63 - // +k8s:customValidation # gvisor or microvm - // +k8s:immutable - string sandbox_class = 9; - // labels mirror the WorkerPool object's Kubernetes labels, which selectors // match against. // Kubernetes does not cap an object's label count, so the bound below is @@ -2331,7 +2323,7 @@ message Worker { // +k8s:maxProperties=64 // +k8s:eachKey=+k8s:format=k8s-label-key // +k8s:eachVal=+k8s:format=k8s-label-value - map labels = 10; + map labels = 9; // epoch counts the runs of the worker pod's worker container: 1 for its first // run and one more for each restart. 0 means it has not been reported. @@ -2345,7 +2337,7 @@ message Worker { // +k8s:minimum=0 // +k8s:update=NoUnset // +k8s:customValidation - int64 epoch = 11; + int64 epoch = 10; // Output-only server-managed state. Absent from Create/Update request // payloads; whatever a request carries here is ignored. A client moves @@ -2353,7 +2345,7 @@ message Worker { // MarkWorkerUnavailable, and the assignments are the scheduler's. // // +k8s:optional - WorkerStatus status = 12; + WorkerStatus status = 11; } enum WorkerState { @@ -2404,7 +2396,8 @@ message WorkerStatus { // Runtime that new and cold-booted actors start under, as reported by the // Worker at registration. Snapshots matching default_runtime can also - // warm-boot on this worker. + // warm-boot on this worker. Its sandbox_class is the Worker's class, fixed + // by the first RegisterWorker call. // // +k8s:optional SandboxRuntime default_runtime = 5; @@ -2658,7 +2651,9 @@ message RegisterWorkerRequest { // Runtime that new and cold-booted actors start under. // Snapshots matching default_runtime can also warm-boot on this worker. - // Like capacity, this replaces what is recorded on the Worker. + // The first registration fixes the Worker's sandbox_class; later reports + // replace version and compatibility attributes, but a different + // sandbox_class is refused. // // +k8s:required SandboxRuntime default_runtime = 3; From 1d999251314f37414c40049285d08516d7d58976 Mon Sep 17 00:00:00 2001 From: Lior Lieberman Date: Wed, 21 Oct 2026 00:16:44 +0000 Subject: [PATCH 30/63] atenet-egress: remove the unused by-address dial path (#2892) The egress handler only ever answers `dial=name`, so the `dial=address` routes, the three ORIGINAL_DST clusters behind them and the `original_dst_address` attributes in atenet-egress.yaml could never match. This removes them plus the extproc constants only they used, and fixes comments and READMEs that still describe removed behavior (ActorIdentity extension, one pool per tunnel, MITM-only injection, egress-only dataplane image). `OriginalDstFilterStateKey` stays since the ingress upstream CONNECT sets it. The passthrough manifest test now rejects any ORIGINAL_DST cluster, and the edited manifest passes `envoy --mode validate` in the dataplane image (11 clusters down to 8). Split out of #2166. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- cmd/atenet/internal/router/README.md | 48 +- .../internal/router/egressmanifest_test.go | 93 ++- .../router/egresspolicymanifest_test.go | 546 +++++++----------- .../router/envoyadmin_manifest_test.go | 2 +- .../internal/router/extproc/attributes.go | 33 +- .../internal/router/extproc/dispatch.go | 5 +- .../internal/router/extproc/metadata_test.go | 5 +- cmd/dataplane/envoy/README.md | 4 +- .../dynamic-modules/egress-policy/README.md | 18 +- manifests/ate-install/atenet-egress.yaml | 206 ++----- 10 files changed, 357 insertions(+), 603 deletions(-) diff --git a/cmd/atenet/internal/router/README.md b/cmd/atenet/internal/router/README.md index 77be87f89b..9200ad33f6 100644 --- a/cmd/atenet/internal/router/README.md +++ b/cmd/atenet/internal/router/README.md @@ -25,12 +25,12 @@ Router has several responsibilities: dataplane container's `preStop` hook. See `drain.go` and `envoydrain.go`. * Authenticates actor identity on egress: on every CONNECT, the egress gateway's ext_proc handler re-verifies the actor's client certificate against - the actor-identity CA, reads the `ActorIdentity` X.509 extension out of it, - and checks the certified UID against the ATE API. + the actor-identity CA, reads the actor from its SPIFFE URI SAN, and checks + with the ATE API that the actor exists and is running. * Authorizes egress against the actor's `EgressPolicy`: every request the - gateway can read is decided on its `Host` and the address the actor dialed, - rules in order, first match wins; what it cannot read is decided by address - at the CONNECT. An actor with no policy gets no tunnel. See + gateway can read is decided on its `Host` and the port the actor dialed, + the most specific rule winning; TLS it does not terminate is decided by SNI + at the ClientHello. An actor with no policy gets no tunnel. See [egress legs](#egress-legs). * Serves arbitrary-port ingress: a client reaches a port on the actor other than its default (80) by sending an HTTP CONNECT to @@ -95,13 +95,12 @@ the request legs match `ports` against it; a dataplane that does not share it gets no port enforcement. The CONNECT leg decides per connection: it refuses an actor without a policy -with rules, and for the rest computes hash of the policy SNI's rules for the -destination port, stores policy in the thread local store using hash as the -key and hands the inner listener the port the actor dialed, as -`envoy.upstream.dynamic_port` and the hash of the policy' as -`dev.ate.policy.egress.hash`. At the ClientHello the egress-policy module -reads the hash, and fetches egress policy from the thrad local store, -applies SNI rules and picks the chain: `egress_tls_mitm` under an `https` +with rules. For the rest it hashes the policy's SNI rules for the destination +port, stores them in the thread-local store under that hash, and hands the +inner listener the port the actor dialed, as `envoy.upstream.dynamic_port`, +and the hash, as `dev.ate.policy.egress.hash`. At the ClientHello the +egress-policy module reads the hash, fetches the rules from the thread-local +store, and picks the chain: `egress_tls_mitm` under an `https` rule, `egress_passthrough` under a `tls_passthrough` rule, and `denied`, which matches no chain and closes the connection, when neither covers the SNI. The passthrough chain resolves the SNI and relays the bytes to it on the dialed @@ -111,9 +110,10 @@ port; the address the actor dialed plays no part (see Identity on the request legs is `dev.ate.actor.identity`, the actor's SPIFFE ID that the outer chain set from the verified peer certificate and shares with the inner listener. Envoy builds every inner connection from the filter state -of the tunnel that created its pool, so `mitm_internal` keeps one pool per -downstream connection, which on the HTTP/1 egress listener is one per tunnel: -no tunnel inherits another's identity, authority or rules. Nothing inside the +of the tunnel that created its pool, so `mitm_internal` keys its pools on the +identity, the CONNECT authority and the policy hash: tunnels share a pool only +when all three match, and no tunnel inherits another's identity, authority or +rules. Nothing inside the tunnel can write any of this; a callout without an identity is refused. Policies are read through a per-actor cache (`--egress-policy-cache-ttl`, 10s @@ -122,24 +122,30 @@ create, update or delete is visible to new requests within one TTL, and a deleted policy becomes a deny. Every policy denial answers a fixed `egress denied` body; the reason is in the sidecar's log. -Credential injection (`replace_headers`) runs on the MITM leg: a header the -request carries is replaced with the credential from the provider, and a +Credential injection (`replace_headers`) runs on both request legs: a header +the request carries is replaced with the credential from the provider, and a request without it is forwarded unchanged. See `docs/egress-credential-injection.md`. ## adding a dataplane attribute The filter-state objects and request attributes a proxy carries alongside -a request are declared once, in `extproc/attributes.go`. +a request are declared once, in `extproc/attributes.go`. The exception is the +keys only the router's generated Envoy config reads, which stay in `xds.go`. | key | direction | purpose | | --- | --- | --- | -| `dev.ate.actor.name` | ingress | carries the actor name across CONNECT re-entry | -| `dev.ate.actor.atespace` | ingress | carries the atespace across CONNECT re-entry | +| `dev.ate.target.actor` | ingress | carries the actor routing target across CONNECT re-entry | | `dev.ate.connect.authority` | ingress, egress | carries the outer CONNECT authority across re-entry: target-port selection on ingress, the dialed port the request legs match `ports` against on egress | | `dev.ate.actor.identity` | egress | carries the authenticated actor identity to the policy ext_proc, the logs and additional ext_proc services | -| `dev.ate.egress:dial` | egress | dynamic metadata: a request leg's answer, `name` or `address`, which picks the route | +| `dev.ate.policy.egress` | egress | dynamic metadata: the CONNECT leg's SNI rules for the dialed port, which the egress-policy module stores by hash | +| `dev.ate.policy.egress.hash` | egress | filter state: the hash the inner listener looks those SNI rules up by | +| `dev.ate.egress:dialed_port` | egress | dynamic metadata: the CONNECT leg's answer, the port the passthrough chain dials the resolved SNI on | +| `dev.ate.egress:dial` | egress | dynamic metadata: a request leg's answer, `name`, which picks the route | +| `dev.ate.egress:host` | egress | dynamic metadata: the name the request was decided on, which the forward proxy dials on the port the actor dialed | +| `dev.ate.egress.filter_chain` | egress | filter state: the egress-policy module's verdict, which selects the inner filter chain | | `dev.ate.extproc.direction` | egress | selects the egress handler for dataplanes without Envoy filter chains | +| `dev.ate.tunnel.{key,worker,actor,port}` | ingress | filter state: the routing decision carried into `actor_connect` (declared in `xds.go`) | ### name it diff --git a/cmd/atenet/internal/router/egressmanifest_test.go b/cmd/atenet/internal/router/egressmanifest_test.go index b55796e5ae..c12b34bd8a 100644 --- a/cmd/atenet/internal/router/egressmanifest_test.go +++ b/cmd/atenet/internal/router/egressmanifest_test.go @@ -15,9 +15,7 @@ package router import ( - "os" "slices" - "strings" "testing" "time" @@ -25,41 +23,36 @@ import ( "sigs.k8s.io/yaml" ) -// egressManifests is the envoy egress gateway ate-setup installs, which -// terminates and re-originates the tunneled TLS. -var egressManifests = []string{ - "../../../../manifests/ate-install/atenet-egress.yaml", -} +// egressManifest is the envoy egress gateway ate-setup installs, which +// terminates and re-originates the tunneled TLS and runs the egress-policy +// module. +const egressManifest = "../../../../manifests/ate-install/atenet-egress.yaml" // TestEgressManifestsDisableTheConnectTimeout is the static-config half of // TestBuildConnectRoutes_DisablesTimeout: Envoy applies a route's timeout to a // CONNECT tunnel's whole lifetime rather than to its headers, so a route left // at Envoy's 15s default caps how long any actor's outbound connection may // exist -- streaming responses, long downloads and SSH sessions all die -// mid-stream at 15s. These manifests are what the gateway actually runs, and -// nothing else in the tree notices when one of them loses the line. +// mid-stream at 15s. This manifest is what the gateway actually runs, and +// nothing else in the tree notices when it loses the line. func TestEgressManifestsDisableTheConnectTimeout(t *testing.T) { - for _, path := range egressManifests { - t.Run(path, func(t *testing.T) { - routes := connectRoutes(t, envoyConfig(t, path)) - if len(routes) == 0 { - t.Fatal("found no connect_matcher route; the manifest changed shape and this test is checking nothing") - } - for _, r := range routes { - if r.Route.Timeout == nil { - t.Errorf("connect_matcher route to cluster %q sets no timeout, so it falls back to Envoy's 15s default and cuts every tunnel after 15s", r.Route.Cluster) - continue - } - d, err := time.ParseDuration(*r.Route.Timeout) - if err != nil { - t.Errorf("connect_matcher route to cluster %q has timeout %q, which is not a duration: %v", r.Route.Cluster, *r.Route.Timeout, err) - continue - } - if d != 0 { - t.Errorf("connect_matcher route to cluster %q has timeout %s; it must be 0 to disable it", r.Route.Cluster, d) - } - } - }) + routes := connectRoutes(t, envoyConfig(t)) + if len(routes) == 0 { + t.Fatal("found no connect_matcher route; the manifest changed shape and this test is checking nothing") + } + for _, r := range routes { + if r.Route.Timeout == nil { + t.Errorf("connect_matcher route to cluster %q sets no timeout, so it falls back to Envoy's 15s default and cuts every tunnel after 15s", r.Route.Cluster) + continue + } + d, err := time.ParseDuration(*r.Route.Timeout) + if err != nil { + t.Errorf("connect_matcher route to cluster %q has timeout %q, which is not a duration: %v", r.Route.Cluster, *r.Route.Timeout, err) + continue + } + if d != 0 { + t.Errorf("connect_matcher route to cluster %q has timeout %s; it must be 0 to disable it", r.Route.Cluster, d) + } } } @@ -117,41 +110,25 @@ func connectRoutes(t *testing.T, raw string) []envoyRoute { return out } -// envoyConfig is the envoy.yaml the atenet-egress ConfigMap in path ships. -func envoyConfig(t *testing.T, path string) string { +// envoyConfig is the envoy.yaml the atenet-egress ConfigMap in egressManifest +// ships. +func envoyConfig(t *testing.T) string { t.Helper() - return envoyConfigFrom(t, path, "atenet-egress") + return envoyConfigFrom(t, egressManifest, "atenet-egress") } // envoyConfigFrom is the envoy.yaml the ConfigMap named configMap in path ships. func envoyConfigFrom(t *testing.T, path, configMap string) string { t.Helper() - manifest, err := os.ReadFile(path) - if err != nil { - t.Fatalf("reading %s: %v", path, err) + var cm struct { + Data map[string]string `json:"data"` } - for _, doc := range strings.Split(string(manifest), "\n---\n") { - var obj struct { - Kind string `json:"kind"` - Metadata struct { - Name string `json:"name"` - } `json:"metadata"` - Data map[string]string `json:"data"` - } - if err := yaml.Unmarshal([]byte(doc), &obj); err != nil { - t.Fatalf("parsing a document of %s: %v", path, err) - } - if obj.Kind != "ConfigMap" || obj.Metadata.Name != configMap { - continue - } - envoyYaml, ok := obj.Data["envoy.yaml"] - if !ok { - t.Fatalf("the %s ConfigMap in %s has no envoy.yaml key", configMap, path) - } - return envoyYaml + manifestObject(t, path, "ConfigMap", configMap, &cm) + envoyYaml, ok := cm.Data["envoy.yaml"] + if !ok { + t.Fatalf("the %s ConfigMap in %s has no envoy.yaml key", configMap, path) } - t.Fatalf("%s has no ConfigMap named %s", path, configMap) - return "" + return envoyYaml } // wantSampledLogFilter is the filter on the access logs of the per-request legs. @@ -186,7 +163,7 @@ func TestEgressManifestsSampleThePerRequestAccessLogs(t *testing.T) { if err := yaml.Unmarshal([]byte(wantSampledLogFilter), &want); err != nil { t.Fatalf("parsing wantSampledLogFilter: %v", err) } - logs := chainAccessLogsByLeg(bootstrapTree(t, egressManifest)) + logs := chainAccessLogsByLeg(bootstrapTree(t)) sampledLegs := []string{"mitm", "cleartext"} for _, leg := range sampledLegs { t.Run(leg, func(t *testing.T) { diff --git a/cmd/atenet/internal/router/egresspolicymanifest_test.go b/cmd/atenet/internal/router/egresspolicymanifest_test.go index 4aae9928fa..e62ce2e185 100644 --- a/cmd/atenet/internal/router/egresspolicymanifest_test.go +++ b/cmd/atenet/internal/router/egresspolicymanifest_test.go @@ -24,7 +24,7 @@ import ( "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" ) -// These tests pin the egress manifests to the Go constants the handler +// These tests pin the egress manifest to the Go constants the handler // dispatches on and answers with. A chain renamed on one side only fails // closed at runtime, and nothing else in the tree would catch it. @@ -32,9 +32,7 @@ const ( extProcFilter = "envoy.filters.http.ext_proc" setFilterStateFilter = "envoy.filters.http.set_filter_state" extProcServerCluster = "ext_proc_server" - passthroughCluster = "egress_tcp_passthrough" passthroughForwardProxyCluster = "egress_forward_proxy_passthrough" - originalDstKey = "envoy.network.transport_socket.original_dst_address" dfpClusterType = "envoy.clusters.dynamic_forward_proxy" ) @@ -43,13 +41,14 @@ var requestLegs = []string{extproc.EgressCleartextFilterChainName, extproc.Egres type node = map[string]any -// bootstrapTree parses the envoy.yaml of the atenet-egress ConfigMap in path -// as a generic tree; the assertions read a few leaves scattered across it. -func bootstrapTree(t *testing.T, path string) node { +// bootstrapTree parses the envoy.yaml of the atenet-egress ConfigMap in +// egressManifest as a generic tree; the assertions read a few leaves scattered +// across it. +func bootstrapTree(t *testing.T) node { t.Helper() var tree node - if err := yaml.Unmarshal([]byte(envoyConfig(t, path)), &tree); err != nil { - t.Fatalf("parsing envoy.yaml of %s: %v", path, err) + if err := yaml.Unmarshal([]byte(envoyConfig(t)), &tree); err != nil { + t.Fatalf("parsing envoy.yaml of %s: %v", egressManifest, err) } return tree } @@ -163,94 +162,69 @@ func outerChain(t *testing.T, tree node) node { } // Every chain that calls ext_proc must be one the handler knows as an egress -// leg, and each manifest must have exactly the legs its topology implies. +// leg, and the manifest must have exactly the legs its topology implies. func TestEgressManifestsNameEveryExtProcChain(t *testing.T) { - want := map[string][]string{ - egressManifests[0]: {extproc.EgressFilterChainName, extproc.EgressTLSMITMFilterChainName, extproc.EgressCleartextFilterChainName}, - } - for _, path := range egressManifests { - t.Run(path, func(t *testing.T) { - var got []string - for _, lc := range allChains(bootstrapTree(t, path)) { - h := hcm(lc.chain) - if h == nil || filterIndex(list(h, "http_filters"), extProcFilter) < 0 { - continue - } - name := str(lc.chain, "name") - if !extproc.IsEgressFilterChain(name) { - t.Errorf("listener %q has a filter chain %q that calls ext_proc but is not a chain the egress handler serves; its callouts would be refused", str(lc.listener, "name"), name) - } - got = append(got, name) - } - slices.Sort(got) - expected := slices.Clone(want[path]) - slices.Sort(expected) - if !slices.Equal(got, expected) { - t.Errorf("ext_proc-calling filter chains = %v, want %v", got, expected) - } - }) + want := []string{extproc.EgressFilterChainName, extproc.EgressTLSMITMFilterChainName, extproc.EgressCleartextFilterChainName} + var got []string + for _, lc := range allChains(bootstrapTree(t)) { + h := hcm(lc.chain) + if h == nil || filterIndex(list(h, "http_filters"), extProcFilter) < 0 { + continue + } + name := str(lc.chain, "name") + if !extproc.IsEgressFilterChain(name) { + t.Errorf("listener %q has a filter chain %q that calls ext_proc but is not a chain the egress handler serves; its callouts would be refused", str(lc.listener, "name"), name) + } + got = append(got, name) + } + slices.Sort(got) + slices.Sort(want) + if !slices.Equal(got, want) { + t.Errorf("ext_proc-calling filter chains = %v, want %v", got, want) } } func TestEgressManifestsDisableWebSocketUpgrades(t *testing.T) { - want := map[string][]string{ - egressManifests[0]: {extproc.EgressCleartextFilterChainName, extproc.EgressTLSMITMFilterChainName}, - } - for _, path := range egressManifests { - t.Run(path, func(t *testing.T) { - tree := bootstrapTree(t, path) - chains := map[string]node{} - for _, lc := range allChains(tree) { - name := str(lc.chain, "name") - if slices.Contains(requestLegs, name) { - chains[name] = lc.chain - } + chains := list(mitmListener(t, bootstrapTree(t)), "filter_chains") + for _, name := range requestLegs { + chain := byName(chains, name) + if chain == nil { + t.Errorf("expected request-leg chain %q is absent", name) + continue + } + if !websocketDisabled(hcm(chain)) { + t.Errorf("chain %q must explicitly disable websocket upgrades", name) + } + var routes []node + for _, virtualHost := range list(child(hcm(chain), "route_config"), "virtual_hosts") { + routes = append(routes, list(virtualHost, "routes")...) + } + if len(routes) == 0 { + t.Errorf("chain %q has no routes", name) + } + for i, route := range routes { + action := child(route, "route") + if action == nil { + t.Errorf("chain %q route %d has no route action", name, i) + continue } - for _, name := range want[path] { - chain, present := chains[name] - if !present { - t.Errorf("expected request-leg chain %q is absent", name) - continue - } - var disabled bool - for _, upgrade := range list(hcm(chain), "upgrade_configs") { - if str(upgrade, "upgrade_type") == "websocket" { - enabled, ok := upgrade["enabled"].(bool) - disabled = ok && !enabled - break - } - } - if !disabled { - t.Errorf("chain %q must explicitly disable websocket upgrades", name) - } - var routes []node - for _, virtualHost := range list(child(hcm(chain), "route_config"), "virtual_hosts") { - routes = append(routes, list(virtualHost, "routes")...) - } - if len(routes) == 0 { - t.Errorf("chain %q has no routes", name) - } - for i, route := range routes { - action := child(route, "route") - if action == nil { - t.Errorf("chain %q route %d has no route action", name, i) - continue - } - disabled := false - for _, upgrade := range list(action, "upgrade_configs") { - if str(upgrade, "upgrade_type") == "websocket" { - enabled, ok := upgrade["enabled"].(bool) - disabled = ok && !enabled - break - } - } - if !disabled { - t.Errorf("chain %q route %d must explicitly disable websocket upgrades", name, i) - } - } + if !websocketDisabled(action) { + t.Errorf("chain %q route %d must explicitly disable websocket upgrades", name, i) } - }) + } + } +} + +// websocketDisabled reports whether n's upgrade_configs turn websocket off +// explicitly. +func websocketDisabled(n node) bool { + for _, upgrade := range list(n, "upgrade_configs") { + if str(upgrade, "upgrade_type") == "websocket" { + enabled, ok := upgrade["enabled"].(bool) + return ok && !enabled + } } + return false } // extProcOf returns the ext_proc filter config of a chain's HCM and its index @@ -270,186 +244,131 @@ func extProcOf(chain node) (node, int, []node) { func TestEgressManifestsExtProcFilters(t *testing.T) { required := map[string][]string{ extproc.EgressFilterChainName: {extproc.FilterChainNameAttribute}, - extproc.EgressCleartextFilterChainName: {extproc.FilterChainNameAttribute, extproc.ActorIdentityFilterStateAttribute, extproc.ConnectAuthorityFilterStateAttribute, extproc.OriginalDstIPAttribute, extproc.OriginalDstPortAttribute}, - extproc.EgressTLSMITMFilterChainName: {extproc.FilterChainNameAttribute, extproc.ActorIdentityFilterStateAttribute, extproc.ConnectAuthorityFilterStateAttribute, extproc.OriginalDstIPAttribute, extproc.OriginalDstPortAttribute}, - } - for _, path := range egressManifests { - t.Run(path, func(t *testing.T) { - prefixes := map[string]string{} // stat_prefix -> chain - for _, lc := range allChains(bootstrapTree(t, path)) { - name := str(lc.chain, "name") - attrs, isLeg := required[name] - if !isLeg { - continue - } - cfg, i, filters := extProcOf(lc.chain) - if cfg == nil { - t.Errorf("chain %q has no ext_proc filter", name) - continue - } - if got := str(child(child(cfg, "grpc_service"), "envoy_grpc"), "cluster_name"); got != extProcServerCluster { - t.Errorf("chain %q ext_proc calls cluster %q, want %q", name, got, extProcServerCluster) - } - if allow, _ := cfg["failure_mode_allow"].(bool); allow { - t.Errorf("chain %q ext_proc has failure_mode_allow: true; a sidecar outage would let every request through", name) - } - if prefix := str(cfg, "stat_prefix"); prefix == "" { - t.Errorf("chain %q ext_proc has no stat_prefix; its failures would be indistinguishable from the other legs'", name) - } else if other, seen := prefixes[prefix]; seen { - t.Errorf("chain %q ext_proc reuses stat_prefix %q of chain %q; their stats would be summed", name, prefix, other) - } else { - prefixes[prefix] = name - } - got := strs(cfg, "request_attributes") - for _, attr := range attrs { - if !slices.Contains(got, attr) { - t.Errorf("chain %q ext_proc does not request %q; the handler would read it as absent and deny", name, attr) - } - } - // The decision has to come before the request is routed, and - // before dynamic_forward_proxy resolves the Host it names. - for _, later := range []string{"envoy.filters.http.dynamic_forward_proxy", "envoy.filters.http.router"} { - if j := filterIndex(filters, later); j >= 0 && j < i { - t.Errorf("chain %q runs %s before ext_proc", name, later) - } - } - // No leg rewrites routing: the name that was policed is the - // name that is dialed. - rules := child(cfg, "mutation_rules") - if isError, _ := rules["disallow_is_error"].(bool); !isError { - t.Errorf("chain %q ext_proc does not set mutation_rules.disallow_is_error", name) - } - routing, _ := rules["allow_all_routing"].(bool) - system, _ := rules["disallow_system"].(bool) - if !system || routing { - t.Errorf("chain %q ext_proc may rewrite routing headers (disallow_system=%v allow_all_routing=%v)", name, system, routing) - } - // Every leg answers with metadata in the egress namespace, the - // CONNECT leg's passthrough destination or a request leg's - // dial, and Envoy drops metadata from a namespace the filter - // does not admit. - admitted := strs(child(child(cfg, "metadata_options"), "receiving_namespaces"), "untyped") - if !slices.Contains(admitted, extproc.EgressMetadataNamespace) { - t.Errorf("chain %q ext_proc does not admit dynamic metadata in %q; its answer would be dropped", name, extproc.EgressMetadataNamespace) - } - } - }) - } -} - -// Envoy buckets a listener's filter chains by transport protocol and never -// falls back out of a populated bucket. egress_cleartext claims raw_buffer -// with HTTP application protocols alone, so a chain that matches nothing is -// unreachable and every opaque or unclassified connection is closed as -// no_filter_chain_match, allowed or not. -// -// The inner listener's filters produce exactly two transport protocols: tls -// from tls_inspector, and raw_buffer for everything else, sniff timeout -// included. Each needs a chain with no application_protocols as its catch-all. -// Every ORIGINAL_DST cluster, the by-address routes' included, dials the -// filter state alone. A metadata_key, use_http_header or port_override would -// give a request a way to name an address other than the one the CONNECT leg -// allowed and the request legs policed. -func TestEgressManifestsOriginalDstClustersDialTheFilterStateAlone(t *testing.T) { - for _, path := range egressManifests { - t.Run(path, func(t *testing.T) { - tree := bootstrapTree(t, path) - found := 0 - for _, cluster := range clusters(tree) { - if str(cluster, "type") != "ORIGINAL_DST" { - continue - } - found++ - if lb := child(cluster, "original_dst_lb_config"); len(lb) != 0 { - t.Errorf("cluster %q has original_dst_lb_config %v; the address must come from the filter state alone", str(cluster, "name"), lb) - } + extproc.EgressCleartextFilterChainName: {extproc.FilterChainNameAttribute, extproc.ActorIdentityFilterStateAttribute, extproc.ConnectAuthorityFilterStateAttribute}, + extproc.EgressTLSMITMFilterChainName: {extproc.FilterChainNameAttribute, extproc.ActorIdentityFilterStateAttribute, extproc.ConnectAuthorityFilterStateAttribute, extproc.RequestedServerNameAttribute}, + } + prefixes := map[string]string{} // stat_prefix -> chain + for _, lc := range allChains(bootstrapTree(t)) { + name := str(lc.chain, "name") + attrs, isLeg := required[name] + if !isLeg { + continue + } + cfg, i, filters := extProcOf(lc.chain) + if cfg == nil { + t.Errorf("chain %q has no ext_proc filter", name) + continue + } + if got := str(child(child(cfg, "grpc_service"), "envoy_grpc"), "cluster_name"); got != extProcServerCluster { + t.Errorf("chain %q ext_proc calls cluster %q, want %q", name, got, extProcServerCluster) + } + if allow, _ := cfg["failure_mode_allow"].(bool); allow { + t.Errorf("chain %q ext_proc has failure_mode_allow: true; a sidecar outage would let every request through", name) + } + if prefix := str(cfg, "stat_prefix"); prefix == "" { + t.Errorf("chain %q ext_proc has no stat_prefix; its failures would be indistinguishable from the other legs'", name) + } else if other, seen := prefixes[prefix]; seen { + t.Errorf("chain %q ext_proc reuses stat_prefix %q of chain %q; their stats would be summed", name, prefix, other) + } else { + prefixes[prefix] = name + } + got := strs(cfg, "request_attributes") + for _, attr := range attrs { + if !slices.Contains(got, attr) { + t.Errorf("chain %q ext_proc does not request %q; the handler would read it as absent and deny", name, attr) } - if found < 2 { - t.Errorf("found %d ORIGINAL_DST clusters, want at least the by-address ones", found) + } + // The decision has to come before the request is routed, and + // before dynamic_forward_proxy resolves the Host it names. + for _, later := range []string{"envoy.filters.http.dynamic_forward_proxy", "envoy.filters.http.router"} { + if j := filterIndex(filters, later); j >= 0 && j < i { + t.Errorf("chain %q runs %s before ext_proc", name, later) } - }) + } + // No leg rewrites routing: the name that was policed is the + // name that is dialed. + rules := child(cfg, "mutation_rules") + if isError, _ := rules["disallow_is_error"].(bool); !isError { + t.Errorf("chain %q ext_proc does not set mutation_rules.disallow_is_error", name) + } + routing, _ := rules["allow_all_routing"].(bool) + system, _ := rules["disallow_system"].(bool) + if !system || routing { + t.Errorf("chain %q ext_proc may rewrite routing headers (disallow_system=%v allow_all_routing=%v)", name, system, routing) + } + // Every leg answers with metadata in the egress namespace, the + // CONNECT leg's dialed port or a request leg's dial, and Envoy + // drops metadata from a namespace the filter does not admit. + admitted := strs(child(child(cfg, "metadata_options"), "receiving_namespaces"), "untyped") + if !slices.Contains(admitted, extproc.EgressMetadataNamespace) { + t.Errorf("chain %q ext_proc does not admit dynamic metadata in %q; its answer would be dropped", name, extproc.EgressMetadataNamespace) + } } } // The identity and the dialed authority cross the inner hop as filter state // shared with the upstream; nothing else carries them. func TestEgressManifestsShareIdentityWithTheInnerListener(t *testing.T) { - for _, path := range egressManifests { - t.Run(path, func(t *testing.T) { - var identity, authority node - for _, f := range list(hcm(outerChain(t, bootstrapTree(t, path))), "http_filters") { - if str(f, "name") != setFilterStateFilter { - continue - } - for _, v := range list(child(f, "typed_config"), "on_request_headers") { - switch str(v, "object_key") { - case extproc.ActorIdentityFilterStateKey: - identity = v - case extproc.ConnectAuthorityFilterStateKey: - authority = v - } - } - } - if identity == nil { - t.Fatalf("the egress chain never sets %s", extproc.ActorIdentityFilterStateKey) - } - if got := str(identity, "shared_with_upstream"); got == "" { - t.Errorf("%s is not shared with upstream; the inner legs would see no actor", extproc.ActorIdentityFilterStateKey) - } - // The dialed port rides along the same way; without it the request - // legs cannot enforce a rule's ports. - if authority == nil { - t.Fatalf("the egress chain never sets %s", extproc.ConnectAuthorityFilterStateKey) - } - if got := str(authority, "shared_with_upstream"); got == "" { - t.Errorf("%s is not shared with upstream; the inner legs would see no dialed port", extproc.ConnectAuthorityFilterStateKey) + var identity, authority node + for _, f := range list(hcm(outerChain(t, bootstrapTree(t))), "http_filters") { + if str(f, "name") != setFilterStateFilter { + continue + } + for _, v := range list(child(f, "typed_config"), "on_request_headers") { + switch str(v, "object_key") { + case extproc.ActorIdentityFilterStateKey: + identity = v + case extproc.ConnectAuthorityFilterStateKey: + authority = v } - }) + } + } + if identity == nil { + t.Fatalf("the egress chain never sets %s", extproc.ActorIdentityFilterStateKey) + } + if got := str(identity, "shared_with_upstream"); got == "" { + t.Errorf("%s is not shared with upstream; the inner legs would see no actor", extproc.ActorIdentityFilterStateKey) + } + // The dialed port rides along the same way; without it the request + // legs cannot enforce a rule's ports. + if authority == nil { + t.Fatalf("the egress chain never sets %s", extproc.ConnectAuthorityFilterStateKey) + } + if got := str(authority, "shared_with_upstream"); got == "" { + t.Errorf("%s is not shared with upstream; the inner legs would see no dialed port", extproc.ConnectAuthorityFilterStateKey) } } // http_inspector steers HTTP/1.0 onto the cleartext chain; the codec has to // accept it there, or those requests fail before any filter runs, unlogged. func TestEgressManifestsCleartextAcceptsHTTP10(t *testing.T) { - for _, path := range egressManifests { - t.Run(path, func(t *testing.T) { - for _, lc := range allChains(bootstrapTree(t, path)) { - if str(lc.chain, "name") != extproc.EgressCleartextFilterChainName { - continue - } - if ok, _ := child(hcm(lc.chain), "http_protocol_options")["accept_http_10"].(bool); !ok { - t.Errorf("%s does not accept HTTP/1.0", extproc.EgressCleartextFilterChainName) - } - } - }) + for _, lc := range allChains(bootstrapTree(t)) { + if str(lc.chain, "name") != extproc.EgressCleartextFilterChainName { + continue + } + if ok, _ := child(hcm(lc.chain), "http_protocol_options")["accept_http_10"].(bool); !ok { + t.Errorf("%s does not accept HTTP/1.0", extproc.EgressCleartextFilterChainName) + } } } -// One pool per tunnel: internal_upstream copies the filter state of the tunnel -// that created the pool, not of the tunnel asking for a connection, and a -// string object is not part of the pool key. A pool per downstream connection -// is a pool per tunnel only while that connection carries one CONNECT. A -// connection its tunnel left behind keeps its pool alive until the idle +// A connection no tunnel holds keeps its inner pool alive until the idle // timeout, which must not be disabled. -func TestEgressManifestsGiveEachTunnelItsOwnInnerPool(t *testing.T) { - for _, path := range egressManifests { - t.Run(path, func(t *testing.T) { - tree := bootstrapTree(t, path) - internal := 0 - for _, cluster := range clusters(tree) { - if !strings.Contains(mustJSON(t, cluster), `"server_listener_name"`) { - continue - } - internal++ - tcp := child(child(cluster, "typed_extension_protocol_options"), "envoy.extensions.upstreams.tcp.v3.TcpProtocolOptions") - if got := str(tcp, "idle_timeout"); got == "" || strings.Trim(got, "0.s") == "" { - t.Errorf("internal cluster %q idle_timeout is %q; want a non-zero timeout so a pool its tunnel left behind is freed", str(cluster, "name"), got) - } - } - if internal == 0 { - t.Fatal("no cluster targets an internal listener") - } - }) +func TestEgressManifestsTimeOutIdleInnerConnections(t *testing.T) { + tree := bootstrapTree(t) + internal := 0 + for _, cluster := range clusters(tree) { + if !strings.Contains(mustJSON(t, cluster), `"server_listener_name"`) { + continue + } + internal++ + tcp := child(child(cluster, "typed_extension_protocol_options"), "envoy.extensions.upstreams.tcp.v3.TcpProtocolOptions") + if got := str(tcp, "idle_timeout"); got == "" || strings.Trim(got, "0.s") == "" { + t.Errorf("internal cluster %q idle_timeout is %q; want a non-zero timeout so a pool its tunnel left behind is freed", str(cluster, "name"), got) + } + } + if internal == 0 { + t.Fatal("no cluster targets an internal listener") } } @@ -501,68 +420,51 @@ func autoSNIAndSAN(cluster node) bool { return false } -// A request leg's answer picks its route: one route per dial and no default, -// so a request the sidecar did not answer for has no route. dial=name goes to -// a dynamic forward proxy cluster, and only with the answered name it dials; -// dial=address to an ORIGINAL_DST cluster fed by the same filter state as the -// passthrough chains. On the MITM leg both re-originate TLS and verify the -// origin against the Host; on the cleartext leg neither wraps the actor's -// plaintext. +// A request leg's answer picks its route: every route matches dial=name and +// there is no default, so a request the sidecar did not answer for has no +// route. The route goes to a dynamic forward proxy cluster, and only with the +// answered name it dials. On the MITM leg it re-originates TLS and verifies +// the origin against the Host; on the cleartext leg it does not wrap the +// actor's plaintext. func TestEgressManifestsRequestLegsRouteByTheDial(t *testing.T) { - for _, path := range egressManifests { - t.Run(path, func(t *testing.T) { - tree := bootstrapTree(t, path) - all := clusters(tree) - for _, lc := range allChains(tree) { - name := str(lc.chain, "name") - if !slices.Contains(requestLegs, name) { + tree := bootstrapTree(t) + all := clusters(tree) + for _, lc := range allChains(tree) { + name := str(lc.chain, "name") + if !slices.Contains(requestLegs, name) { + continue + } + routes := 0 + for _, vh := range list(child(hcm(lc.chain), "route_config"), "virtual_hosts") { + for _, r := range list(vh, "routes") { + clusterName := str(child(r, "route"), "cluster") + cluster := byName(all, clusterName) + if cluster == nil { + t.Errorf("chain %q routes to cluster %q, which does not exist", name, clusterName) continue } - dials := map[string]int{} - for _, vh := range list(child(hcm(lc.chain), "route_config"), "virtual_hosts") { - for _, r := range list(vh, "routes") { - dial := dialMatchOf(child(r, "match")) - clusterName := str(child(r, "route"), "cluster") - cluster := byName(all, clusterName) - if cluster == nil { - t.Errorf("chain %q routes to cluster %q, which does not exist", name, clusterName) - continue - } - dials[dial]++ - switch dial { - case extproc.EgressDialName: - if got := str(child(cluster, "cluster_type"), "name"); got != dfpClusterType { - t.Errorf("chain %q sends dial=name to %q of type %q, want a dynamic forward proxy", name, clusterName, got) - } - if !requiresDialHost(child(r, "match")) { - t.Errorf("chain %q sends dial=name to %q without requiring %s:%s; an answer without the name would dial the Host's port", name, clusterName, extproc.EgressMetadataNamespace, extproc.EgressDialHostKey) - } - case extproc.EgressDialAddress: - if got := str(cluster, "type"); got != "ORIGINAL_DST" { - t.Errorf("chain %q sends dial=address to %q of type %q, want ORIGINAL_DST", name, clusterName, got) - } - if lb := child(cluster, "original_dst_lb_config"); len(lb) != 0 { - t.Errorf("cluster %q has original_dst_lb_config %v; the address must come from the filter state alone", clusterName, lb) - } - default: - t.Errorf("chain %q has a route to %q that matches no dial; a request the sidecar did not answer for would take it", name, clusterName) - } - tls := cluster["transport_socket"] != nil - if name == extproc.EgressTLSMITMFilterChainName && (!tls || !autoSNIAndSAN(cluster)) { - t.Errorf("chain %q routes to %q, which does not re-originate TLS with the SNI and certificate check taken from the Host", name, clusterName) - } - if name == extproc.EgressCleartextFilterChainName && tls { - t.Errorf("chain %q routes to %q, which wraps the actor's plaintext in TLS", name, clusterName) - } - } + routes++ + if dial := dialMatchOf(child(r, "match")); dial != extproc.EgressDialName { + t.Errorf("chain %q has a route to %q matching dial=%q, want %q; a request the sidecar did not answer for would take it", name, clusterName, dial, extproc.EgressDialName) } - for _, dial := range []string{extproc.EgressDialName, extproc.EgressDialAddress} { - if dials[dial] == 0 { - t.Errorf("chain %q has no route for dial=%s", name, dial) - } + if got := str(child(cluster, "cluster_type"), "name"); got != dfpClusterType { + t.Errorf("chain %q sends dial=name to %q of type %q, want a dynamic forward proxy", name, clusterName, got) + } + if !requiresDialHost(child(r, "match")) { + t.Errorf("chain %q sends dial=name to %q without requiring %s:%s; an answer without the name would dial the Host's port", name, clusterName, extproc.EgressMetadataNamespace, extproc.EgressDialHostKey) + } + tls := cluster["transport_socket"] != nil + if name == extproc.EgressTLSMITMFilterChainName && (!tls || !autoSNIAndSAN(cluster)) { + t.Errorf("chain %q routes to %q, which does not re-originate TLS with the SNI and certificate check taken from the Host", name, clusterName) + } + if name == extproc.EgressCleartextFilterChainName && tls { + t.Errorf("chain %q routes to %q, which wraps the actor's plaintext in TLS", name, clusterName) } } - }) + } + if routes == 0 { + t.Errorf("chain %q has no route for dial=%s", name, extproc.EgressDialName) + } } } @@ -574,7 +476,7 @@ func TestEgressManifestsRequestLegsRouteByTheDial(t *testing.T) { // cluster only with allow_dynamic_host_from_filter_state; without it, it // resolves the Host as sent, into the DNS cache all actors share. func TestEgressManifestsRequestLegsDialTheAnsweredName(t *testing.T) { - tree := bootstrapTree(t, egressManifest) + tree := bootstrapTree(t) if writers := filterStateWriters(tree, extproc.UpstreamDynamicHostFilterStateKey); len(writers) != len(requestLegs) { t.Errorf("%s is set by %d filters, want one per request leg", extproc.UpstreamDynamicHostFilterStateKey, len(writers)) } @@ -630,9 +532,6 @@ func mustJSON(t *testing.T, n node) string { return string(j) } -// egressManifest is the envoy gateway, which runs the egress-policy module. -var egressManifest = egressManifests[0] - // mitmListener returns the egress manifest's inner listener. func mitmListener(t *testing.T, tree node) node { t.Helper() @@ -647,7 +546,7 @@ func mitmListener(t *testing.T, tree node) node { // "denied" mapping to no chain. The module runs after both inspectors, and no // chain keeps a filter_chain_match, which Envoy ignores once a matcher is set. func TestEgressManifestsInnerListenerSelectsOnTheModuleVerdict(t *testing.T) { - tree := bootstrapTree(t, egressManifest) + tree := bootstrapTree(t) l := mitmListener(t, tree) filters := list(l, "listener_filters") @@ -695,9 +594,10 @@ func TestEgressManifestsInnerListenerSelectsOnTheModuleVerdict(t *testing.T) { } // The outer ext_proc must accept dev.ate.policy.egress, and the outer chain -// must copy it after ext_proc into shared filter state for the module. +// must copy the policy's hash after ext_proc into filter state shared with the +// inner listener, where the module looks the rules up by it. func TestEgressManifestsConnectLegHandsTheSNIRulesHashToTheInnerListener(t *testing.T) { - tree := bootstrapTree(t, egressManifest) + tree := bootstrapTree(t) outer := outerChain(t, tree) cfg, extProcAt, filters := extProcOf(outer) if cfg == nil { @@ -744,7 +644,7 @@ func TestEgressManifestsConnectLegHandsTheSNIRulesHashToTheInnerListener(t *test } // These keys determine pool selection and their values are bound to the pool at the // time of pool creation. All CONNECT tunnels that have the same values for these filter - // state keys will share to pool. + // state keys share a pool. for _, key := range []string{ extproc.ActorIdentityFilterStateKey, extproc.ConnectAuthorityFilterStateKey, @@ -756,17 +656,17 @@ func TestEgressManifestsConnectLegHandsTheSNIRulesHashToTheInnerListener(t *test } } if at < 0 { - t.Fatalf("the writer of %s is not on the outer chain, where the answer is", extproc.EgressPolicyMetadataNamespace) + t.Fatalf("the writer of %s is not on the outer chain, where the answer is", extproc.EgressPolicyHashFilterStateKey) } if at < extProcAt { - t.Errorf("%s is set at http_filters[%d], before ext_proc at [%d]; the metadata it reads does not exist yet", extproc.EgressPolicyMetadataNamespace, at, extProcAt) + t.Errorf("%s is set at http_filters[%d], before ext_proc at [%d]; the metadata it reads does not exist yet", extproc.EgressPolicyHashFilterStateKey, at, extProcAt) } format := child(entry, "format_string") if got := str(child(format, "text_format_source"), "inline_string"); got != extproc.EgressPolicyHashFormat { t.Errorf("%s is set from %q, want %q", extproc.EgressPolicyHashFilterStateKey, got, extproc.EgressPolicyHashFormat) } if skip, _ := entry["skip_if_empty"].(bool); !skip { - t.Errorf("%s is not skip_if_empty; an absent answer would be written as unparseable JSON", extproc.EgressPolicyMetadataNamespace) + t.Errorf("%s is not skip_if_empty; an absent hash would be written as an empty one", extproc.EgressPolicyHashFilterStateKey) } } @@ -774,7 +674,7 @@ func TestEgressManifestsConnectLegHandsTheSNIRulesHashToTheInnerListener(t *test // record. It must log only NR connections and include the SNI, actor, and // verdict. func TestEgressManifestsInnerListenerLogsDeniedConnections(t *testing.T) { - tree := bootstrapTree(t, egressManifest) + tree := bootstrapTree(t) logs := list(mitmListener(t, tree), "access_log") if len(logs) == 0 { t.Fatal("mitm_listener has no access_log; a ClientHello the module denies would leave no record") @@ -808,7 +708,7 @@ func TestEgressManifestsInnerListenerLogsDeniedConnections(t *testing.T) { // actor dialed must have no way in, and only this chain may dial the raw // cluster, since nothing else authorizes a by-name dial without a request leg. func TestEgressManifestsPassthroughChainDialsTheResolvedSNI(t *testing.T) { - tree := bootstrapTree(t, egressManifest) + tree := bootstrapTree(t) chain := byName(list(mitmListener(t, tree), "filter_chains"), "egress_passthrough") if chain == nil { t.Fatal("no egress_passthrough chain on mitm_listener") @@ -867,12 +767,12 @@ func TestEgressManifestsPassthroughChainDialsTheResolvedSNI(t *testing.T) { // Nothing may dial the address the actor chose. for _, c := range clusters(tree) { - if str(c, "type") == "ORIGINAL_DST" && str(c, "name") == passthroughCluster { - t.Errorf("cluster %q still exists; the passthrough chain must dial the resolved SNI, not the actor's address", passthroughCluster) + if str(c, "type") == "ORIGINAL_DST" { + t.Errorf("cluster %q is ORIGINAL_DST; the gateway dials only names it resolved, never the actor's address", str(c, "name")) } } - if writers := filterStateWriters(tree, originalDstKey); len(writers) != 0 { - t.Errorf("%s is written by %d filters; the gateway must not carry the address the actor dialed to the inner listener", originalDstKey, len(writers)) + if writers := filterStateWriters(tree, extproc.OriginalDstFilterStateKey); len(writers) != 0 { + t.Errorf("%s is written by %d filters; the gateway must not carry the address the actor dialed to the inner listener", extproc.OriginalDstFilterStateKey, len(writers)) } } @@ -881,7 +781,7 @@ func TestEgressManifestsPassthroughChainDialsTheResolvedSNI(t *testing.T) { // else. The forward proxy falls back to its configured port when the state is // absent, so a missing port would silently redirect the connection. func TestEgressManifestsConnectLegHandsTheDialedPortToTheInnerListener(t *testing.T) { - tree := bootstrapTree(t, egressManifest) + tree := bootstrapTree(t) writers := filterStateWriters(tree, extproc.UpstreamDynamicPortFilterStateKey) if len(writers) != 1 { t.Fatalf("%s is set by %d filters, want exactly the CONNECT leg's copy of its answer", extproc.UpstreamDynamicPortFilterStateKey, len(writers)) diff --git a/cmd/atenet/internal/router/envoyadmin_manifest_test.go b/cmd/atenet/internal/router/envoyadmin_manifest_test.go index 93e684eb33..84cd0e5dd4 100644 --- a/cmd/atenet/internal/router/envoyadmin_manifest_test.go +++ b/cmd/atenet/internal/router/envoyadmin_manifest_test.go @@ -38,7 +38,7 @@ var envoyAdminManifests = []struct { path, configMap, deployment, requiredPath string }{ {routerManifestPath, "atenet-router-envoy-config", "atenet-router", "/stats/prometheus"}, // PodMonitoring - {egressManifests[0], "atenet-egress", "atenet-egress", "/ready"}, // kubelet probes + {egressManifest, "atenet-egress", "atenet-egress", "/ready"}, // kubelet probes } type socketAddress struct { diff --git a/cmd/atenet/internal/router/extproc/attributes.go b/cmd/atenet/internal/router/extproc/attributes.go index 2d0a5a717e..5be874d818 100644 --- a/cmd/atenet/internal/router/extproc/attributes.go +++ b/cmd/atenet/internal/router/extproc/attributes.go @@ -42,7 +42,7 @@ const ( ConnectAuthorityFilterStateAttribute = "filter_state['" + ConnectAuthorityFilterStateKey + "']" // ActorIdentityFilterStateKey holds the actor's SPIFFE ID - // (resources.ActorSPIFFEID), read from the peer certificate's URI SAN. + // (resources.WorkerForActorSPIFFEID), read from the peer certificate's URI SAN. // The outer CONNECT chain sets it from %DOWNSTREAM_PEER_URI_SAN% and // shares it with the inner legs, which have no certificate of their own. ActorIdentityFilterStateKey = "dev.ate.actor.identity" @@ -80,7 +80,7 @@ const ( EgressFilterChainFilterStateKey = "dev.ate.egress.filter_chain" // EgressFilterChainMITM: TLS terminated on EgressTLSMITMFilterChainName. EgressFilterChainMITM = "mitm" - // EgressFilterChainPassthrough: forwarded unread. Unused for now. + // EgressFilterChainPassthrough: forwarded unread to the resolved SNI. EgressFilterChainPassthrough = "passthrough" // EgressFilterChainCleartext: not TLS, EgressCleartextFilterChainName. EgressFilterChainCleartext = "cleartext" @@ -88,20 +88,15 @@ const ( EgressFilterChainDenied = "denied" // EgressDialKey, under EgressMetadataNamespace, is a request leg's answer // for an allowed request: where it goes. The manifests' routes match on - // it, one route per value and none without, so a request with no answer - // has no route. + // it and have no default, so a request with no answer has no route. EgressDialKey = "dial" - // EgressDialName: a hostname rule matched, so the forward proxy resolves - // EgressDialHostKey and dials it on the port the actor dialed. + // EgressDialName: the forward proxy resolves EgressDialHostKey and dials + // it on the port the actor dialed. EgressDialName = "name" - // EgressDialAddress: an address or all rule matched, so the request goes - // to the address the actor dialed, read from the ORIGINAL_DST filter state - // the CONNECT leg's answer set. - EgressDialAddress = "address" // EgressDialHostKey, under EgressMetadataNamespace, comes with // EgressDialName: the name the request was decided on, without the Host's // port, which was never checked. The request chains copy it into - // UpstreamDynamicHostFilterStateKey, and their by-name routes need it. + // UpstreamDynamicHostFilterStateKey, and their routes need it. EgressDialHostKey = "host" // directionAttribute carries the Direction outright, for dataplanes that @@ -149,22 +144,10 @@ const UpstreamDynamicHostFilterStateKey = "envoy.upstream.dynamic_host" const EgressPolicyHashFormat = "%FILTER_STATE(" + EgressPolicyHashFilterStateKey + ".not_shared:PLAIN)%" // OriginalDstFilterStateKey is Envoy's filter-state key for the address an -// ORIGINAL_DST cluster dials. The ingress router sets it for actor tunnels. -// The egress gateway never writes it and dials only by name; its request legs -// read it as an attribute. +// ORIGINAL_DST cluster dials. The ingress router sets it for actor tunnels; +// the egress gateway dials only by name and never uses it. const OriginalDstFilterStateKey = "envoy.network.transport_socket.original_dst_address" -// OriginalDstIPAttribute and OriginalDstPortAttribute are the CEL expressions -// that read OriginalDstFilterStateKey, one field each. The object as a whole -// is not readable: Envoy's CEL presents an object with field support as a -// map, which ext_proc renders as the literal "CelMap value". The field names -// are Envoy's (the same ones %FILTER_STATE(key:FIELD:ip)% takes); the port -// arrives as a number. -const ( - OriginalDstIPAttribute = "filter_state['" + OriginalDstFilterStateKey + "'].ip" - OriginalDstPortAttribute = "filter_state['" + OriginalDstFilterStateKey + "'].port" -) - // RequestedServerNameAttribute is the SNI of the connection a request arrived // on. The handler logs it next to the Host it authorized. const RequestedServerNameAttribute = "connection.requested_server_name" diff --git a/cmd/atenet/internal/router/extproc/dispatch.go b/cmd/atenet/internal/router/extproc/dispatch.go index 5bcafa0f04..d82856f1df 100644 --- a/cmd/atenet/internal/router/extproc/dispatch.go +++ b/cmd/atenet/internal/router/extproc/dispatch.go @@ -35,13 +35,14 @@ const ( // manifests/ate-install/atenet-egress.yaml. const ( // EgressFilterChainName terminates the actor's outer mTLS CONNECT: the - // certificate is authenticated and the address rules decided here. + // certificate is authenticated and the SNI rules for the dialed port + // handed to the dataplane here. EgressFilterChainName = "egress" // EgressTLSMITMFilterChainName is the gateway's chain for TLS it // terminated with a minted leaf. The handler authorizes every request. EgressTLSMITMFilterChainName = "egress_tls_mitm" // EgressCleartextFilterChainName is the chain for HTTP the actor sent in - // the clear, on both gateways. The handler authorizes every request. + // the clear. The handler authorizes every request. EgressCleartextFilterChainName = "egress_cleartext" ) diff --git a/cmd/atenet/internal/router/extproc/metadata_test.go b/cmd/atenet/internal/router/extproc/metadata_test.go index e33ee4f04b..e025efbac7 100644 --- a/cmd/atenet/internal/router/extproc/metadata_test.go +++ b/cmd/atenet/internal/router/extproc/metadata_test.go @@ -195,12 +195,13 @@ func TestRequestMetadataHeaderIsCaseInsensitive(t *testing.T) { } func TestRequestMetadataAttributeNumber(t *testing.T) { + const port = "filter_state['some.object'].port" md := NewRequestMetadata(nil, map[string]*structpb.Struct{ "envoy.filters.http.ext_proc": {Fields: map[string]*structpb.Value{ - OriginalDstPortAttribute: structpb.NewNumberValue(8443), + port: structpb.NewNumberValue(8443), }}, }) - if got, want := md.Attribute(OriginalDstPortAttribute), "8443"; got != want { + if got, want := md.Attribute(port), "8443"; got != want { t.Errorf("Attribute() for a number = %q, want %q", got, want) } } diff --git a/cmd/dataplane/envoy/README.md b/cmd/dataplane/envoy/README.md index e86f6f2db7..b543cce1d9 100644 --- a/cmd/dataplane/envoy/README.md +++ b/cmd/dataplane/envoy/README.md @@ -1,6 +1,6 @@ # Envoy Dataplane Image -This directory contains the build definition and custom extensions for the Envoy dataplane container image (`envoy-dataplane`) used by Agent Substrate's `atenet-egress` gateway. +This directory contains the build definition and custom extensions for the Envoy dataplane container image (`envoy-dataplane`) used by Agent Substrate's `atenet-router` and `atenet-egress` gateways. ## Directory Contents @@ -19,7 +19,7 @@ cmd/dataplane/envoy/ ## Building and Deployment -During a build from source, `ate-setup` builds the image from this directory via `docker buildx`, pushes it to `$KO_DOCKER_REPO/envoy-dataplane`, and its resolved digest replaces the `${ENVOY_DATAPLANE_IMAGE}` placeholder in the egress manifest (`manifests/ate-install/atenet-egress.yaml`). +During a build from source, `ate-setup` builds the image from this directory via `docker buildx`, pushes it to `$KO_DOCKER_REPO/envoy-dataplane`, and its resolved digest replaces the `${ENVOY_DATAPLANE_IMAGE}` placeholder in the router and egress manifests (`manifests/ate-install/atenet-router.yaml`, `manifests/ate-install/atenet-egress.yaml`). A pre-built install (`ate-setup deploy --image-repo REPO --image-tag TAG`) builds nothing and pins `REPO/envoy-dataplane:TAG` instead, so a release publishes it with the other images: diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy/README.md b/cmd/dataplane/envoy/dynamic-modules/egress-policy/README.md index 91daa78aa9..dbc11f27bd 100644 --- a/cmd/dataplane/envoy/dynamic-modules/egress-policy/README.md +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy/README.md @@ -12,13 +12,13 @@ listener that terminates TCP connection encapsulated in the CONNECT request. In the outer CONNECT filter chain the policy store HTTP filter is instantiated after the ext_proc filter. After ext_proc successfully looks up egress policy for the given actor and -destinaion port, the subsequent policy store filter: +destination port, the subsequent policy store filter: -1. Creates EngressPolicy object by parsing JSON from metadata sent by the ext_proc server and computes its hash. +1. Creates EgressPolicy object by parsing JSON from metadata sent by the ext_proc server and computes its hash. 1. Checks if the policy with this hash already exists in the worker local policy store. 1. If it does it adds a reference to the existing policy. 1. Otherwise it stores the new policy in the store. -1. The policy hash is written to the filter state shared with the upstream. +1. The policy hash is written to filter state, which the chain's set_filter_state filter copies into filter state shared with the upstream. In the inner listener filter chain, the policy enforcement listener filter: @@ -26,7 +26,7 @@ In the inner listener filter chain, the policy enforcement listener filter: 1. Looks up the policy object by hash in the worker local policy store. 1. Evaluates and enforces egress policy. -With this approch, egress policy objects are shared by many CONNECT requests. Per request state is +With this approach, egress policy objects are shared by many CONNECT requests. Per request state is a 64 bit hash. For example if all actors have the same policy, Envoy will keep only one policy object in memory and all requests will share the same object. @@ -34,18 +34,18 @@ and all requests will share the same object. Ownership of EgressPolicy objects is shared by policy store HTTP filters and the worker local policy store. As such policy object is guaranteed to outlive all CONNECT requests that share the same policy. Lifetime of the CONNECT request -is at least as long as the lifetime of the inner request, so it is guaratedd that inner request will always +is at least as long as the lifetime of the inner request, so it is guaranteed that inner request will always find a policy with hash it received from the outer CONNECT request. To avoid leaking policy object in the local policy store, policy store HTTP filter, during its cleanup, checks the -reference count value of the EgressPolicy. If this cound vaoue is 2, then only this filter and policy store have outstanding -references, and it this point it is safe to remove policy object from the store. This guaratees that policy is detroyed, +reference count value of the EgressPolicy. If this count value is 2, then only this filter and policy store have outstanding +references, and at this point it is safe to remove policy object from the store. This guarantees that policy is destroyed, when no more CONNECT requests are referencing it. ## Policy Hashing -[Rapidhash](https://docs.rs/rapidhash/latest/rapidhash/) is considered one of the best noon-cryptographic hashing funections -wth near [ideal collision probability](https://chromium.googlesource.com/chromium/src/+/HEAD/third_party/rapidhash/README.md). +[Rapidhash](https://docs.rs/rapidhash/latest/rapidhash/) is considered one of the best non-cryptographic hashing functions +with near [ideal collision probability](https://chromium.googlesource.com/chromium/src/+/HEAD/third_party/rapidhash/README.md). Other options are xxhash which can produce 128 bit hashes or cryptographic hash such as SHA-256. 64 bit Rapidhash offers the best balance between quality and performance, with approximately 1 in 10^19 probability of collision. diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 1554712f85..ce1f59c4d5 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -15,16 +15,17 @@ # contract expected by atunnel's egress client: # * downstream mTLS on :443 (present servicedns identity, require + verify an # actor-identity client cert), -# * terminate the actor's HTTP CONNECT and tunnel raw TCP to the CONNECT -# authority (the actor's original destination, always sent as IP:port). +# * terminate the actor's HTTP CONNECT, whose authority is the actor's +# original destination, always sent as IP:port. The gateway never dials +# that address. # # What goes through the tunnel is authorized against the actor's EgressPolicy # by the ext_proc sidecar. Cleartext HTTP and the TLS this gateway terminates -# are decided per request: the rules in policy order, over the Host and the -# address the actor dialed, and the request goes to what the matching rule -# checked: the Host's name resolved here, on the port the actor dialed, or the -# dialed address. Anything else reaches the dialed address only when an address -# rule allowed it at the CONNECT. See cmd/atenet/internal/router/README.md. +# are decided per request, on the Host and the port the actor dialed, and the +# request goes to the Host's name resolved here, on that port. Other TLS is +# decided at the ClientHello by the SNI rules the sidecar returned on CONNECT, +# and relayed unread to the resolved SNI. See +# cmd/atenet/internal/router/README.md. apiVersion: v1 kind: ServiceAccount metadata: @@ -100,9 +101,8 @@ data: validation_context: # Actors authenticate with an actor-identity client cert. # Envoy enforces chain, signature, and validity period here, so - # an unsigned or expired actor cert never reaches ext_proc; the - # ActorIdentity extension is checked there because Envoy cannot - # read custom X.509 extensions. + # an unsigned or expired actor cert never reaches ext_proc, + # which reads the actor from its SPIFFE URI SAN. trusted_ca: { filename: /run/actor-id-ca-certs/ca.crt } filters: - name: envoy.filters.network.http_connection_manager @@ -301,9 +301,8 @@ data: # request the policy denies never reaches it. # # The passthrough chain is a tcp_proxy, so it has no HTTP filters of its - # own, and it decides nothing: it dials the address the CONNECT leg - # allowed, carried here as filter state, and closes when there is none. - # Nothing selects it yet. + # own, and it decides nothing: the module already matched the SNI to a + # tls_passthrough rule, and the chain relays the bytes to that name. # --------------------------------------------------------------------- - name: mitm_listener stat_prefix: mitm @@ -402,8 +401,8 @@ data: stream_idle_timeout: 0s # This is the leg that knows where the traffic actually went. # REQUESTED_SERVER_NAME is the SNI the leaf was minted for and it - # should equal the authority, which is what a by-name route - # resolves and what both routes verify the origin against. + # should equal the authority, which is what the routes resolve + # and verify the origin against. access_log: - name: envoy.access_loggers.file # Every actor request lands here, so log the ones that went @@ -461,9 +460,9 @@ data: # There is no default route: the initial route lookup cannot # match before ext_proc supplies dev.ate.egress:dial. This # lets the request filter return the unsupported-upgrade - # response before route-level upgrade rejection. A by-name - # route also needs the answered name (dev.ate.egress:host); - # without it the forward proxy would dial the Host's port. + # response before route-level upgrade rejection. Each route + # also needs the answered name (dev.ate.egress:host); without + # it the forward proxy would dial the Host's port. # Native gRPC uses upstream ALPN to negotiate h2 and carry # status in trailers. Other requests stay on HTTP/1.1; the # content-type match keeps those protocol choices separate. @@ -486,22 +485,6 @@ data: - upgrade_type: websocket enabled: false timeout: 0s - - match: - prefix: "/" - headers: - - name: content-type - string_match: - prefix: application/grpc - dynamic_metadata: - - filter: dev.ate.egress - path: [{ key: dial }] - value: { string_match: { exact: address } } - route: - cluster: egress_original_dst_tls_grpc - upgrade_configs: - - upgrade_type: websocket - enabled: false - timeout: 0s - match: prefix: "/" dynamic_metadata: @@ -519,18 +502,6 @@ data: # The route timeout runs until the response completes, which # for SSE or long-poll is never, so 0. timeout: 0s - - match: - prefix: "/" - dynamic_metadata: - - filter: dev.ate.egress - path: [{ key: dial }] - value: { string_match: { exact: address } } - route: - cluster: egress_original_dst_tls - upgrade_configs: - - upgrade_type: websocket - enabled: false - timeout: 0s http_filters: # Every request on this chain is authorized against the actor's # EgressPolicy by the same sidecar; the actor comes from the filter @@ -551,8 +522,6 @@ data: - xds.filter_chain_name - filter_state['dev.ate.actor.identity'] - filter_state['dev.ate.connect.authority'] - - filter_state['envoy.network.transport_socket.original_dst_address'].ip - - filter_state['envoy.network.transport_socket.original_dst_address'].port - connection.requested_server_name processing_mode: request_header_mode: SEND @@ -587,13 +556,12 @@ data: inline_string: "%DYNAMIC_METADATA(dev.ate.egress:host)%" omit_empty_values: true #ATE_MITM_EXTPROC_FILTER -- see the mitm_listener comment above. - # Resolves the by-name route's name before the cluster dials it. + # Resolves the route's name before the cluster dials it. # allow_dynamic_host_from_filter_state makes it resolve # envoy.upstream.dynamic_host on the port the actor dialed, the # entry the cluster dials; without it, it resolves the Host as # sent, and every port or spelling an actor puts there takes a - # slot in the DNS cache all actors share. It does nothing for a - # route to any other kind of cluster. + # slot in the DNS cache all actors share. - name: envoy.filters.http.dynamic_forward_proxy typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig @@ -715,10 +683,11 @@ data: virtual_hosts: - name: all domains: ["*"] - # One route per dial the sidecar answers with, and no default: - # a request it did not answer for has no route. The by-name - # route also needs the answered name (dev.ate.egress:host); - # without it the forward proxy would dial the Host's port. + # Only the dial the sidecar answers with has a route, and + # there is no default: a request it did not answer for has + # no route. The route also needs the answered name + # (dev.ate.egress:host); without it the forward proxy would + # dial the Host's port. routes: - match: prefix: "/" @@ -735,18 +704,6 @@ data: - upgrade_type: websocket enabled: false timeout: 0s - - match: - prefix: "/" - dynamic_metadata: - - filter: dev.ate.egress - path: [{ key: dial }] - value: { string_match: { exact: address } } - route: - cluster: egress_original_dst_cleartext - upgrade_configs: - - upgrade_type: websocket - enabled: false - timeout: 0s http_filters: # Every request on this chain is authorized against the actor's # EgressPolicy by the same sidecar; the actor comes from the filter @@ -767,8 +724,6 @@ data: - xds.filter_chain_name - filter_state['dev.ate.actor.identity'] - filter_state['dev.ate.connect.authority'] - - filter_state['envoy.network.transport_socket.original_dst_address'].ip - - filter_state['envoy.network.transport_socket.original_dst_address'].port - connection.requested_server_name processing_mode: request_header_mode: SEND @@ -803,13 +758,12 @@ data: inline_string: "%DYNAMIC_METADATA(dev.ate.egress:host)%" omit_empty_values: true #ATE_MITM_EXTPROC_FILTER -- Don't modify this line. - # Resolves the by-name route's name before the cluster dials it. + # Resolves the route's name before the cluster dials it. # allow_dynamic_host_from_filter_state makes it resolve # envoy.upstream.dynamic_host on the port the actor dialed, the # entry the cluster dials; without it, it resolves the Host as # sent, and every port or spelling an actor puts there takes a - # slot in the DNS cache all actors share. It does nothing for a - # route to any other kind of cluster. + # slot in the DNS cache all actors share. - name: envoy.filters.http.dynamic_forward_proxy typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig @@ -921,8 +875,8 @@ data: # actor ID from "dev.ate.actor.identity" # CONNECT authority from "dev.ate.connect.authority" # Egress policy hash from "dev.ate.policy.egress.hash" - # TODO(yanavlasov): use just the port from CONNECT athority to minimize number of pools. - # Pools are create using hashes of filter state, which includes filter state objects that use + # TODO(yanavlasov): use just the port from CONNECT authority to minimize number of pools. + # Pools are created using hashes of filter state, which includes filter state objects that use # envoy.hashable_string factory. connect_timeout: 1s # Envoy always opens a new pool's first connection. @@ -936,9 +890,8 @@ data: typed_extension_protocol_options: envoy.extensions.upstreams.tcp.v3.TcpProtocolOptions: "@type": type.googleapis.com/envoy.extensions.upstreams.tcp.v3.TcpProtocolOptions - # Closes a connection no tunnel holds, which its one-tunnel pool - # will never use again. The timer runs only while no tunnel holds - # the connection, so a quiet tunnel is never cut by it. + # Closes a connection no tunnel holds. The timer runs only while no + # tunnel holds the connection, so a quiet tunnel is never cut by it. idle_timeout: 30s # Copies every filter state object shared with the upstream onto the # internal listener's downstream connection: dev.ate.actor.identity, @@ -979,10 +932,9 @@ data: # for one, which surfaces as a handshake failure and looks exactly like # the allowlist denying the name -- silently, at a threshold nothing # else in this file mentions. 32768 is deliberately ~3x the - # 10,000-name target rather than equal to it: the memory guards above - # are what should bind first, at ~13,200, and a limit that trips before - # them would replace a diagnosable overload with a mystery. Raise it - # when you raise the target. + # 10,000-name target rather than equal to it: memory should bind first, + # and a limit that trips before it would replace a diagnosable overload + # with a mystery. Raise it when you raise the target. circuit_breakers: thresholds: - priority: DEFAULT @@ -1023,9 +975,9 @@ data: socket_address: address: 127.0.0.1 port_value: 50051 - # The only cluster that dials the internet. Reached from the MITM leg, so - # it dials the name the sidecar answered for the decrypted request, not - # the CONNECT authority, on the port the actor dialed. + # The MITM leg's forward proxy. It dials the name the sidecar answered for + # the decrypted request, not the CONNECT authority, on the port the actor + # dialed. - name: egress_forward_proxy lb_policy: CLUSTER_PROVIDED connect_timeout: 5s @@ -1138,67 +1090,6 @@ data: http_protocol_options: {} http2_protocol_options: {} - # The MITM leg's by-address upstreams: the address the actor dialed, from - # the original_dst_address filter state. Nothing sets this anymore. - # Same TLS as the forward proxies above, SNI and certificate check from - # the Host, so a Host the dialed origin cannot prove fails the handshake. - - name: egress_original_dst_tls - type: ORIGINAL_DST - lb_policy: CLUSTER_PROVIDED - connect_timeout: 5s - typed_extension_protocol_options: - envoy.extensions.upstreams.http.v3.HttpProtocolOptions: - "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions - upstream_http_protocol_options: - auto_sni: true - auto_san_validation: true - explicit_http_config: - http_protocol_options: {} - transport_socket: - name: envoy.transport_sockets.tls - typed_config: - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext - common_tls_context: - validation_context: - trusted_ca: - filename: /etc/ssl/certs/ca-certificates.crt - - - name: egress_original_dst_tls_grpc - type: ORIGINAL_DST - lb_policy: CLUSTER_PROVIDED - connect_timeout: 5s - typed_extension_protocol_options: - envoy.extensions.upstreams.http.v3.HttpProtocolOptions: - "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions - upstream_http_protocol_options: - auto_sni: true - auto_san_validation: true - auto_config: - http_protocol_options: {} - http2_protocol_options: {} - transport_socket: - name: envoy.transport_sockets.tls - typed_config: - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext - common_tls_context: - validation_context: - trusted_ca: - filename: /etc/ssl/certs/ca-certificates.crt - - # The cleartext chain's by-address upstream: the address the actor - # dialed, from the same original_dst_address filter state the passthrough - # chains use. Plaintext, like the forward proxy above. - - name: egress_original_dst_cleartext - type: ORIGINAL_DST - lb_policy: CLUSTER_PROVIDED - connect_timeout: 5s - typed_extension_protocol_options: - envoy.extensions.upstreams.http.v3.HttpProtocolOptions: - "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions - use_downstream_protocol_config: - http_protocol_options: {} - http2_protocol_options: {} - # The passthrough chain's upstream: the MITM leg's forward proxy on the # same DNS cache, with no transport socket (the bytes are the actor's own # TLS) and no HTTP options. Only egress_passthrough may dial it; nothing @@ -1288,8 +1179,7 @@ spec: # The same image as the ext-proc container below, run as a different # subcommand. Sharing the binary is what keeps the two from drifting; # staying two CONTAINERS is what keeps the MITM signing key off the - # data plane and gives the minter the separate memory limit the header - # of this file depends on. + # data plane and gives the minter its own memory limit. image: ko://go.ate.dev/substrate/cmd/atenet restartPolicy: Always args: @@ -1310,13 +1200,13 @@ spec: # TTL, Envoy holds a secret indefinitely, serves the leaf past its # notAfter, and the handshake still completes -- actors cannot object # because they reach the gateway with verification off, nothing in the - # cluster trusting the MITM anchor. Measured against Envoy 1.37.5 in - # poc/sdsmint/expiry, both with the TTL and without it. + # cluster trusting the MITM anchor. Measured against Envoy 1.37.5, both + # with the TTL and without it. # # It does not bound validity exactly: leaves are back-dated 5m for # clock skew and NotAfter is ttl past issuance, so the window a client - # would accept is ttl+5m. See Signer.Sign in - # cmd/atenet/internal/sdsmint/certauth. + # would accept is ttl+5m. See minter.certificate in + # cmd/atenet/internal/sdsmint. - "--leaf-cert-ttl=15m" - "--log-level=info" env: @@ -1329,18 +1219,14 @@ spec: # that churn produces. Set below the container limit so the # collector starts working before the kernel starts killing. If sdsmint # ever reaches this figure with a genuinely live heap it will thrash the - # GC and then OOM anyway; that is the missing shed path described in the - # header, not a misconfiguration here. + # GC and then OOM anyway; that is a missing shed path, not a + # misconfiguration here. - name: GOMEMLIMIT value: "640MiB" # Both of these scale with the live secret count, not with request rate. # # memory ~44,750 B per live secret x 10,000 => ~427 MiB, against a - # 768Mi limit. That limit is 0.85 - # x Envoy's fixed_heap, which is the ordering invariant from the - # header reduced to one number: at the ~13,200 names where Envoy - # starts refusing, this container is at 73% and still alive. See - # the header for why this is stream overhead and not + # 768Mi limit, mostly stream overhead rather than # certificates. The request is measured steady state rather than # a token floor, on purpose -- the scheduler places pods on # requests, so a 64Mi request on a container that grows to 427 @@ -1469,8 +1355,8 @@ spec: - --ateapi-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem - --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem # Same bundle Envoy validates the handshake against. The handler - # re-verifies the chain in Go and reads the ActorIdentity extension out - # of it; without this flag the router has no actor-identity roots and + # re-verifies the chain in Go and reads the actor from its SPIFFE URI + # SAN; without this flag the router has no actor-identity roots and # denies every egress CONNECT with a 503. - --actor-identity-ca-file=/run/actor-id-ca-certs/ca.crt # Disables Envoy-side tracing only (a span per CONNECT tunnel is From 87baab01d40b4628737bd5869a7c3856d8f502eb Mon Sep 17 00:00:00 2001 From: Sairaj Pokale Date: Wed, 21 Oct 2026 00:27:31 +0000 Subject: [PATCH 31/63] Retry GCS rate-limit errors when deleting and copying snapshots (#2143) ### What this PR does and why we need it ? GCS answers 429 while it scales a cold bucket up to a burst of suspends. ateapi used the default retry policy, which never retries a delete or copy without a precondition, so a single 429 failed a snapshot release or a tag copy. List did retry, but with no cap, until the caller's context ended. This change: * **ateapi:** every GCS call (list, delete, copy) retries transient errors with RetryAlways, backing off from 250ms up to 5s, at most 7 attempts per request (at most 12.75s of sleep). The settings sit on the bucket handle, so objects inherit them. That includes the copy's destination, which is where the SDK reads them. * **atelet:** drops `WithMaxAttempts(5)`, which never capped uploads, and backs off from 250ms up to 5s. On transient errors, compose, delete, copy and reads now retry until their context ends instead of stopping after 5 attempts. * **atelet:** scratch cleanup (`removeScratch`, for upload parts and compose intermediates) ran on `context.WithoutCancel(ctx)` with no deadline, so without the cap it could retry forever. Each cleanup now deletes its objects 8 at a time within a 30s timeout. It is best-effort: objects it can't delete are logged and left next to the object, instead of failing an upload that already succeeded. RetryAlways is safe here. A retried delete that already landed gets a 404, which Delete already treats as success. A retried copy writes the same unchanging source to the same destination. **Behavior change** * ateapi: the cap covers every caller, not just suspend: actor delete, revert, and tag create and delete. Delete and copy used to fail on the first 429, so for them this is only more retrying. List used to retry until the caller's context ended. In a long GCS outage, these calls now fail after at most 12.75s of sleep: * Delete, revert and tag return the error, and their caller must retry. * Suspend logs the error and leaves the replaced snapshot in storage until the actor is deleted. * atelet: in a long GCS outage, calls retry until their context ends. For work ateapi starts over gRPC, that is its deadline of at most 10 minutes. The golden-actor template reconciler is the exception: it calls `SuspendActor`, `ResumeActor` and `DeleteActor` in-process with no deadline, so the atelet calls it causes have none. Uploads already ignored the cap there. A follow-up can bound those calls. * atelet: as before, an upload the SDK sends as resumable (at least its Writer's `ChunkSize`) also gives up once a chunk has been retrying for 32s, the Writer's default `ChunkRetryDeadline`. * atelet: a scratch cleanup failure no longer fails the upload. ### Testing * `internal/objectstore/gcs_retry_test.go`: an httptest fake GCS answers a scripted number of errors per call. The tests check attempt counts, not timing (they back off 1ms), and run in `make test` without an emulator. * `pkg/objectstorage/gcs_retry_test.go`, against the same kind of fake: * An upload and a compose each get through six 429s. * An upload that only ever gets 429s keeps retrying until its context ends. * Scratch deletes run in parallel, within the limit. * Deletes that keep failing stop at the cleanup timeout, even when the upload's context is already canceled. * An upload whose part deletes all fail still succeeds. * Each check fails if its change is reverted: 5 attempts in ateapi, `WithMaxAttempts(5)` in atelet, no cleanup timeout, cleanup on the upload's context, deleting one object at a time, or returning the cleanup error. * The new and changed tests pass 20 runs in a row with `-race`. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- internal/objectstore/export_test.go | 29 +++ internal/objectstore/gcs.go | 33 ++- internal/objectstore/gcs_retry_test.go | 307 +++++++++++++++++++++++++ pkg/objectstorage/gcs.go | 16 +- pkg/objectstorage/gcs_retry_test.go | 258 ++++++++++++++++++--- pkg/objectstorage/gcscompose.go | 61 +++-- pkg/objectstorage/gcssparse.go | 2 +- 7 files changed, 656 insertions(+), 50 deletions(-) create mode 100644 internal/objectstore/export_test.go create mode 100644 internal/objectstore/gcs_retry_test.go diff --git a/internal/objectstore/export_test.go b/internal/objectstore/export_test.go new file mode 100644 index 0000000000..666b79766a --- /dev/null +++ b/internal/objectstore/export_test.go @@ -0,0 +1,29 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstore + +import ( + "testing" + + "github.com/googleapis/gax-go/v2" +) + +// SetGCSBackoffForTest makes GCS calls back off by b until t ends, so a test +// can drive many retries quickly. +func SetGCSBackoffForTest(t *testing.T, b gax.Backoff) { + old := gcsBackoff + gcsBackoff = b + t.Cleanup(func() { gcsBackoff = old }) +} diff --git a/internal/objectstore/gcs.go b/internal/objectstore/gcs.go index ee3ecff98e..a017c74e91 100644 --- a/internal/objectstore/gcs.go +++ b/internal/objectstore/gcs.go @@ -18,11 +18,23 @@ import ( "context" "errors" "fmt" + "time" "cloud.google.com/go/storage" + "github.com/googleapis/gax-go/v2" "google.golang.org/api/iterator" ) +// gcsBackoff and gcsMaxAttempts set how gcsStore retries transient errors +// (408, 429, 5xx), such as the 429s GCS returns while it scales a cold +// bucket. The first retry comes within 250ms, so one transient error adds +// little latency. Every caller waits on these calls, and suspend holds the +// actor's lease while it does, so each call gives up after 7 attempts (at +// most 12.75s of sleep). +var gcsBackoff = gax.Backoff{Initial: 250 * time.Millisecond, Max: 5 * time.Second, Multiplier: 2} + +const gcsMaxAttempts = 7 + type gcsStore struct { client *storage.Client } @@ -32,13 +44,25 @@ func NewGCS(client *storage.Client) Store { return &gcsStore{client: client} } +// bucket returns a handle to the named bucket that retries transient errors. +// Objects taken from it inherit the setting. RetryAlways is safe: a repeated +// delete gets a 404, which Delete treats as success, and a repeated copy +// rewrites the same object. +func (g *gcsStore) bucket(name string) *storage.BucketHandle { + return g.client.Bucket(name).Retryer( + storage.WithPolicy(storage.RetryAlways), + storage.WithBackoff(gcsBackoff), + storage.WithMaxAttempts(gcsMaxAttempts), + ) +} + func (g *gcsStore) List(ctx context.Context, bucket, prefix string) ([]string, error) { query := &storage.Query{Prefix: prefix} // Only the name is ever used, and asking for less makes GCS send less. if err := query.SetAttrSelection([]string{"Name"}); err != nil { return nil, fmt.Errorf("while selecting object attributes: %w", err) } - it := g.client.Bucket(bucket).Objects(ctx, query) + it := g.bucket(bucket).Objects(ctx, query) var objects []string for { attrs, err := it.Next() @@ -53,7 +77,7 @@ func (g *gcsStore) List(ctx context.Context, bucket, prefix string) ([]string, e } func (g *gcsStore) Delete(ctx context.Context, bucket, object string) error { - err := g.client.Bucket(bucket).Object(object).Delete(ctx) + err := g.bucket(bucket).Object(object).Delete(ctx) // An object that is already gone is the state this asks for. if err != nil && !errors.Is(err, storage.ErrObjectNotExist) { return fmt.Errorf("while deleting gs://%s/%s: %w", bucket, object, err) @@ -62,8 +86,9 @@ func (g *gcsStore) Delete(ctx context.Context, bucket, object string) error { } func (g *gcsStore) Copy(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string) error { - src := g.client.Bucket(srcBucket).Object(srcObject) - dst := g.client.Bucket(dstBucket).Object(dstObject) + src := g.bucket(srcBucket).Object(srcObject) + // The copier retries with dst's settings, not src's. + dst := g.bucket(dstBucket).Object(dstObject) // Copier.Run drives GCS's rewrite API, following the rewrite token until // the copy completes. The bytes move inside GCS whatever the object's size, // so a multi-gigabyte memory image never reaches this process. diff --git a/internal/objectstore/gcs_retry_test.go b/internal/objectstore/gcs_retry_test.go new file mode 100644 index 0000000000..fed1e16c74 --- /dev/null +++ b/internal/objectstore/gcs_retry_test.go @@ -0,0 +1,307 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstore_test + +import ( + "context" + "errors" + "fmt" + "maps" + "net/http" + "net/http/httptest" + "net/url" + "slices" + "strings" + "sync" + "testing" + "time" + + "cloud.google.com/go/storage" + "github.com/google/go-cmp/cmp" + "github.com/googleapis/gax-go/v2" + "go.ate.dev/substrate/internal/objectstore" + "google.golang.org/api/googleapi" +) + +// fakeGCS is a minimal GCS JSON API (list, delete, rewrite) over one in-memory +// bucket. A request first gets the next status scripted for its key ("LIST +// ", "DELETE " or "COPY ") and is served normally +// once none are left, so a test sets exactly how many errors each call sees. +type fakeGCS struct { + t *testing.T + + mu sync.Mutex + objects map[string]bool + script map[string][]int + attempts map[string]int +} + +// newFakeGCS starts a fake GCS holding objects and returns it with a Store +// built the way cmd/ateapi builds one, backing off 1ms between attempts. +func newFakeGCS(t *testing.T, objects ...string) (*fakeGCS, objectstore.Store) { + t.Helper() + objectstore.SetGCSBackoffForTest(t, gax.Backoff{Initial: time.Millisecond, Max: time.Millisecond}) + fake := &fakeGCS{t: t, objects: map[string]bool{}, script: map[string][]int{}, attempts: map[string]int{}} + for _, name := range objects { + fake.objects[name] = true + } + srv := httptest.NewServer(fake) + t.Cleanup(srv.Close) + // The storage client sends every call to the emulator host when it is set. + t.Setenv("STORAGE_EMULATOR_HOST", srv.URL) + client, err := storage.NewClient(t.Context()) + if err != nil { + t.Fatalf("storage client: %v", err) + } + t.Cleanup(func() { client.Close() }) + return fake, objectstore.NewGCS(client) +} + +// respond scripts the statuses key's next requests get. +func (f *fakeGCS) respond(key string, statuses ...int) { + f.mu.Lock() + defer f.mu.Unlock() + f.script[key] = statuses +} + +// attemptsOf returns how many requests key has seen. +func (f *fakeGCS) attemptsOf(key string) int { + f.mu.Lock() + defer f.mu.Unlock() + return f.attempts[key] +} + +// names returns the bucket's object names, sorted. +func (f *fakeGCS) names() []string { + f.mu.Lock() + defer f.mu.Unlock() + return slices.Sorted(maps.Keys(f.objects)) +} + +func (f *fakeGCS) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Object names arrive path-escaped ("snap%2Ffile"), so split the path + // before unescaping them. There is one bucket, so its name is dropped. + rest, ok := strings.CutPrefix(r.URL.EscapedPath(), "/storage/v1/b/") + _, rest, _ = strings.Cut(rest, "/") + switch { + case ok && r.Method == http.MethodGet && rest == "o": + prefix := r.URL.Query().Get("prefix") + if f.scripted(w, "LIST "+prefix) { + return + } + var items []string + for _, name := range f.names() { + if strings.HasPrefix(name, prefix) { + items = append(items, fmt.Sprintf(`{"name":%q}`, name)) + } + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"items":[%s]}`, strings.Join(items, ",")) + + case ok && r.Method == http.MethodDelete && strings.HasPrefix(rest, "o/"): + object := f.unescape(strings.TrimPrefix(rest, "o/")) + if f.scripted(w, "DELETE "+object) { + return + } + f.mu.Lock() + found := f.objects[object] + delete(f.objects, object) + f.mu.Unlock() + if !found { + writeGCSError(w, http.StatusNotFound) + return + } + w.WriteHeader(http.StatusNoContent) + + case ok && r.Method == http.MethodPost && strings.Contains(rest, "/rewriteTo/b/"): + src, dst, _ := strings.Cut(strings.TrimPrefix(rest, "o/"), "/rewriteTo/b/") + _, dst, _ = strings.Cut(dst, "/o/") + src, dst = f.unescape(src), f.unescape(dst) + if f.scripted(w, "COPY "+dst) { + return + } + f.mu.Lock() + found := f.objects[src] + if found { + f.objects[dst] = true + } + f.mu.Unlock() + if !found { + writeGCSError(w, http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"done":true,"resource":{"name":%q}}`, dst) + + default: + f.t.Errorf("fake GCS got an unexpected request: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusBadRequest) + } +} + +// scripted counts a request for key and answers it with key's next scripted +// status, if there is one. It reports whether it answered. +func (f *fakeGCS) scripted(w http.ResponseWriter, key string) bool { + f.mu.Lock() + f.attempts[key]++ + var code int + if next := f.script[key]; len(next) > 0 { + code, f.script[key] = next[0], next[1:] + } + f.mu.Unlock() + if code == 0 { + return false + } + writeGCSError(w, code) + return true +} + +func (f *fakeGCS) unescape(s string) string { + name, err := url.PathUnescape(s) + if err != nil { + f.t.Errorf("unescaping %q: %v", s, err) + } + return name +} + +func writeGCSError(w http.ResponseWriter, code int) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + fmt.Fprintf(w, `{"error":{"code":%d,"message":%q}}`, code, http.StatusText(code)) +} + +// callCtx bounds a call, so a retry loop that never gives up fails the test +// instead of hanging it. +func callCtx(t *testing.T) context.Context { + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + t.Cleanup(cancel) + return ctx +} + +// httpCode returns the HTTP status err carries, or 0 if it carries none. +func httpCode(err error) int { + if gerr, ok := errors.AsType[*googleapi.Error](err); ok { + return gerr.Code + } + return 0 +} + +// TestGCSDelete covers how Delete retries: transient errors until it +// succeeds or has made 7 attempts, permanent errors not at all. +func TestGCSDelete(t *testing.T) { + tests := []struct { + name string + // statuses are answered before the fake serves the delete. + statuses []int + wantCode int // HTTP status of the returned error; 0 for success + wantAttempts int + }{ + {name: "429s, then success", statuses: []int{429, 429}, wantAttempts: 3}, + // The first delete landed but its response was lost. The retry's 404 + // still means the object is gone. + {name: "503, then 404", statuses: []int{503, 404}, wantAttempts: 2}, + {name: "gives up after 7 attempts", statuses: slices.Repeat([]int{429}, 10), wantCode: 429, wantAttempts: 7}, + {name: "403 is not retried", statuses: []int{403}, wantCode: 403, wantAttempts: 1}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fake, store := newFakeGCS(t, "snap/file") + fake.respond("DELETE snap/file", tc.statuses...) + + ctx := callCtx(t) + err := store.Delete(ctx, "bucket", "snap/file") + switch { + case tc.wantCode == 0 && err != nil: + t.Fatalf("Delete() = %v, want nil", err) + case httpCode(err) != tc.wantCode: + t.Fatalf("Delete() = %v, want HTTP %d", err, tc.wantCode) + case ctx.Err() != nil: + t.Fatal("Delete() ran until its context ended, want it to stop on its own") + } + if got := fake.attemptsOf("DELETE snap/file"); got != tc.wantAttempts { + t.Errorf("Delete() made %d attempts, want %d", got, tc.wantAttempts) + } + }) + } +} + +// TestGCSDeletePrefixRetries429 replays a snapshot release during a suspend +// burst on a cold bucket: every delete gets 429s and must be retried rather +// than fail the release. +func TestGCSDeletePrefixRetries429(t *testing.T) { + uri := mustActorSnapshotURI(t, testLocation, "team-a", "actor-1", "snap-1") + _, prefix, err := objectstore.BucketPrefix(uri.Prefix()) + if err != nil { + t.Fatalf("BucketPrefix() = %v", err) + } + var objects []string + for i := range 50 { + objects = append(objects, fmt.Sprintf("%sfile-%02d", prefix, i)) + } + fake, store := newFakeGCS(t, objects...) + for _, object := range objects { + fake.respond("DELETE "+object, 429, 429) + } + + if err := objectstore.DeletePrefix(callCtx(t), store, uri.Prefix()); err != nil { + t.Fatalf("DeletePrefix() = %v, want nil", err) + } + for _, object := range objects { + if got := fake.attemptsOf("DELETE " + object); got != 3 { + t.Errorf("deleting %s took %d attempts, want 3", object, got) + } + } + if left := fake.names(); len(left) != 0 { + t.Errorf("DeletePrefix() left %d objects, want 0", len(left)) + } +} + +// TestGCSCopyRetries429 covers a copy through 429s. The copier takes its retry +// settings from the destination handle, so this fails if only the source has +// them. +func TestGCSCopyRetries429(t *testing.T) { + fake, store := newFakeGCS(t, "snap/file") + fake.respond("COPY tag/file", 429, 429) + + if err := store.Copy(callCtx(t), "bucket", "snap/file", "bucket", "tag/file"); err != nil { + t.Fatalf("Copy() = %v, want nil", err) + } + if got := fake.attemptsOf("COPY tag/file"); got != 3 { + t.Errorf("Copy() made %d attempts, want 3", got) + } + if diff := cmp.Diff([]string{"snap/file", "tag/file"}, fake.names()); diff != "" { + t.Errorf("objects after Copy() differ (-want +got):\n%s", diff) + } +} + +// TestGCSListGivesUp covers a list that only ever gets 429s. The default +// policy would retry it until the context ends, so List must give up on its +// own. +func TestGCSListGivesUp(t *testing.T) { + fake, store := newFakeGCS(t, "snap/file") + fake.respond("LIST snap/", slices.Repeat([]int{429}, 10)...) + + ctx := callCtx(t) + _, err := store.List(ctx, "bucket", "snap/") + switch { + case httpCode(err) != http.StatusTooManyRequests: + t.Fatalf("List() = %v, want HTTP 429", err) + case ctx.Err() != nil: + t.Fatal("List() ran until its context ended, want it to give up on its own") + } + if got := fake.attemptsOf("LIST snap/"); got != 7 { + t.Errorf("List() made %d attempts, want 7", got) + } +} diff --git a/pkg/objectstorage/gcs.go b/pkg/objectstorage/gcs.go index 1445f406a1..6aa9efe9a4 100644 --- a/pkg/objectstorage/gcs.go +++ b/pkg/objectstorage/gcs.go @@ -86,16 +86,24 @@ func NewGCSClient(ctx context.Context, opts ...option.ClientOption) (ObjectStora // snapshot. RetryAlways is safe for this client because every write targets a // unique name (snapshot UUID directories, runID-suffixed part names) or is // idempotent (compose and copy from fixed sources, delete). +// +// Nothing caps the attempts: an operation retries until its context ends, so +// every caller must bound its context. Part cleanup uses cleanupTimeout. One +// exception: an upload the SDK sends as resumable (one at least as large as its +// Writer's ChunkSize, rounded up to 256 KiB) also gives up once a chunk has +// been retrying for the Writer's ChunkRetryDeadline, 32s by default. func setRetry(c *storage.Client) { c.SetRetry( storage.WithPolicy(storage.RetryAlways), - // Suspend is latency-sensitive, so bound the worst case: at most 5 - // attempts, under 5s of backoff sleep in total (250ms+500ms+1s+2s). - storage.WithBackoff(gax.Backoff{Initial: 250 * time.Millisecond, Max: 2 * time.Second, Multiplier: 2}), - storage.WithMaxAttempts(5), + storage.WithBackoff(retryBackoff), ) } +// retryBackoff is how long setRetry waits between attempts. Suspend is +// latency-sensitive, so the first retry comes within 250ms, and no wait is +// longer than 5s. +var retryBackoff = gax.Backoff{Initial: 250 * time.Millisecond, Max: 5 * time.Second, Multiplier: 2} + // supportsStreamingPut is the streamingPutter marker: the GCS client's PutObject // accepts a non-seekable streaming body without buffering (it copies the reader // straight into a storage.Writer — no Content-Length / signing requirement), so diff --git a/pkg/objectstorage/gcs_retry_test.go b/pkg/objectstorage/gcs_retry_test.go index fe1c089c3f..5112d926af 100644 --- a/pkg/objectstorage/gcs_retry_test.go +++ b/pkg/objectstorage/gcs_retry_test.go @@ -18,11 +18,16 @@ import ( "context" "fmt" "io" + "math" "net/http" "net/http/httptest" "strings" "sync" "testing" + "time" + + "cloud.google.com/go/storage" + "github.com/googleapis/gax-go/v2" ) // fakeGCSClient returns a production-configured client bound to srv. @@ -38,58 +43,255 @@ func fakeGCSClient(t *testing.T, srv *httptest.Server) *gcsClient { return g } -// A transient 429 on an upload must be retried, not surfaced: GCS sheds write -// bursts with rateLimitExceeded while it scales a bucket's key ranges, and the -// default client policy (RetryIdempotent) would fail the whole snapshot on the -// first one because plain object writes carry no precondition. -func TestPutObjectRetriesTransient429(t *testing.T) { +// serveGCS points the storage client at a fake GCS that serves uploads, +// composes, copies and deletes. It answers the first fail[kind] requests of +// each kind ("upload", "compose", "copy" or "delete") with 429 and serves the +// rest. It returns a func that reports how many requests of a kind it has seen. +func serveGCS(t *testing.T, fail map[string]int) (attempts func(kind string) int) { + t.Helper() var ( - mu sync.Mutex - attempts int - srvURL string + mu sync.Mutex + seen = map[string]int{} + srvURL string ) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var kind string switch { case strings.Contains(r.URL.Path, "/upload/"): + kind = "upload" + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/compose"): + kind = "compose" + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/rewriteTo/"): + kind = "copy" + case r.Method == http.MethodDelete: + kind = "delete" + } + if kind != "" { mu.Lock() - attempts++ - n := attempts + seen[kind]++ + throttled := seen[kind] <= fail[kind] mu.Unlock() - if n == 1 { + if throttled { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusTooManyRequests) fmt.Fprint(w, `{"error":{"code":429,"message":"Your request distribution is too uneven across the key-ranges in your bucket.","errors":[{"reason":"rateLimitExceeded"}]}}`) return } - _, _ = io.Copy(io.Discard, r.Body) - if r.URL.Query().Get("uploadType") == "resumable" { - // Resumable initiation: hand back a session for the chunk PUTs. - w.Header().Set("Location", srvURL+"/upload-session") - return - } - // Single-shot multipart upload: done in one request. + } + _, _ = io.Copy(io.Discard, r.Body) + switch { + case kind == "delete": + w.WriteHeader(http.StatusNoContent) + case kind == "copy": + // The whole copy in one rewrite call. w.Header().Set("Content-Type", "application/json") - fmt.Fprint(w, `{"name":"snap/pages.img.zstd","bucket":"snapshots"}`) - case r.URL.Path == "/upload-session": - _, _ = io.Copy(io.Discard, r.Body) + fmt.Fprint(w, `{"done":true,"resource":{"name":"snap/pages.img.zstd","bucket":"snapshots"}}`) + case kind == "upload" && r.URL.Query().Get("uploadType") == "resumable": + // Resumable initiation: hand back a session for the chunk PUTs. + w.Header().Set("Location", srvURL+"/upload-session") + case kind != "", r.URL.Path == "/upload-session": + // A single-shot upload, a compose or a session's chunk: each is + // done in one request. w.Header().Set("Content-Type", "application/json") fmt.Fprint(w, `{"name":"snap/pages.img.zstd","bucket":"snapshots"}`) default: http.NotFound(w, r) } })) - defer srv.Close() + t.Cleanup(srv.Close) srvURL = srv.URL - store := fakeGCSClient(t, srv) - ctx := context.Background() + // The storage client routes everything, uploads included, at the emulator. + t.Setenv("STORAGE_EMULATOR_HOST", srv.URL) + return func(kind string) int { + mu.Lock() + defer mu.Unlock() + return seen[kind] + } +} + +// newTestGCSClient returns a client from NewGCSClient, closed with its +// connection pool when t ends. +func newTestGCSClient(t *testing.T) ObjectStorage { + t.Helper() + store, err := NewGCSClient(context.Background()) + if err != nil { + t.Fatalf("storage client: %v", err) + } + g := store.(*gcsClient) + t.Cleanup(func() { + g.client.Close() + for _, c := range g.pool { + c.Close() + } + }) + return store +} + +// fastRetries makes clients built after it back off 1ms between attempts +// until t ends, so a test can drive many retries quickly. +func fastRetries(t *testing.T) { + old := retryBackoff + retryBackoff = gax.Backoff{Initial: time.Millisecond, Max: time.Millisecond} + t.Cleanup(func() { retryBackoff = old }) +} + +// A transient 429 on an upload must be retried, not surfaced: GCS sheds write +// bursts with rateLimitExceeded while it scales a bucket's key ranges, and the +// default client policy (RetryIdempotent) would fail the whole snapshot on the +// first one because plain object writes carry no precondition. +func TestPutObjectRetriesTransient429(t *testing.T) { + tests := []struct { + name string + fail int // upload requests answered with 429 before one succeeds + }{ + {name: "one 429", fail: 1}, + {name: "six 429s", fail: 6}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fastRetries(t) + attempts := serveGCS(t, map[string]int{"upload": tc.fail}) + store := newTestGCSClient(t) + // Bounded, so an upload that never stops retrying fails the test + // instead of hanging it. + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() + + if err := store.PutObject(ctx, "snapshots", "snap/pages.img.zstd", strings.NewReader("payload")); err != nil { + t.Fatalf("PutObject after %d transient 429s: %v", tc.fail, err) + } + if got, want := attempts("upload"), tc.fail+1; got != want { + t.Fatalf("upload attempts = %d, want %d (%d 429s, one success)", got, want, tc.fail) + } + }) + } +} + +// An upload that only ever gets 429s keeps retrying until its context ends, +// as setRetry's comment says, and then fails. +func TestPutObjectRetriesUntilDeadline(t *testing.T) { + serveGCS(t, map[string]int{"upload": math.MaxInt}) + store := newTestGCSClient(t) + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + + err := store.PutObject(ctx, "snapshots", "snap/pages.img.zstd", strings.NewReader("payload")) + if err == nil || ctx.Err() == nil { + t.Fatalf("PutObject() = %v before its context ended, want it to retry until then", err) + } +} + +// Compose has no attempt cap either, so it gets through six 429s. +func TestComposeRetriesPast5Attempts(t *testing.T) { + fastRetries(t) + attempts := serveGCS(t, map[string]int{"compose": 6}) + bkt := newTestGCSClient(t).(*gcsClient).client.Bucket("snapshots") + parts := []*storage.ObjectHandle{bkt.Object("snap/part-0"), bkt.Object("snap/part-1")} + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() + + if err := composeAll(ctx, bkt, "snap/pages.img.zstd", parts, "run"); err != nil { + t.Fatalf("composeAll() after six 429s: %v", err) + } + if got := attempts("compose"); got != 7 { + t.Errorf("compose attempts = %d, want 7", got) + } +} + +// Scratch deletes that keep failing stop at cleanupTimeout, even though the +// upload's own context is done by the time its cleanup runs. +func TestRemoveScratchStopsAtCleanupTimeout(t *testing.T) { + fastRetries(t) + old := cleanupTimeout + cleanupTimeout = 200 * time.Millisecond + t.Cleanup(func() { cleanupTimeout = old }) + serveGCS(t, map[string]int{"delete": math.MaxInt}) + bkt := newTestGCSClient(t).(*gcsClient).client.Bucket("snapshots") + objs := []*storage.ObjectHandle{bkt.Object("snap/part-0"), bkt.Object("snap/part-1")} + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + start := time.Now() + done := make(chan struct{}) + go func() { + defer close(done) + removeScratch(ctx, objs) + }() + select { + case <-done: + // Returning sooner would mean the deletes ran on the upload's context. + if elapsed := time.Since(start); elapsed < 200*time.Millisecond { + t.Fatalf("removeScratch() returned after %v, want it to retry for cleanupTimeout", elapsed) + } + case <-time.After(10 * time.Second): + t.Fatal("removeScratch() still running after 10s, want it to stop at cleanupTimeout") + } +} + +// A part cleanup that fails does not fail an upload whose object is already +// in place. +func TestPutCompositeSucceedsWhenCleanupFails(t *testing.T) { + fastRetries(t) + old := cleanupTimeout + cleanupTimeout = 200 * time.Millisecond + t.Cleanup(func() { cleanupTimeout = old }) + attempts := serveGCS(t, map[string]int{"delete": math.MaxInt}) + g := newTestGCSClient(t).(*gcsClient) + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() - if err := store.PutObject(ctx, "snapshots", "snap/pages.img.zstd", strings.NewReader("payload")); err != nil { - t.Fatalf("PutObject after a transient 429: %v", err) + // One small part: composeAll copies it into place, then every delete of + // it fails. + if err := g.putComposite(ctx, "snapshots", "snap/pages.img.zstd", strings.NewReader("payload"), strings.NewReader("")); err != nil { + t.Fatalf("putComposite() = %v, want nil although its part cleanup failed", err) } + if attempts("copy") == 0 || attempts("delete") == 0 { + t.Fatalf("copies = %d, deletes = %d, want both to have run", attempts("copy"), attempts("delete")) + } +} + +// removeScratch deletes in parallel, at most cleanupConcurrency at once. One +// at a time, a large upload's parts would leave little of cleanupTimeout for +// retries. +func TestRemoveScratchRunsInParallel(t *testing.T) { + var ( + mu sync.Mutex + inFlight, peak, deletions int + ) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + http.NotFound(w, r) + return + } + mu.Lock() + inFlight++ + peak = max(peak, inFlight) + mu.Unlock() + // Long enough for parallel deletes to overlap. + time.Sleep(20 * time.Millisecond) + mu.Lock() + inFlight-- + deletions++ + mu.Unlock() + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(srv.Close) + t.Setenv("STORAGE_EMULATOR_HOST", srv.URL) + bkt := newTestGCSClient(t).(*gcsClient).client.Bucket("snapshots") + var objs []*storage.ObjectHandle + for i := range 4 * cleanupConcurrency { + objs = append(objs, bkt.Object(fmt.Sprintf("snap/part-%02d", i))) + } + + removeScratch(t.Context(), objs) + mu.Lock() defer mu.Unlock() - if attempts != 2 { - t.Fatalf("upload attempts = %d, want 2 (one 429, one retry)", attempts) + if deletions != len(objs) { + t.Errorf("removeScratch() made %d deletes, want %d", deletions, len(objs)) + } + if peak < 2 || peak > cleanupConcurrency { + t.Errorf("removeScratch() ran %d deletes at once, want 2 to %d", peak, cleanupConcurrency) } } diff --git a/pkg/objectstorage/gcscompose.go b/pkg/objectstorage/gcscompose.go index 545339dfd4..d26dea0857 100644 --- a/pkg/objectstorage/gcscompose.go +++ b/pkg/objectstorage/gcscompose.go @@ -22,6 +22,8 @@ import ( "fmt" "io" "log/slog" + "sync" + "time" "cloud.google.com/go/storage" "golang.org/x/sync/errgroup" @@ -92,7 +94,7 @@ const ( // putComposite uploads head followed by rest as concurrent parts, then composes them // into object. Only for objects past uploadCompositeMin; putSingle handles the rest. -func (g *gcsClient) putComposite(ctx context.Context, bucket, object string, head io.Reader, rest io.Reader) (err error) { +func (g *gcsClient) putComposite(ctx context.Context, bucket, object string, head io.Reader, rest io.Reader) error { bkt := g.client.Bucket(bucket) runID, err := newRunID() if err != nil { @@ -100,7 +102,8 @@ func (g *gcsClient) putComposite(ctx context.Context, bucket, object string, hea } var parts []*storage.ObjectHandle - defer func() { err = removeScratch(ctx, parts, err) }() + // Parts are scratch either way. A closure, because parts grows below. + defer func() { removeScratch(ctx, parts) }() // The stream is read in order here and whole parts handed to the uploaders; buffers // cycle through free so the peak stays at uploadConcurrency of them. @@ -155,17 +158,49 @@ func newRunID() (string, error) { return hex.EncodeToString(b[:]), nil } -// removeScratch deletes objs, which are scratch whether or not the upload succeeded. -// It returns err, or the first delete failure if err is nil, so cleanup never masks -// the real error. -func removeScratch(ctx context.Context, objs []*storage.ObjectHandle, err error) error { +// cleanupTimeout bounds removeScratch. Cleanup runs on a context detached from +// the upload's, which may be the reason it is unwinding, and setRetry retries +// until the context ends. +var cleanupTimeout = 30 * time.Second + +// cleanupConcurrency is how many objects removeScratch deletes at once, as +// internal/objectstore's DeletePrefix does. Deleting a large upload's parts one +// at a time would leave little of cleanupTimeout for retries. +const cleanupConcurrency = 8 + +// removeScratch deletes objs, which are scratch whether or not the upload +// succeeded, cleanupConcurrency at a time and all within one cleanupTimeout. It +// is best-effort and never fails the upload: objects it cannot delete are left +// behind, and how many, with the first error, is logged. +func removeScratch(ctx context.Context, objs []*storage.ObjectHandle) { + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout) + defer cancel() + var ( + mu sync.Mutex + left int + first error + g errgroup.Group + ) + g.SetLimit(cleanupConcurrency) for _, o := range objs { - if delErr := o.Delete(context.WithoutCancel(ctx)); delErr != nil && - !errors.Is(delErr, storage.ErrObjectNotExist) && err == nil { - err = fmt.Errorf("while removing scratch object %q: %w", o.ObjectName(), delErr) - } + g.Go(func() error { + err := o.Delete(ctx) + if err == nil || errors.Is(err, storage.ErrObjectNotExist) { + return nil + } + mu.Lock() + defer mu.Unlock() + left++ + if first == nil { + first = fmt.Errorf("while removing scratch object %q: %w", o.ObjectName(), err) + } + return nil + }) + } + _ = g.Wait() + if left > 0 { + slog.WarnContext(ctx, "Leaving scratch objects behind", slog.Int("count", left), slog.Any("err", first)) } - return err } // writeObject writes data to obj in one request. @@ -183,12 +218,12 @@ func writeObject(ctx context.Context, obj *storage.ObjectHandle, data []byte, si // composeAll composes parts into object, folding in rounds when there are more parts // than GCS accepts in one call. The intermediates it creates are removed on return; the // caller removes parts. -func composeAll(ctx context.Context, bkt *storage.BucketHandle, object string, parts []*storage.ObjectHandle, runID string) (err error) { +func composeAll(ctx context.Context, bkt *storage.BucketHandle, object string, parts []*storage.ObjectHandle, runID string) error { if len(parts) == 0 { return fmt.Errorf("no parts to compose into %q", object) } var mids []*storage.ObjectHandle - defer func() { err = removeScratch(ctx, mids, err) }() + defer func() { removeScratch(ctx, mids) }() round := 0 for len(parts) > maxComposeSources { diff --git a/pkg/objectstorage/gcssparse.go b/pkg/objectstorage/gcssparse.go index 16224b1993..d5f6369d77 100644 --- a/pkg/objectstorage/gcssparse.go +++ b/pkg/objectstorage/gcssparse.go @@ -69,7 +69,7 @@ func (g *gcsClient) PutSparseFile(ctx context.Context, bucket, object string, f for i := range groups { parts[i] = bkt.Object(fmt.Sprintf("%s.part-%s-%04d", object, runID, i)) } - defer func() { err = removeScratch(ctx, parts, err) }() + defer removeScratch(ctx, parts) grp, gctx := errgroup.WithContext(ctx) for i, ranges := range groups { From 4f71f0ea2dff6080b35ab91096c127c141227c35 Mon Sep 17 00:00:00 2001 From: Jing Date: Wed, 21 Oct 2026 00:42:04 +0000 Subject: [PATCH 32/63] Run gVisor sandboxes without a pause container (#2773) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each actor sandbox had a root container that ran /pause from a pause image. atelet pulled that image and built its bundle, and every SandboxConfig had to name it. runsc can now boot a sandbox with no root container and checkpoint and restore it, so the pause container is not necessary. ateworker-gvisor now creates the _pause sandbox with -no-root-container and writes its spec with no process, rootfs, or mounts. The spec keeps the network namespace, cgroup, resource limits, and durable-dir mount hints. Application containers do not change. Remove pauseImage from the SandboxConfig CRD, from SandboxAssets in the atelet API, from the snapshot manifest, from the sandbox runtime hash, and from asset prewarm. Remove the pause_rootfs restore phase. > The PR title becomes the release note. Write it for users: what changed for them, not how the code changed. #### Breaking change - pauseImage is removed from SandboxConfig; - the minimum gVisor release is #15551 (the 2026-10-17 nightly or later); - old snapshots don't restore, so golden snapshots need recutting; - _pause is now _sandbox, which changes the log label, the kubectl -c filter, the cgroup leaf and the phase names. Fixes #967 > It's a good idea to open an issue first for discussion. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --------- Signed-off-by: Jing Chen --- .golangci-kal.yaml | 2 +- benchmarking/analysis/README.md | 2 +- benchmarking/analysis/phase_report.py | 2 +- cmd/ate-setup/differences.md | 6 +- cmd/ate-setup/internal/images/prebuilt.go | 14 +- .../internal/images/prebuilt_test.go | 8 +- .../workerpool_sandboxconfig_test.go | 5 - .../internal/apivalidation/validation_test.go | 6 +- cmd/atelet/lifecycle_test.go | 99 ++++++------ cmd/atelet/main.go | 51 ++----- cmd/atelet/main_test.go | 77 +--------- cmd/atelet/runtime_prewarm.go | 45 ++---- cmd/atelet/runtime_prewarm_test.go | 120 ++++----------- cmd/atelet/sandbox_assets.go | 22 +-- cmd/atelet/sandbox_runtime_index.go | 12 +- cmd/atelet/sandbox_runtime_index_test.go | 7 +- cmd/atelet/snapshot_transfer_test.go | 8 +- cmd/atelet/worker_sandbox_syncer.go | 5 +- cmd/atelet/worker_sandbox_syncer_test.go | 6 +- cmd/atelet/worker_sandbox_versions.go | 77 +++------- cmd/atelet/worker_sandbox_versions_test.go | 125 +++------------ cmd/ateworker-gvisor/ckptcgroup.go | 24 +-- cmd/ateworker-gvisor/ckptcgroup_test.go | 34 ++--- cmd/ateworker-gvisor/cleanup_test.go | 10 +- cmd/ateworker-gvisor/crash_test.go | 2 +- cmd/ateworker-gvisor/main.go | 106 ++++++------- cmd/ateworker-gvisor/main_test.go | 12 +- cmd/ateworker-gvisor/phaselog.go | 36 +++-- cmd/ateworker-gvisor/phaselog_test.go | 48 +++--- cmd/ateworker-gvisor/runsc.go | 60 ++++---- cmd/ateworker-gvisor/runsc_test.go | 143 +++++++++++++----- cmd/ateworker-gvisor/sandboxexit.go | 6 +- cmd/ateworker-gvisor/sandboxexit_test.go | 2 +- cmd/ateworker-gvisor/stats.go | 18 +-- cmd/ateworker-gvisor/stats_test.go | 16 +- cmd/kubectl-ate/internal/cmd/actor.go | 4 +- cmd/kubectl-ate/internal/cmd/actor_test.go | 34 ++--- docs/api-guide.md | 6 +- docs/architecture.md | 2 +- docs/glossary.md | 9 +- docs/observability.md | 10 +- docs/sandbox-asset-cache.md | 10 +- .../ateworkervalidation/validation_test.go | 2 +- internal/hardware/hardware.go | 3 +- internal/imagecache/imagecache.go | 7 +- internal/ocispec/gvisor.go | 11 +- internal/ocispec/gvisor_test.go | 10 +- internal/ocispec/ocispec.go | 42 ++--- internal/ocispec/ocispec_test.go | 6 +- internal/ocispec/parity_test.go | 39 ++++- internal/proto/ateletpb/atelet.pb.go | 14 +- internal/proto/ateletpb/atelet.proto | 14 +- internal/proto/ateworkerpb/ateworker.pb.go | 4 +- internal/proto/ateworkerpb/ateworker.proto | 4 +- .../generated/ate.dev_sandboxconfigs.yaml | 29 +--- .../ate-install/sandboxconfig-gvisor.yaml | 6 +- pkg/api/v1alpha1/sandboxconfig_types.go | 27 +--- .../v1alpha1/sandboxconfig_validation_test.go | 68 ++------- 58 files changed, 611 insertions(+), 966 deletions(-) diff --git a/.golangci-kal.yaml b/.golangci-kal.yaml index 8ae78cb43e..cc773f3336 100644 --- a/.golangci-kal.yaml +++ b/.golangci-kal.yaml @@ -83,7 +83,7 @@ linters: - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' text: '^(optionalfields|requiredfields): .*\bSandboxConfigSpec\.SandboxClass\b' - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' - text: '^(nomaps|optionalfields): .*\bSandboxVersionConfig\.(Assets|PauseImage)\b' + text: '^(nomaps|optionalfields): .*\bSandboxVersionConfig\.Assets\b' - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' text: '^(nonpointerstructs|requiredfields): .*\bSandboxConfig\.Spec\b' - path: 'pkg/api/v1alpha1/csidriverconfig_types\.go' diff --git a/benchmarking/analysis/README.md b/benchmarking/analysis/README.md index 3109de03fb..b2a6735bec 100644 --- a/benchmarking/analysis/README.md +++ b/benchmarking/analysis/README.md @@ -85,7 +85,7 @@ restore between `volume_mount`, `manifest_fetch`, `sandbox_assets`, `prep` / `pause` / `snapshot` / `durable_dir` / `rootfs_upper` / `teardown` on a checkpoint and `prep` / … / `vm_restore` / `wakeup_probe` on a restore; for gVisor `pause` / `checkpoint` / `fs_checkpoint` / `resume` / `teardown` and -`prep` / `net_setup` / … / `pause_restore` / `app_restore` / +`prep` / `net_setup` / … / `sandbox_restore` / `app_restore` / `wakeup_probe` / `activate` (the full lists are in `cmd/ateworker-*/phaselog.go`). The ateworker records carry no sandbox class or kind of their own; a paired one takes both diff --git a/benchmarking/analysis/phase_report.py b/benchmarking/analysis/phase_report.py index 2d7c154f40..edbe814b8c 100755 --- a/benchmarking/analysis/phase_report.py +++ b/benchmarking/analysis/phase_report.py @@ -94,7 +94,7 @@ # ateworker restore: micro-VM (cmd/ateworker-microvm/phaselog.go) and gVisor # (cmd/ateworker-gvisor/phaselog.go), each in its own sequential order "prep", "egress_prepare", "net_setup", "bundles", "upper_join", "lowers", - "pause_rootfs", "pause_create", "pause_restore", + "sandbox_create", "sandbox_restore", "app_rootfs", "app_create", "app_restore", "tap", "vmm_launch", "vm_restore", "resume", "wakeup_probe", "activate", # atelet + ateworker checkpoint diff --git a/cmd/ate-setup/differences.md b/cmd/ate-setup/differences.md index d03d7b2385..e54eb99e31 100644 --- a/cmd/ate-setup/differences.md +++ b/cmd/ate-setup/differences.md @@ -121,9 +121,9 @@ The rewritten reference then keeps that tag and adds the digest it names, as tagged release. Resolving the digest is one HEAD request per image, cached for the install, so `--image-repo` needs read access to the registry from wherever `ate-setup` runs rather than only from the cluster. It is not optional: an -ActorTemplate's container image, an image volume's reference, and a -SandboxConfig's `pauseImage` each carry the CEL rule `self.contains('@')`, so an -unpinned reference is rejected by admission and every demo fails to deploy. +ActorTemplate's container image and an image volume's reference each carry the +CEL rule `self.contains('@')`, so an unpinned reference is rejected by admission +and every demo fails to deploy. Pinning also gives a pre-built install the property a `ko` install had for free, that a tag moving afterwards cannot change what is running. diff --git a/cmd/ate-setup/internal/images/prebuilt.go b/cmd/ate-setup/internal/images/prebuilt.go index 63656b29ad..873dcb01e8 100644 --- a/cmd/ate-setup/internal/images/prebuilt.go +++ b/cmd/ate-setup/internal/images/prebuilt.go @@ -79,13 +79,13 @@ func (p *Prebuilt) ResolveBytes(ctx context.Context, manifest []byte) ([]byte, e // it, such as envoy-dataplane: a release publishes that under the same // repository and tag as the rest, so it is found the same way. // -// The digest is not decoration. An ActorTemplate's container image, an image -// volume's reference, and a SandboxConfig's pauseImage each carry the CEL rule -// self.contains('@'), so a tag on its own is rejected by admission and every -// demo would fail to deploy. Pinning also restores what ko gave the control -// plane deployments for free: an install that cannot shift under a tag someone -// moves later. Keeping the tag alongside the digest is ko's own output shape, -// and leaves the reference legible in `kubectl get`. +// The digest is not decoration. An ActorTemplate's container image and an image +// volume's reference each carry the CEL rule self.contains('@'), so a tag on +// its own is rejected by admission and every demo would fail to deploy. Pinning +// also restores what ko gave the control plane deployments for free: an install +// that cannot shift under a tag someone moves later. Keeping the tag alongside +// the digest is ko's own output shape, and leaves the reference legible in +// `kubectl get`. func (p *Prebuilt) Pin(ctx context.Context, image string) (string, error) { tagged := p.src.Repo + "/" + image + ":" + p.src.Tag // A tag that already carries one is pinned as it stands. Looking it up diff --git a/cmd/ate-setup/internal/images/prebuilt_test.go b/cmd/ate-setup/internal/images/prebuilt_test.go index 1264caa09a..41a36bf181 100644 --- a/cmd/ate-setup/internal/images/prebuilt_test.go +++ b/cmd/ate-setup/internal/images/prebuilt_test.go @@ -370,10 +370,10 @@ func TestResolveEveryInstallManifest(t *testing.T) { if refs := koRefPattern.FindAllString(string(out), -1); len(refs) > 0 { t.Errorf("unresolved references survived: %v", refs) } - // Every reference has to name a digest. ActorTemplate images, - // image volume references, and a SandboxConfig's pauseImage all - // carry the CEL rule self.contains('@'), so an unpinned reference - // is not applied at all, it is rejected by admission. + // Every reference has to name a digest. ActorTemplate images and + // image volume references carry the CEL rule self.contains('@'), so + // an unpinned reference is not applied at all, it is rejected by + // admission. for _, ref := range resolvedRef.FindAllString(string(out), -1) { if !strings.Contains(ref, "@sha256:") { t.Errorf("%s is not pinned to a digest", ref) diff --git a/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig_test.go b/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig_test.go index f01ba39756..e5b1e7dbfc 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig_test.go @@ -48,8 +48,6 @@ import ( // testGvisorDefault is the gvisor class default TestMain creates. const testGvisorDefault = "gvisor-default-test" -const testPauseImage = "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" - // testHoldFinalizer keeps a SandboxConfig on the API server after it is // deleted, so a test can observe how a config that is being deleted is treated. const testHoldFinalizer = "test.ate.dev/hold" @@ -2397,9 +2395,6 @@ func TestAddProtectionFinalizer(t *testing.T) { func makeSandboxConfig(name string, class atev1alpha1.SandboxClass, isDefault bool) *atev1alpha1.SandboxConfig { v := atev1alpha1.SandboxVersionConfig{Name: "v1"} - if class == atev1alpha1.SandboxClassGvisor { - v.PauseImage = testPauseImage - } sc := &atev1alpha1.SandboxConfig{ ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: atev1alpha1.SandboxConfigSpec{ diff --git a/cmd/atelet/internal/apivalidation/validation_test.go b/cmd/atelet/internal/apivalidation/validation_test.go index fff8d0b2b6..9fcfdf371b 100644 --- a/cmd/atelet/internal/apivalidation/validation_test.go +++ b/cmd/atelet/internal/apivalidation/validation_test.go @@ -477,7 +477,7 @@ func TestValidateCheckpointRequest(t *testing.T) { }, { name: "spec errors surface at the spec's path", obj: valid(func(r *ateletpb.CheckpointRequest) { - r.Spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "_pause", Image: testDigestImage}}} + r.Spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "_sandbox", Image: testDigestImage}}} }), want: field.ErrorList{field.Invalid(field.NewPath("spec", "containers").Index(0).Child("name"), nil, "").WithOrigin("format=k8s-short-name")}, }, { @@ -652,7 +652,7 @@ func TestValidateRestoreRequest(t *testing.T) { }, { name: "spec errors surface at the spec's path", obj: valid(func(r *ateletpb.RestoreRequest) { - r.Spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "_pause", Image: testDigestImage}}} + r.Spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "_sandbox", Image: testDigestImage}}} }), want: field.ErrorList{field.Invalid(field.NewPath("spec", "containers").Index(0).Child("name"), nil, "").WithOrigin("format=k8s-short-name")}, }, { @@ -939,7 +939,7 @@ func TestValidateTerminateRequest(t *testing.T) { }, { name: "spec errors surface at the spec's path", obj: valid(func(r *ateletpb.TerminateRequest) { - r.Spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "_pause", Image: testDigestImage}}} + r.Spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "_sandbox", Image: testDigestImage}}} }), want: field.ErrorList{field.Invalid(field.NewPath("spec", "containers").Index(0).Child("name"), nil, "").WithOrigin("format=k8s-short-name")}, }, { diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index 898a16e066..9ae9efeb43 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -16,6 +16,7 @@ package main import ( "context" + "crypto/sha256" "errors" "fmt" "net" @@ -246,7 +247,7 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) { systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } - sandboxRec := testSandboxRecord(release, pinnedImage) + sandboxRec := testSandboxRecord(release) spec := &ateletpb.WorkloadSpec{ Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, } @@ -422,26 +423,27 @@ func TestRestoreUsesRequestSandboxRuntime(t *testing.T) { host := imageVolumeTestRegistry(t) image := host + "/actor:v1" pinnedImage := pushTestImage(t, image, singleFileLayer(t, "bin/app", "app")) - checkpointPause := pushTestImage(t, host+"/pause:v1", singleFileLayer(t, "pause", "pause-v1")) - restorePause := pushTestImage(t, host+"/pause:v2", singleFileLayer(t, "pause", "pause-v2")) - release := newGVisorTestRelease(t) + // The two releases differ in content, so they have distinct bundle hashes. + release := func(runsc string) gvisorTestRelease { + tarball := gvisorReleaseTarballWith(t, runsc) + return gvisorTestRelease{tarball: tarball, entry: assetEntry{URL: "gs://test-bucket/gvisor.tar", SHA256: fmt.Sprintf("%x", sha256.Sum256(tarball))}} + } + checkpointRelease, restoreRelease := release("runsc v1"), release("runsc v2") s := &Atelet{ workerDialer: newWorkerDialer(1), imageCache: newImageVolumeStore(t), sandboxAssetCache: newTestAssetCache(t), - anonGCSClient: fakeObjectStorage{data: release.tarball}, + anonGCSClient: fakeObjectStorage{data: checkpointRelease.tarball}, systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } - recWithPause := func(pause string) *sandboxAssetsRecord { - return testSandboxRecord(release, pause) - } + checkpointRec, restoreRec := testSandboxRecord(checkpointRelease), testSandboxRecord(restoreRelease) spec := &ateletpb.WorkloadSpec{ Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, } - advertiseRuntime(t, s, recWithPause(checkpointPause)) + advertiseRuntime(t, s, checkpointRec) if _, err := s.Run(ctx, &ateletpb.RunRequest{ Atespace: atespace, ActorName: actorName, @@ -449,7 +451,7 @@ func TestRestoreUsesRequestSandboxRuntime(t *testing.T) { ActorTemplateAtespace: "default", ActorTemplateName: "counter", WorkerPodUid: workerPodUID, - SandboxRuntime: runtimeFor(t, recWithPause(checkpointPause)), + SandboxRuntime: runtimeFor(t, checkpointRec), Spec: spec, }); err != nil { t.Fatalf("Run: %v", err) @@ -477,11 +479,12 @@ func TestRestoreUsesRequestSandboxRuntime(t *testing.T) { if err != nil { t.Fatalf("unmarshalling snapshot manifest: %v", err) } - if manifestRec.PauseImage != checkpointPause { - t.Fatalf("manifest pause image = %q, want %q", manifestRec.PauseImage, checkpointPause) + if got, want := manifestRec.Assets[gvisorAssetName].SHA256, checkpointRelease.entry.SHA256; got != want { + t.Fatalf("manifest gvisor sha256 = %q, want %q", got, want) } - advertiseRuntime(t, s, recWithPause(restorePause)) + s.anonGCSClient = fakeObjectStorage{data: restoreRelease.tarball} + advertiseRuntime(t, s, restoreRec) if _, err := s.Restore(ctx, &ateletpb.RestoreRequest{ Atespace: atespace, ActorName: actorName, @@ -489,7 +492,7 @@ func TestRestoreUsesRequestSandboxRuntime(t *testing.T) { ActorTemplateAtespace: "default", ActorTemplateName: "counter", WorkerPodUid: workerPodUID, - SandboxRuntime: runtimeFor(t, recWithPause(restorePause)), + SandboxRuntime: runtimeFor(t, restoreRec), Spec: spec, Snapshot: lifecycleSnapshot(actorUID, snapshotName, 1, ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY), }); err != nil { @@ -500,8 +503,8 @@ func TestRestoreUsesRequestSandboxRuntime(t *testing.T) { if err != nil { t.Fatalf("reading on-node sandbox record: %v", err) } - if got.PauseImage != restorePause { - t.Errorf("restored actor pause image = %q, want the request's %q", got.PauseImage, restorePause) + if sha, want := got.Assets[gvisorAssetName].SHA256, restoreRelease.entry.SHA256; sha != want { + t.Errorf("restored actor gvisor sha256 = %q, want the request's %q", sha, want) } } @@ -546,7 +549,7 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { release := newGVisorTestRelease(t) if tc.restore && tc.failure != "manifest" { writeLocalSnapshot(t, ateletpath.LocalSnapshotDir(actorUID, snapshotName), sandboxAssetsRecord{ - SandboxClass: "gvisor", PauseImage: testPauseImage, + SandboxClass: "gvisor", Assets: map[string]assetEntry{gvisorAssetName: release.entry}, SnapshotFiles: []string{"checkpoint.img"}, }, map[string]string{"checkpoint.img": "guest-memory"}) @@ -586,7 +589,7 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { s := newPluginHerder(t, &recordingObjectStorage{}) s.anonGCSClient = storage s.systemInfoVolumes = refresher - rec := testSandboxRecord(release, testPauseImage) + rec := testSandboxRecord(release) advertiseRuntime(t, s, rec) if tc.restore { _, err = s.Restore(ctx, &ateletpb.RestoreRequest{ @@ -674,12 +677,13 @@ func TestRunFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } spec := &ateletpb.WorkloadSpec{ + Containers: []*ateletpb.Container{{Name: "app", Image: missingImage(t)}}, Volumes: []*ateletpb.Volume{{ Name: "trust", SystemInfo: trustVolumeSpec("ca.pem"), }}, } - rec := testSandboxRecord(release, missingPauseImage(t)) + rec := testSandboxRecord(release) advertiseRuntime(t, s, rec) _, err := s.Run(t.Context(), &ateletpb.RunRequest{ Atespace: "team-a", @@ -689,7 +693,7 @@ func TestRunFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { SandboxRuntime: runtimeFor(t, rec), Spec: spec, }) - if err == nil || !strings.Contains(err.Error(), "while creating pause OCI bundle") { + if err == nil || !strings.Contains(err.Error(), `while creating "app" OCI bundle`) { t.Fatalf("Run error = %v, want the post-registration OCI preparation failure", err) } if _, err := os.ReadFile(filepath.Join(ateletpath.SystemInfoVolumeRoot(actorUID, "trust"), "ca.pem")); err != nil { @@ -700,12 +704,12 @@ func TestRunFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { } } -// missingPauseImage returns a digest-pinned reference that passes request +// missingImage returns a digest-pinned reference that passes request // validation but names a manifest the local test registry does not serve, so // the pull fails only once OCI bundle preparation starts. -func missingPauseImage(t *testing.T) string { +func missingImage(t *testing.T) string { t.Helper() - return imageVolumeTestRegistry(t) + "/pause@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + return imageVolumeTestRegistry(t) + "/app@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" } func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { @@ -714,22 +718,24 @@ func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { actorUID = "0f6c2b0e-1b7e-4b3f-9a6a-1d2c3e4f5a63" snapshotName = "pause-snap-after" ) - pauseImage := missingPauseImage(t) + image := missingImage(t) store := newCTBStore(t) store.set(t, string(testCertPEM(t))) refresher := newSystemInfoVolumeRefresher(trustbundle.NewSource(store.lister.Get, nil), nil) release := newGVisorTestRelease(t) writeLocalSnapshot(t, ateletpath.LocalSnapshotDir(actorUID, snapshotName), sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: pauseImage, Assets: map[string]assetEntry{gvisorAssetName: release.entry}, SnapshotFiles: []string{"checkpoint.img"}, }, map[string]string{"checkpoint.img": "guest-memory"}) - spec := &ateletpb.WorkloadSpec{Volumes: []*ateletpb.Volume{{ - Name: "trust", - SystemInfo: trustVolumeSpec("ca.pem"), - }}} + spec := &ateletpb.WorkloadSpec{ + Containers: []*ateletpb.Container{{Name: "app", Image: image}}, + Volumes: []*ateletpb.Volume{{ + Name: "trust", + SystemInfo: trustVolumeSpec("ca.pem"), + }}, + } s := &Atelet{ sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{data: release.tarball}, @@ -737,7 +743,7 @@ func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { systemInfoVolumes: refresher, snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } - rec := testSandboxRecord(release, pauseImage) + rec := testSandboxRecord(release) advertiseRuntime(t, s, rec) _, err := s.Restore(t.Context(), &ateletpb.RestoreRequest{ Atespace: "team-a", @@ -748,7 +754,7 @@ func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { Spec: spec, Snapshot: lifecycleSnapshot(actorUID, snapshotName, 1, ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY), }) - if err == nil || !strings.Contains(err.Error(), "while creating pause OCI bundle") { + if err == nil || !strings.Contains(err.Error(), `while creating "app" OCI bundle`) { t.Fatalf("Restore error = %v, want the post-registration OCI preparation failure", err) } if _, err := os.ReadFile(filepath.Join(ateletpath.SystemInfoVolumeRoot(actorUID, "trust"), "ca.pem")); err != nil { @@ -780,13 +786,12 @@ func TestRestorePrefersLocalAndFallsBackToObjectStorage(t *testing.T) { pinnedImage := pushTestImage(t, host+"/actor:v1", singleFileLayer(t, "bin/app", "app")) release := newGVisorTestRelease(t) - sandboxRec := testSandboxRecord(release, pinnedImage) + sandboxRec := testSandboxRecord(release) spec := &ateletpb.WorkloadSpec{ Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, } manifestRec := sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: pinnedImage, Assets: map[string]assetEntry{gvisorAssetName: release.entry}, SnapshotFiles: []string{"checkpoint.img"}, } @@ -919,10 +924,9 @@ func TestRestoreFailureRemovesStagedSnapshotFiles(t *testing.T) { host := imageVolumeTestRegistry(t) pinnedImage := pushTestImage(t, host+"/actor:v1", singleFileLayer(t, "bin/app", "app")) release := newGVisorTestRelease(t) - sandboxRec := testSandboxRecord(release, pinnedImage) + sandboxRec := testSandboxRecord(release) manifestRec := sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: pinnedImage, Assets: map[string]assetEntry{gvisorAssetName: release.entry}, SnapshotFiles: []string{"checkpoint.img"}, } @@ -993,7 +997,7 @@ func TestRestoreWithoutUsableLocalCopy(t *testing.T) { snapshotName = "pause-snap-1" ) release := newGVisorTestRelease(t) - rec := testSandboxRecord(release, testPauseImage) + rec := testSandboxRecord(release) for _, tc := range []struct { name string // unusableLocal writes a manifest that lists a file that is not there. @@ -1011,7 +1015,7 @@ func TestRestoreWithoutUsableLocalCopy(t *testing.T) { useTempNodeDirs(t) if tc.unusableLocal { writeLocalSnapshot(t, ateletpath.LocalSnapshotDir(actorUID, snapshotName), sandboxAssetsRecord{ - SandboxClass: "gvisor", PauseImage: testPauseImage, SnapshotFiles: []string{"checkpoint.img"}, + SandboxClass: "gvisor", SnapshotFiles: []string{"checkpoint.img"}, }, nil) } // The object store is empty: nothing was ever uploaded. @@ -1138,7 +1142,7 @@ func runPluginActor(t *testing.T) *pluginActor { a.s.sandboxAssetCache = newTestAssetCache(t) a.s.anonGCSClient = fakeObjectStorage{data: release.tarball} a.s.systemInfoVolumes = newSystemInfoVolumeRefresher(nil, nil) - a.rec = testSandboxRecord(release, pinnedImage) + a.rec = testSandboxRecord(release) a.spec = &ateletpb.WorkloadSpec{ Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, } @@ -1512,7 +1516,7 @@ func TestConcurrentRunsShareOneAssetDownload(t *testing.T) { systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } - sandboxRec := testSandboxRecord(release, pinnedImage) + sandboxRec := testSandboxRecord(release) spec := &ateletpb.WorkloadSpec{ Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, } @@ -2004,10 +2008,9 @@ func TestPromoteSnapshotFoundAlreadyUploaded(t *testing.T) { // testSandboxRecord is the record of a gVisor runtime whose only asset is // release, as the syncer plans it from a SandboxConfig version. -func testSandboxRecord(release gvisorTestRelease, pauseImage string) *sandboxAssetsRecord { +func testSandboxRecord(release gvisorTestRelease) *sandboxAssetsRecord { return &sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: pauseImage, Assets: map[string]assetEntry{gvisorAssetName: release.entry}, } } @@ -2021,10 +2024,7 @@ func advertiseRuntime(t *testing.T, s *Atelet, rec *sandboxAssetsRecord) { if s.runtimes == nil { s.runtimes = newRuntimeIndex() } - hash, err := versionBundleHash(rec) - if err != nil { - t.Fatalf("versionBundleHash: %v", err) - } + hash := bundleHash(rec) s.runtimes.register("test-worker-pod", advertisedClass{ class: v1alpha1.SandboxClass(rec.SandboxClass), config: "test-config", @@ -2050,7 +2050,7 @@ func TestRunAndRestoreRejectUnadvertisedRuntime(t *testing.T) { anonGCSClient: fakeObjectStorage{data: release.tarball, gets: &gets}, systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), } - rec := testSandboxRecord(release, testPauseImage) + rec := testSandboxRecord(release) for _, tc := range []struct { name string @@ -2106,10 +2106,7 @@ func TestRunAndRestoreRejectUnadvertisedRuntime(t *testing.T) { // registration, whose sandbox_bundle_hash is the hash of rec. func runtimeFor(t *testing.T, rec *sandboxAssetsRecord) *ateletpb.SandboxRuntime { t.Helper() - hash, err := versionBundleHash(rec) - if err != nil { - t.Fatalf("versionBundleHash: %v", err) - } + hash := bundleHash(rec) return &ateletpb.SandboxRuntime{ SandboxClass: rec.SandboxClass, Version: &ateletpb.VersionedSandboxCompat{ @@ -2569,7 +2566,6 @@ func TestPromoteSnapshotRejectsStaleGeneration(t *testing.T) { } { writeLocalSnapshot(t, ateletpath.LocalSnapshotDir(pluginActorUID, stale.GetSnapshotUuid()), sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: a.rec.PauseImage, SnapshotFiles: []string{"checkpoint.img"}, Fidelity: "MEMORY", }, map[string]string{"checkpoint.img": "stale-memory"}) @@ -2703,7 +2699,6 @@ func TestCheckpointSkipsStaleCommitAndKeepsNewerLocal(t *testing.T) { snap3Dir := ateletpath.LocalSnapshotDir(pluginActorUID, snap3.GetSnapshotUuid()) writeLocalSnapshot(t, snap3Dir, sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: a.rec.PauseImage, SnapshotFiles: []string{"checkpoint.img"}, Fidelity: "MEMORY", }, map[string]string{"checkpoint.img": "newer-memory"}) diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 73ff2dadf5..2cf156a7ca 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -53,7 +53,6 @@ import ( "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/internal/installdefaults" "go.ate.dev/substrate/internal/nodepath" - "go.ate.dev/substrate/internal/ocispec" "go.ate.dev/substrate/internal/otlprelay" "go.ate.dev/substrate/internal/pluginsocket" "go.ate.dev/substrate/internal/preview" @@ -471,7 +470,7 @@ func main() { // the worker, so the first Run/Restore on this node hits the cache. // Best-effort: on failure the on-demand fetch in ensureSandboxAssets // still covers correctness. - prewarmer := newRuntimePrewarmer(runtimes, wmService, imageCache) + prewarmer := newRuntimePrewarmer(runtimes, wmService) go prewarmer.run(ctx) // Keep a sandbox runtimes file in each local worker's directory: the // runtimes of the enabled versions of its pool's SandboxConfig, @@ -781,10 +780,9 @@ func (s *Atelet) Run(ctx context.Context, req *ateletpb.RunRequest) (resp *atele actorUID := req.GetActorUid() actorRef := resources.ActorRef{Atespace: req.GetAtespace(), Name: req.GetActorName()} - // The sandbox (binaries + pause image) is the runtime the control plane - // assigned the actor to, resolved through the index. Resolved before any - // on-node work, BeginStart included, so an invalid request changes - // nothing. + // The sandbox binaries are the runtime the control plane assigned the + // actor to, resolved through the index. Resolved before any on-node work, + // BeginStart included, so an invalid request changes nothing. sandboxRec, err := s.advertisedRuntime(ctx, req.GetSandboxRuntime()) if err != nil { return nil, err @@ -854,9 +852,7 @@ func (s *Atelet) Run(ctx context.Context, req *ateletpb.RunRequest) (resp *atele if registration, err = s.systemInfoVolumes.Register(actorUID, actorRef, systemInfoVolumesFor(actorUID, req.GetSpec())); err != nil { return nil, err } - containerSpecs, err := s.prepareOCIBundles(ctx, actorUID, actorRef, - req.GetSpec(), sandboxRec.PauseImage, - ) + containerSpecs, err := s.prepareOCIBundles(ctx, actorUID, actorRef, req.GetSpec()) if err != nil { return nil, err } @@ -1430,11 +1426,11 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res actor := actorRef{atespace: req.GetAtespace(), name: req.GetActorName(), uid: actorUID} actorRef := resources.ActorRef{Atespace: req.GetAtespace(), Name: req.GetActorName()} - // The sandbox (binaries + pause image) that runs the restored workload is - // the runtime the control plane assigned the actor to, resolved through - // the index. The snapshot manifests only supply the files to restore and - // the actor identity. Resolved before any on-node work so an invalid - // request changes nothing. + // The sandbox binaries that run the restored workload are the runtime the + // control plane assigned the actor to, resolved through the index. The + // snapshot manifests only supply the files to restore and the actor + // identity. Resolved before any on-node work so an invalid request changes + // nothing. runtimeRec, err := s.advertisedRuntime(ctx, req.GetSandboxRuntime()) if err != nil { return nil, err @@ -1623,7 +1619,7 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res return err } t := time.Now() - containerSpecs, err = s.prepareOCIBundles(gctx, actorUID, actorRef, req.GetSpec(), runtimeRec.PauseImage) + containerSpecs, err = s.prepareOCIBundles(gctx, actorUID, actorRef, req.GetSpec()) dBundles = time.Since(t) if err != nil { prepFailedPhase = ateattr.SnapshotPhaseOCIUnpack @@ -1646,8 +1642,8 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res return nil, err } - // Tell worker to do runsc create + runsc restore for pause container and - // all application containers. + // Tell worker to do runsc create + runsc restore for the sandbox and all + // application containers. spec, err := buildWorkerWorkloadSpec(req.GetSpec(), containerSpecs) if err != nil { return nil, apierror.InvalidArgument("invalid workload spec: %v", err) @@ -1940,19 +1936,14 @@ func prepareDurableDirVolume(path string) error { return nil } -// prepareOCIBundles pulls images and assembles OCI bundles for the pause -// container and every application container in spec, in parallel. pauseImage -// comes from the sandbox record, not the workload spec: it is sandbox -// configuration, and on a restore it must be the image the snapshot was taken -// with. It is empty for sandboxes without a pause container, which get no -// pause bundle. It returns each application container's spec, indexed like -// spec.GetContainers(). +// prepareOCIBundles pulls images and assembles OCI bundles for every +// application container in spec, in parallel. It returns each application +// container's spec, indexed like spec.GetContainers(). func (s *Atelet) prepareOCIBundles( ctx context.Context, actorUID string, actorRef resources.ActorRef, spec *ateletpb.WorkloadSpec, - pauseImage string, ) ([]*ateworkerpb.ContainerSpec, error) { // Prepare host folders for volume types that need them. for _, vol := range spec.GetVolumes() { @@ -1965,16 +1956,6 @@ func (s *Atelet) prepareOCIBundles( g, gCtx := errgroup.WithContext(ctx) - if pauseImage != "" { - g.Go(func() error { - // pause is sandbox infra; it mounts no volumes. - if _, err := prepareOCIDirectory(gCtx, s.imageCache, actorUID, ocispec.PauseContainer, pauseImage, nil, nil); err != nil { - return wrapFileSystemErr("while creating pause OCI bundle", err) - } - return nil - }) - } - // Application containers. containerSpecs := make([]*ateworkerpb.ContainerSpec, len(spec.GetContainers())) for i, ctr := range spec.GetContainers() { diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index cae10e8d8b..ebe036aa8d 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -42,8 +42,6 @@ import ( "go.ate.dev/substrate/internal/ateattr" "go.ate.dev/substrate/internal/ateinterceptors" "go.ate.dev/substrate/internal/atelet" - "go.ate.dev/substrate/internal/imagecache" - "go.ate.dev/substrate/internal/ocispec" "go.ate.dev/substrate/internal/proto/ateletpb" "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/internal/resources" @@ -58,8 +56,6 @@ import ( "google.golang.org/protobuf/types/known/emptypb" ) -const testPauseImage = "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" - const ( snapshotOwnerUID = "3d7f1b62-8a04-4c19-b5e6-2f9c7a1d0e38" @@ -309,7 +305,7 @@ func TestSnapshotManifestActorMetadata(t *testing.T) { // Run/Restore, which has no fidelity, parses back with an empty fidelity and // does not serialize a fidelity key at all. func TestSnapshotManifestFidelityAbsent(t *testing.T) { - legacy := []byte(`{"sandboxClass":"gvisor","pauseImage":"` + testPauseImage + `","snapshotFiles":["checkpoint.img"]}`) + legacy := []byte(`{"sandboxClass":"gvisor","snapshotFiles":["checkpoint.img"]}`) rec, err := unmarshalSandboxRecord(legacy) if err != nil { t.Fatalf("unmarshalSandboxRecord(legacy manifest): %v", err) @@ -327,70 +323,6 @@ func TestSnapshotManifestFidelityAbsent(t *testing.T) { } } -// TestSnapshotManifestPauseImage pins that only a gVisor manifest needs a -// pause image: micro-VM sandboxes run no pause container. -func TestSnapshotManifestPauseImage(t *testing.T) { - for _, tc := range []struct { - name string - manifest string - wantErr bool - }{ - {"gvisor without pause image", `{"sandboxClass":"gvisor","snapshotFiles":["checkpoint.img"]}`, true}, - {"gvisor with pause image", `{"sandboxClass":"gvisor","pauseImage":"` + testPauseImage + `","snapshotFiles":["checkpoint.img"]}`, false}, - {"microvm without pause image", `{"sandboxClass":"microvm","snapshotFiles":["checkpoint.img"]}`, false}, - } { - t.Run(tc.name, func(t *testing.T) { - _, err := unmarshalSandboxRecord([]byte(tc.manifest)) - if tc.wantErr { - if err == nil { - t.Fatal("unmarshalSandboxRecord succeeded, want an error") - } - if !strings.Contains(err.Error(), "pauseImage") { - t.Errorf("error = %v, want it to name pauseImage", err) - } - } else if err != nil { - t.Fatalf("unmarshalSandboxRecord: %v", err) - } - }) - } -} - -// TestPrepareOCIBundlesPause pins that the pause bundle is built only when the -// sandbox has a pause image. -func TestPrepareOCIBundlesPause(t *testing.T) { - host := imageVolumeTestRegistry(t) - image := host + "/actor:v1" - pushTestImage(t, image, singleFileLayer(t, "bin/app", "app")) - spec := &ateletpb.WorkloadSpec{ - Containers: []*ateletpb.Container{{Name: "app", Image: image, Command: []string{"/bin/app"}}}, - } - - for _, tc := range []struct { - name string - pauseImage string - wantPause bool - }{ - {"with pause image", image, true}, - {"without pause image", "", false}, - } { - t.Run(tc.name, func(t *testing.T) { - useTempNodeDirs(t) - const actorUID = "actor-uid-1" - s := &Atelet{imageCache: newImageVolumeStore(t)} - if _, err := s.prepareOCIBundles(t.Context(), actorUID, resources.ActorRef{}, spec, tc.pauseImage); err != nil { - t.Fatalf("prepareOCIBundles: %v", err) - } - if _, err := os.Stat(filepath.Join(ateletpath.OCIBundlePath(actorUID, "app"), imagecache.OverlaySpecFileName)); err != nil { - t.Errorf("app bundle: %v", err) - } - _, err := os.Stat(ateletpath.OCIBundlePath(actorUID, ocispec.PauseContainer)) - if gotPause := err == nil; gotPause != tc.wantPause { - t.Errorf("pause bundle exists = %v, want %v (stat: %v)", gotPause, tc.wantPause, err) - } - }) - } -} - // TestPrepareOCIBundlesFillsContainerSpec pins that each container's spec // carries its resolved args, env, capabilities and resource limits. func TestPrepareOCIBundlesFillsContainerSpec(t *testing.T) { @@ -413,7 +345,7 @@ func TestPrepareOCIBundlesFillsContainerSpec(t *testing.T) { } s := &Atelet{imageCache: newImageVolumeStore(t)} - got, err := s.prepareOCIBundles(t.Context(), "actor-uid-1", resources.ActorRef{}, spec, "") + got, err := s.prepareOCIBundles(t.Context(), "actor-uid-1", resources.ActorRef{}, spec) if err != nil { t.Fatalf("prepareOCIBundles: %v", err) } @@ -431,7 +363,6 @@ func TestPrepareOCIBundlesFillsContainerSpec(t *testing.T) { func TestSnapshotManifestRejectsNonLocalFile(t *testing.T) { manifest, err := json.Marshal(sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: testPauseImage, SnapshotFiles: []string{"../outside"}, }) if err != nil { @@ -445,7 +376,6 @@ func TestSnapshotManifestRejectsNonLocalFile(t *testing.T) { func TestSnapshotManifestRejectsDataFileNotInSnapshotFiles(t *testing.T) { manifest, err := json.Marshal(sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: testPauseImage, SnapshotFiles: []string{"checkpoint.img"}, DataSnapshotFiles: []string{"data.tar"}, }) @@ -1462,7 +1392,6 @@ func TestUploadLocalCheckpointDir(t *testing.T) { fullRec := func(class string) sandboxAssetsRecord { return sandboxAssetsRecord{ SandboxClass: class, - PauseImage: testPauseImage, SnapshotFiles: []string{"config.json", "memory-ranges", "data.tar"}, DataSnapshotFiles: []string{"data.tar"}, Fidelity: ateattr.SnapshotFidelityMemory, @@ -1513,7 +1442,6 @@ func TestUploadLocalCheckpointDir(t *testing.T) { dir := filepath.Join(t.TempDir(), "pause-snap-1") writeLocalSnapshot(t, dir, sandboxAssetsRecord{ SandboxClass: "microvm", - PauseImage: testPauseImage, SnapshotFiles: []string{"rootfs-upper-app.tar", "data.tar"}, DataSnapshotFiles: []string{"data.tar"}, Fidelity: ateattr.SnapshotFidelityRootfs, @@ -1577,7 +1505,6 @@ func TestUploadLocalCheckpointDir(t *testing.T) { dir := filepath.Join(t.TempDir(), "pause-snap-1") writeLocalSnapshot(t, dir, sandboxAssetsRecord{ SandboxClass: "microvm", - PauseImage: testPauseImage, SnapshotFiles: []string{"data.tar"}, }, map[string]string{"data.tar": "data"}) diff --git a/cmd/atelet/runtime_prewarm.go b/cmd/atelet/runtime_prewarm.go index 6801ed0706..ec648b138f 100644 --- a/cmd/atelet/runtime_prewarm.go +++ b/cmd/atelet/runtime_prewarm.go @@ -21,19 +21,14 @@ import ( "io/fs" "log/slog" "os" - "sync" "time" "k8s.io/client-go/util/workqueue" - - "go.ate.dev/substrate/internal/imagecache" ) // defaultPrewarmAttemptTimeout bounds one prewarm attempt. The queue is // drained by one worker, so an attempt that never returned would block -// every other runtime's prewarm. The asset downloads are bounded by the -// asset cache's own fetch timeout; this covers the pause image pull beside -// them. +// every other runtime's prewarm. const defaultPrewarmAttemptTimeout = 10 * time.Minute // prewarmMaxRetries bounds how often one failed prewarm is retried before @@ -60,7 +55,7 @@ type prewarmItem struct { } // runtimePrewarmer fetches the default runtime of each worker on this node -// into the asset and image caches before any actor asks for it, so the +// into the asset cache before any actor asks for it, so the // fetch inside the first Run/Restore on the node is a cache hit instead of // a download+extract on the critical path. The worker sandbox syncer hands // it each default runtime a worker advertises, once the worker's file says @@ -76,14 +71,12 @@ type prewarmItem struct { // without fetching, retrying, or recreating anything. // // Prewarming is purely a latency optimization: every failure is logged and -// left to the on-demand fetch in ensureSandboxAssets and the pull inside -// prepareOCIBundles, which remain the correctness path. Only default -// runtimes are prewarmed; the first restore of another version a worker -// advertises pays its download. +// left to the on-demand fetch in ensureSandboxAssets, which remains the +// correctness path. Only default runtimes are prewarmed; the first restore of +// another version a worker advertises pays its download. type runtimePrewarmer struct { index *runtimeIndex assets sandboxAssetFetcher - images *imagecache.Store // queue carries items. It dedupes while an item waits, and it // rate-limits retries after failures. A single worker drains it, which // also serializes downloads so concurrent prewarms never compete for @@ -100,11 +93,10 @@ type runtimePrewarmer struct { onFinished func(item prewarmItem, err error) } -func newRuntimePrewarmer(index *runtimeIndex, assets sandboxAssetFetcher, images *imagecache.Store) *runtimePrewarmer { +func newRuntimePrewarmer(index *runtimeIndex, assets sandboxAssetFetcher) *runtimePrewarmer { return &runtimePrewarmer{ index: index, assets: assets, - images: images, attemptTimeout: defaultPrewarmAttemptTimeout, // Downloads fail on the scale of network timeouts, not API conflicts, // so back off in seconds and cap in minutes rather than the @@ -183,11 +175,10 @@ func (p *runtimePrewarmer) process(ctx context.Context, item prewarmItem) { } // prewarm fetches the runtime's assets into the asset cache, linked into -// the item's directory, and its pause image into the image cache, the two -// side by side. Racing an on-demand ensureSandboxAssets for the same assets -// is safe: the asset cache runs one download per asset and serves every -// caller from it, and the image cache collapses concurrent pulls of one -// digest. A directory the syncer already filled costs a stat per asset. +// the item's directory. Racing an on-demand ensureSandboxAssets for the same +// assets is safe: the asset cache runs one download per asset and serves +// every caller from it. A directory the syncer already filled costs a stat +// per asset. // // Nothing here creates a directory: the staging goes into the item's // directory as it is, so a directory removed before or during the work, by @@ -204,20 +195,7 @@ func (p *runtimePrewarmer) prewarm(ctx context.Context, item prewarmItem, rt *in return err } - var assetErr, imageErr error - var wg sync.WaitGroup - wg.Go(func() { - _, assetErr = p.assets.stageSandboxAssets(ctx, rt.rec, item.dir) - }) - if rt.rec.PauseImage != "" { - wg.Go(func() { - if _, err := p.images.EnsureImage(ctx, rt.rec.PauseImage); err != nil { - imageErr = fmt.Errorf("while prewarming pause image %q: %w", rt.rec.PauseImage, err) - } - }) - } - wg.Wait() - if err := errors.Join(assetErr, imageErr); err != nil { + if _, err := p.assets.stageSandboxAssets(ctx, rt.rec, item.dir); err != nil { if gone, gerr := dirGone(item.dir); gerr == nil && gone { slog.DebugContext(ctx, "Sandbox runtime prewarm superseded while staging", slog.String("runtime_hash", item.hash), slog.String("dir", item.dir)) return nil @@ -229,7 +207,6 @@ func (p *runtimePrewarmer) prewarm(ctx context.Context, item prewarmItem, rt *in slog.String("config", rt.config), slog.String("version", rt.version), slog.Int("assets", len(rt.rec.Assets)), - slog.String("pause_image", rt.rec.PauseImage), slog.String("dir", item.dir), slog.Duration("duration", time.Since(t))) return nil diff --git a/cmd/atelet/runtime_prewarm_test.go b/cmd/atelet/runtime_prewarm_test.go index 4dc1d72482..19105e3cf9 100644 --- a/cmd/atelet/runtime_prewarm_test.go +++ b/cmd/atelet/runtime_prewarm_test.go @@ -30,11 +30,6 @@ import ( "testing" "time" - "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/empty" - "github.com/google/go-containerregistry/pkg/v1/mutate" - "github.com/google/go-containerregistry/pkg/v1/remote" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sruntime "k8s.io/apimachinery/pkg/runtime" kubeinformers "k8s.io/client-go/informers" @@ -42,34 +37,11 @@ import ( "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" "go.ate.dev/substrate/cmd/atelet/internal/filecache" - "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/pkg/api/v1alpha1" atefake "go.ate.dev/substrate/pkg/client/clientset/versioned/fake" "go.ate.dev/substrate/pkg/client/informers/externalversions" ) -// pushPauseImage pushes a tiny image to ref and returns its manifest digest, -// so tests can later assert a cache hit by digest with the registry gone. -func pushPauseImage(t *testing.T, ref string) v1.Hash { - t.Helper() - img, err := mutate.AppendLayers(empty.Image, singleFileLayer(t, "pause", "pause bytes")) - if err != nil { - t.Fatalf("mutate.AppendLayers: %v", err) - } - digest, err := img.Digest() - if err != nil { - t.Fatalf("img.Digest: %v", err) - } - tag, err := name.ParseReference(ref, name.Insecure) - if err != nil { - t.Fatalf("name.ParseReference(%q): %v", ref, err) - } - if err := remote.Write(tag, img); err != nil { - t.Fatalf("remote.Write(%q): %v", ref, err) - } - return digest -} - func gvisorConfig(name, url, sha string) *v1alpha1.SandboxConfig { return &v1alpha1.SandboxConfig{ ObjectMeta: metav1.ObjectMeta{Name: name}, @@ -77,8 +49,7 @@ func gvisorConfig(name, url, sha string) *v1alpha1.SandboxConfig { SandboxClass: v1alpha1.SandboxClassGvisor, DefaultVersion: "v1", Versions: []v1alpha1.SandboxVersionConfig{{ - Name: "v1", - PauseImage: "registry.k8s.io/pause@sha256:abc", + Name: "v1", Assets: map[string]map[string]v1alpha1.AssetFile{ runtime.GOARCH: { gvisorAssetName: {URL: url, SHA256: sha}, @@ -100,35 +71,21 @@ func (hangingObjectStorage) GetObject(ctx context.Context, _, _ string) (io.Read func (hangingObjectStorage) PutObject(_ context.Context, _, _ string, _ io.Reader) error { return nil } -// pauseImageInTestRegistry pushes a pause image to a test registry and -// returns its digest-pinned reference, the only form a runtime hash accepts. -func pauseImageInTestRegistry(t *testing.T) (ref string, host string) { - t.Helper() - host = imageVolumeTestRegistry(t) - digest := pushPauseImage(t, host+"/pause:3.10") - return host + "/pause@" + digest.String(), host -} - // indexedTestRuntime registers a runtime with one gvisor release asset, the -// tarball content, and the given pause image as a worker's default, and -// returns its hash. -func indexedTestRuntime(t *testing.T, index *runtimeIndex, content []byte, pauseImage string) string { +// tarball content, as a worker's default, and returns its hash. +func indexedTestRuntime(t *testing.T, index *runtimeIndex, content []byte) string { t.Helper() - return indexedTestRuntimeAt(t, index, "gs://bucket/gvisor.tar", content, pauseImage) + return indexedTestRuntimeAt(t, index, "gs://bucket/gvisor.tar", content) } // indexedTestRuntimeAt is indexedTestRuntime with the asset at url. -func indexedTestRuntimeAt(t *testing.T, index *runtimeIndex, url string, content []byte, pauseImage string) string { +func indexedTestRuntimeAt(t *testing.T, index *runtimeIndex, url string, content []byte) string { t.Helper() rec := &sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: pauseImage, Assets: map[string]assetEntry{gvisorAssetName: {URL: url, SHA256: fmt.Sprintf("%x", sha256.Sum256(content))}}, } - hash, err := versionBundleHash(rec) - if err != nil { - t.Fatal(err) - } + hash := bundleHash(rec) index.register("pod-a", advertisedClass{class: v1alpha1.SandboxClassGvisor, config: "gvisor-release", hashed: []hashedVersion{{ versionPlan: versionPlan{name: "v1", isDefault: true, rec: rec}, hash: hash, }}}) @@ -147,25 +104,16 @@ func defaultItem(t *testing.T, root, hash string) prewarmItem { return prewarmItem{hash: hash, dir: dir} } -func newTestImageStore(t *testing.T) *imagecache.Store { - t.Helper() - store, err := imagecache.New(t.TempDir()) - if err != nil { - t.Fatalf("imagecache.New: %v", err) - } - return store -} - // TestRuntimePrewarmProcess covers the worker's failure handling: a failed // prewarm is requeued with backoff until prewarmMaxRetries and then dropped, // and an item naming a hash that is not in the index is dropped at once. func TestRuntimePrewarmProcess(t *testing.T) { ctx := context.Background() index := newRuntimeIndex() - hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t), "") + hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t)) root := t.TempDir() herder := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}} - p := newRuntimePrewarmer(index, herder, newTestImageStore(t)) + p := newRuntimePrewarmer(index, herder) item := defaultItem(t, root, hash) for i := 1; i <= prewarmMaxRetries; i++ { @@ -194,7 +142,7 @@ func TestRuntimePrewarmSuperseded(t *testing.T) { ctx := context.Background() index := newRuntimeIndex() content := gvisorReleaseTarball(t) - hash := indexedTestRuntime(t, index, content, "") + hash := indexedTestRuntime(t, index, content) root := t.TempDir() herder := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: content}} @@ -203,7 +151,7 @@ func TestRuntimePrewarmSuperseded(t *testing.T) { if err := os.RemoveAll(item.dir); err != nil { t.Fatal(err) } - p := newRuntimePrewarmer(index, herder, newTestImageStore(t)) + p := newRuntimePrewarmer(index, herder) p.process(ctx, item) if got := p.queue.NumRequeues(item); got != 0 { t.Errorf("NumRequeues after a superseded prewarm = %d, want 0 (not retried)", got) @@ -219,7 +167,7 @@ func TestRuntimePrewarmSuperseded(t *testing.T) { t.Run("worker gone while staging", func(t *testing.T) { item := defaultItem(t, root, hash) removing := &removingFetcher{inner: herder, remove: filepath.Join(root, "workers", "pod-a")} - p := newRuntimePrewarmer(index, removing, newTestImageStore(t)) + p := newRuntimePrewarmer(index, removing) p.process(ctx, item) if got := p.queue.NumRequeues(item); got != 0 { t.Errorf("NumRequeues after the worker went mid-staging = %d, want 0 (not retried)", got) @@ -248,25 +196,23 @@ func (r *removingFetcher) stageSandboxAssets(ctx context.Context, rec *sandboxAs return r.inner.stageSandboxAssets(ctx, rec, dstDir) } -// TestRuntimePrewarmFillsBothCaches covers a successful prewarm: the asset +// TestRuntimePrewarmFillsTheAssetCache covers a successful prewarm: the asset // lands in the asset cache, linked into the runtime's directory so an -// eviction pass keeps it, and the pause image lands in the image cache. A -// second prewarm of the same directory fetches nothing. -func TestRuntimePrewarmFillsBothCaches(t *testing.T) { +// eviction pass keeps it. A second prewarm of the same directory fetches +// nothing. +func TestRuntimePrewarmFillsTheAssetCache(t *testing.T) { ctx := context.Background() - pauseRef, _ := pauseImageInTestRegistry(t) content := gvisorReleaseTarball(t) index := newRuntimeIndex() - hash := indexedTestRuntime(t, index, content, pauseRef) + hash := indexedTestRuntime(t, index, content) rt, _ := index.lookup(hash) root := t.TempDir() - images := newTestImageStore(t) var gets atomic.Int32 // No min age, so the only thing standing between the entry and a pass is // the directory's link. s := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root, filecache.WithMinAge(0)), anonGCSClient: fakeObjectStorage{data: content, gets: &gets}} - p := newRuntimePrewarmer(index, s, images) + p := newRuntimePrewarmer(index, s) item := defaultItem(t, root, hash) if err := p.prewarm(ctx, item, rt); err != nil { t.Fatalf("prewarm: %v", err) @@ -277,9 +223,6 @@ func TestRuntimePrewarmFillsBothCaches(t *testing.T) { if _, err := os.Stat(filepath.Join(item.dir, gvisorAssetName, "runsc")); err != nil { t.Errorf("prewarm did not link the runtime's asset into its directory: %v", err) } - if _, err := images.EnsureImage(ctx, pauseRef); err != nil { - t.Errorf("pause image not prewarmed into the image cache: %v", err) - } stats, err := s.sandboxAssetCache.EvictUnused(ctx, 1<<40, false) if err != nil { t.Fatalf("EvictUnused: %v", err) @@ -307,24 +250,19 @@ func TestRuntimePrewarmFillsBothCaches(t *testing.T) { } } -// TestRuntimePrewarmAssetFailureStillPullsImage pins that the two halves -// are independent: a failing asset fetch does not stop the pause image pull. -func TestRuntimePrewarmAssetFailureStillPullsImage(t *testing.T) { +// TestRuntimePrewarmAssetFailure pins that a failing asset fetch fails the +// prewarm. +func TestRuntimePrewarmAssetFailure(t *testing.T) { ctx := context.Background() - pauseRef, _ := pauseImageInTestRegistry(t) index := newRuntimeIndex() - hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t), pauseRef) + hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t)) rt, _ := index.lookup(hash) - images := newTestImageStore(t) root := t.TempDir() - p := newRuntimePrewarmer(index, &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}}, images) + p := newRuntimePrewarmer(index, &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}}) if err := p.prewarm(ctx, defaultItem(t, root, hash), rt); err == nil { t.Error("prewarm returned nil despite the asset fetch failing") } - if _, err := images.EnsureImage(ctx, pauseRef); err != nil { - t.Errorf("pause image not pulled when the asset fetch failed: %v", err) - } } // TestRuntimePrewarmAttemptTimeout verifies a single attempt is bounded by @@ -332,10 +270,10 @@ func TestRuntimePrewarmAssetFailureStillPullsImage(t *testing.T) { // that never returned would block every other runtime's prewarm. func TestRuntimePrewarmAttemptTimeout(t *testing.T) { index := newRuntimeIndex() - hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t), "") + hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t)) rt, _ := index.lookup(hash) root := t.TempDir() - p := newRuntimePrewarmer(index, &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: hangingObjectStorage{}}, newTestImageStore(t)) + p := newRuntimePrewarmer(index, &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: hangingObjectStorage{}}) p.attemptTimeout = 50 * time.Millisecond item := defaultItem(t, root, hash) @@ -479,9 +417,9 @@ func (r *recordingPrewarm) count() int { // request. func TestDefaultRuntimePrewarmThroughSyncer(t *testing.T) { useTempAssetScratch(t) // the syncer's runsc probe stages into it - // The worker's own prewarm call returns only when the asset has landed - // and the pause-image pull has finished. Waiting for the call keeps the - // pull's last writes from racing the TempDir cleanup. + // The worker's own prewarm call returns only when the asset has landed. + // Waiting for the call keeps its last writes from racing the TempDir + // cleanup. prewarmed := make(chan error, 1) onFinished := func(_ prewarmItem, err error) { // Non-blocking: a failed attempt is retried, and the worker must not @@ -492,10 +430,8 @@ func TestDefaultRuntimePrewarmThroughSyncer(t *testing.T) { } } - pauseRef, _ := pauseImageInTestRegistry(t) content := gvisorReleaseTarball(t) cfg := gvisorConfig("gvisor-release", "gs://bucket/gvisor.tar", fmt.Sprintf("%x", sha256.Sum256(content))) - cfg.Spec.Versions[0].PauseImage = pauseRef kube := kubefake.NewClientset([]k8sruntime.Object{workerPod("pod-a", "pool")}...) ate := atefake.NewSimpleClientset(resolvedPool("pool", "gvisor-release"), cfg) @@ -507,7 +443,7 @@ func TestDefaultRuntimePrewarmThroughSyncer(t *testing.T) { workersDir := filepath.Join(root, "workers") // on the cache's filesystem index := newRuntimeIndex() herder := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: content}} - p := newRuntimePrewarmer(index, herder, newTestImageStore(t)) + p := newRuntimePrewarmer(index, herder) p.onFinished = onFinished go p.run(ctx) // Join the worker before the TempDir cleanups remove what it writes into. diff --git a/cmd/atelet/sandbox_assets.go b/cmd/atelet/sandbox_assets.go index c5043cbecc..bea59b8a62 100644 --- a/cmd/atelet/sandbox_assets.go +++ b/cmd/atelet/sandbox_assets.go @@ -42,7 +42,6 @@ import ( "go.ate.dev/substrate/cmd/atelet/internal/filecache" "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/resources" - "go.ate.dev/substrate/pkg/api/v1alpha1" "go.ate.dev/substrate/pkg/objectstorage" objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc/codes" @@ -77,19 +76,14 @@ func (e assetEntry) displayURL() string { } // sandboxAssetsRecord is the sandbox runtime an actor is running, projected onto -// the local node's architecture: the sandbox class and pause image plus the -// asset set keyed by asset name (gVisor uses a single "gvisor" release-tarball -// asset). +// the local node's architecture: the sandbox class plus the asset set keyed by +// asset name (gVisor uses a single "gvisor" release-tarball asset). // It is both the per-actor on-node record (written at Run/Restore, read at // Checkpoint) and the snapshot manifest (written at Checkpoint, read at // Restore). type sandboxAssetsRecord struct { SandboxClass string `json:"sandboxClass"` Assets map[string]assetEntry `json:"assets"` - // PauseImage is the root sandbox container's image. It is recorded here - // rather than taken from the request at Restore so a snapshot is rebuilt - // with the same sandbox it was captured from. Only gVisor has one. - PauseImage string `json:"pauseImage,omitempty"` // Actor identity makes a flat snapshot self-identifying if control-plane // persistence is unavailable. Atespace string `json:"atespace,omitempty"` @@ -114,15 +108,6 @@ type sandboxAssetsRecord struct { Fidelity string `json:"fidelity,omitempty"` } -// checkPauseImage requires a pause image for gVisor, the only sandbox class -// that runs a pause container. -func checkPauseImage(sandboxClass, pauseImage string) error { - if sandboxClass == string(v1alpha1.SandboxClassGvisor) && pauseImage == "" { - return fmt.Errorf("gvisor sandbox has no pauseImage") - } - return nil -} - // ensureSandboxAssets stages every asset in the record under dstDir, named // by asset name, and returns a map of asset name to the local path the worker // runs it from. gVisor has a single "gvisor" release-tarball asset (staged @@ -676,9 +661,6 @@ func unmarshalSandboxRecord(data []byte) (*sandboxAssetsRecord, error) { if err := json.Unmarshal(data, rec); err != nil { return nil, fmt.Errorf("while parsing sandbox record/manifest: %w", err) } - if err := checkPauseImage(rec.SandboxClass, rec.PauseImage); err != nil { - return nil, fmt.Errorf("invalid sandbox record/manifest: %w", err) - } if err := validateSnapshotFiles(rec.SnapshotFiles); err != nil { return nil, fmt.Errorf("sandbox record/manifest has invalid snapshotFiles: %w", err) } diff --git a/cmd/atelet/sandbox_runtime_index.go b/cmd/atelet/sandbox_runtime_index.go index 0dc8050808..335e06935d 100644 --- a/cmd/atelet/sandbox_runtime_index.go +++ b/cmd/atelet/sandbox_runtime_index.go @@ -158,11 +158,7 @@ func checkIndexedRuntime(hash string, e indexedRuntimeFile) error { if e.Record == nil { return errors.New("no record") } - got, err := versionBundleHash(e.Record) - if err != nil { - return fmt.Errorf("while hashing the record: %w", err) - } - if got != hash { + if got := bundleHash(e.Record); got != hash { return fmt.Errorf("record hashes to %s", got) } return nil @@ -237,9 +233,9 @@ func (x *runtimeIndex) registerRuntimes(config string, hashed []hashedVersion) e // addRuntime records v under its hash, unless the hash is already known, // persisting the index first when it has a path. The hash covers every -// asset's name and digest and the pause image, so two versions with one -// hash differ at most in where identical bytes are fetched from; keeping -// the first keeps the version the file was built from. Called with mu held. +// asset's name and digest, so two versions with one hash differ at most in +// where identical bytes are fetched from; keeping the first keeps the version +// the file was built from. Called with mu held. func (x *runtimeIndex) addRuntime(config string, v hashedVersion) error { if _, ok := x.runtimes[v.hash]; ok { return nil diff --git a/cmd/atelet/sandbox_runtime_index_test.go b/cmd/atelet/sandbox_runtime_index_test.go index 535939924c..d58195c3e9 100644 --- a/cmd/atelet/sandbox_runtime_index_test.go +++ b/cmd/atelet/sandbox_runtime_index_test.go @@ -35,14 +35,9 @@ func hashedTestVersion(t *testing.T, name string, isDefault bool, sha string) ha t.Helper() rec := &sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: testPauseImage, Assets: map[string]assetEntry{gvisorAssetName: {URL: "gs://bucket/" + name, SHA256: sha}}, } - h, err := versionBundleHash(rec) - if err != nil { - t.Fatal(err) - } - return hashedVersion{versionPlan: versionPlan{name: name, isDefault: isDefault, rec: rec}, hash: h} + return hashedVersion{versionPlan: versionPlan{name: name, isDefault: isDefault, rec: rec}, hash: bundleHash(rec)} } func TestRuntimeIndex(t *testing.T) { diff --git a/cmd/atelet/snapshot_transfer_test.go b/cmd/atelet/snapshot_transfer_test.go index 3847a66a50..466ac3c850 100644 --- a/cmd/atelet/snapshot_transfer_test.go +++ b/cmd/atelet/snapshot_transfer_test.go @@ -105,7 +105,7 @@ func TestUploadSnapshotManifestLast(t *testing.T) { t.Fatal(err) } dir := t.TempDir() - rec := sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: testPauseImage, SnapshotFiles: []string{"a", "b", "c"}} + rec := sandboxAssetsRecord{SandboxClass: "gvisor", SnapshotFiles: []string{"a", "b", "c"}} for _, f := range rec.SnapshotFiles { if err := os.WriteFile(dir+"/"+f, []byte(f), 0o600); err != nil { t.Fatal(err) @@ -181,7 +181,6 @@ func TestObjectStorePluginErrorCodeThroughAtelet(t *testing.T) { } rec := sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: testPauseImage, SnapshotFiles: []string{"a"}, Fidelity: ateattr.SnapshotFidelityMemory, } @@ -245,7 +244,7 @@ func TestMintErrorCodeThroughAtelet(t *testing.T) { if err != nil { t.Fatal(err) } - rec := sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: testPauseImage, SnapshotFiles: []string{"a"}, Fidelity: ateattr.SnapshotFidelityMemory} + rec := sandboxAssetsRecord{SandboxClass: "gvisor", SnapshotFiles: []string{"a"}, Fidelity: ateattr.SnapshotFidelityMemory} transfers := []struct { name string retryable bool @@ -304,7 +303,7 @@ func TestObjectStorePluginErrorKeepsCredentialsOut(t *testing.T) { if err != nil { t.Fatal(err) } - rec := sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: testPauseImage, SnapshotFiles: []string{"a"}, Fidelity: ateattr.SnapshotFidelityMemory} + rec := sandboxAssetsRecord{SandboxClass: "gvisor", SnapshotFiles: []string{"a"}, Fidelity: ateattr.SnapshotFidelityMemory} transfers := []struct { name string uri string // the URI the transfer names in its error, when it takes one @@ -462,7 +461,6 @@ func TestCheckpointUploadWaitsOutPluginOutage(t *testing.T) { } rec := sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: testPauseImage, SnapshotFiles: []string{"a"}, Fidelity: ateattr.SnapshotFidelityMemory, } diff --git a/cmd/atelet/worker_sandbox_syncer.go b/cmd/atelet/worker_sandbox_syncer.go index ac2b7582f8..81ff49866f 100644 --- a/cmd/atelet/worker_sandbox_syncer.go +++ b/cmd/atelet/worker_sandbox_syncer.go @@ -369,10 +369,7 @@ func (s *workerSandboxSyncer) registerAhead(pool *v1alpha1.WorkerPool) { if err != nil { return } - hashed, err := hashVersions(context.Background(), plan) - if err != nil { - return - } + hashed := hashVersions(plan) if err := s.index.registerRuntimes(plan.config, hashed); err != nil { // The syncs write it again, and report it if it keeps failing. slog.Warn("Could not index a changed pool's runtimes ahead of its sync", slog.String("pool", pool.Name), slog.Any("err", err)) diff --git a/cmd/atelet/worker_sandbox_syncer_test.go b/cmd/atelet/worker_sandbox_syncer_test.go index 38cb0a5b3b..c6585e5e00 100644 --- a/cmd/atelet/worker_sandbox_syncer_test.go +++ b/cmd/atelet/worker_sandbox_syncer_test.go @@ -394,11 +394,7 @@ func TestWorkerSandboxSyncerRegistersAheadOfJitter(t *testing.T) { if err != nil { t.Fatal(err) } - hashed, err := hashVersions(context.Background(), plan) - if err != nil { - t.Fatal(err) - } - for _, v := range hashed { + for _, v := range hashVersions(plan) { if _, ok := s.index.lookup(v.hash); !ok { t.Errorf("runtime %s (%s) does not resolve before the jittered sync", v.name, v.hash) } diff --git a/cmd/atelet/worker_sandbox_versions.go b/cmd/atelet/worker_sandbox_versions.go index d964c9c4de..f5bfcb0e54 100644 --- a/cmd/atelet/worker_sandbox_versions.go +++ b/cmd/atelet/worker_sandbox_versions.go @@ -18,10 +18,10 @@ package main // in the steps syncPod runs: // - resolve: find the SandboxConfig of the pod's WorkerPool // - plan: pick the config's versions this node can run -// - build: hash each version's assets and pause image digests into its -// sandbox_bundle_hash, and probe what a snapshot taken on the runtime -// must find again: the host CPU identity for microVM, the runsc -// binary's CPU features for gVisor +// - build: hash each version's asset digests into its sandbox_bundle_hash, +// and probe what a snapshot taken on the runtime must find again: the +// host CPU identity for microVM, the runsc binary's CPU features for +// gVisor // - write: replace the file in the worker directory if its content changed // // worker_sandbox_syncer.go decides when each pod is synced, and creates and @@ -42,7 +42,6 @@ import ( "strings" "time" - "github.com/google/go-containerregistry/pkg/name" corev1 "k8s.io/api/core/v1" "go.ate.dev/substrate/internal/hardware" @@ -55,8 +54,8 @@ import ( // errPermanent marks a failure that retrying cannot fix, so the syncer drops // the item instead of backing off until the next event for the pod: // - the pod has no worker pool label -// - the config's default version is missing, has no assets for this -// node's architecture, or has a pause image not pinned by digest +// - the config's default version is missing or has no assets for this +// node's architecture // - the default version's runsc runs but fails, prints no features, or // prints one this atelet does not know; the binary is content-addressed, // so running it again gives the same answer @@ -197,7 +196,6 @@ func recordForArch(cfg *v1alpha1.SandboxConfig, ver *v1alpha1.SandboxVersionConf } rec := &sandboxAssetsRecord{ SandboxClass: string(cfg.Spec.SandboxClass), - PauseImage: ver.PauseImage, Assets: make(map[string]assetEntry, len(files)), } for name, f := range files { @@ -213,24 +211,13 @@ type hashedVersion struct { } // hashVersions hashes each version of plan. The hashes come from the -// SandboxConfig spec alone, so it does no IO. A version that cannot be -// hashed is left out, unless it is the default, without which no worker of -// the pool could start actors on this node. -func hashVersions(ctx context.Context, plan *sandboxPlan) ([]hashedVersion, error) { +// SandboxConfig spec alone, so it does no IO. +func hashVersions(plan *sandboxPlan) []hashedVersion { hashed := make([]hashedVersion, 0, len(plan.versions)) for _, v := range plan.versions { - h, err := versionBundleHash(v.rec) - if err != nil { - err = fmt.Errorf("SandboxConfig %q, version %q: %w", plan.config, v.name, err) - if v.isDefault { - return nil, fmt.Errorf("%w: %w", errPermanent, err) - } - slog.WarnContext(ctx, "Leaving a sandbox version out of the worker sandbox runtimes file", slog.Any("err", err)) - continue - } - hashed = append(hashed, hashedVersion{versionPlan: v, hash: h}) + hashed = append(hashed, hashedVersion{versionPlan: v, hash: bundleHash(v.rec)}) } - return hashed, nil + return hashed } // buildFile returns the file content naming the runtimes of the hashed @@ -248,10 +235,10 @@ func hashVersions(ctx context.Context, plan *sandboxPlan) ([]hashedVersion, erro // fetched and probed, serially. // // Listing a runtime does not promise its assets are on the node: probing -// fetches a gVisor version's runsc the first time this atelet sees it, but -// not its pause image, and nothing of a microVM version. The syncer has -// the default runtime prewarmed once the file is written, and Run/Restore -// fetch any other version on demand. +// fetches a gVisor version's runsc the first time this atelet sees it, and +// nothing of a microVM version. The syncer has the default runtime prewarmed +// once the file is written, and Run/Restore fetch any other version on +// demand. func (s *workerSandboxSyncer) buildFile(ctx context.Context, plan *sandboxPlan, hashed []hashedVersion) (*sandboxruntimes.File, error) { out := &sandboxruntimes.File{} seen := make(map[string]bool, len(hashed)) @@ -424,34 +411,15 @@ func runRunscCPUFeatures(ctx context.Context, runscPath string) ([]string, error return strings.Split(printed, ","), nil } -// versionBundleHash returns the bundle hash of one version. The CRD requires -// every pause image to be pinned by digest, which is the digest the image -// cache resolves the ref to, so the hash needs no pull. An unpinned one is -// errPermanent: only a spec change fixes it. -func versionBundleHash(rec *sandboxAssetsRecord) (string, error) { - var pauseDigest string - if rec.PauseImage != "" { - ref, err := name.NewDigest(rec.PauseImage) - if err != nil { - return "", fmt.Errorf("%w: pause image %q is not pinned by digest: %w", errPermanent, rec.PauseImage, err) - } - pauseDigest = ref.DigestStr() - } - return bundleHash(rec, pauseDigest), nil -} - // bundleHash is the sandbox_bundle_hash of a version: a sha256 over one -// sorted "key=value" line per asset digest and pause image digest. Version -// names and URLs are left out, so the same binaries under another name or in -// another bucket are the same runtime. -func bundleHash(rec *sandboxAssetsRecord, pauseDigest string) string { - lines := make([]string, 0, len(rec.Assets)+1) +// sorted "asset/=" line per asset. Version names and URLs are +// left out, so the same binaries under another name or in another bucket are +// the same runtime. +func bundleHash(rec *sandboxAssetsRecord) string { + lines := make([]string, 0, len(rec.Assets)) for name, a := range rec.Assets { lines = append(lines, "asset/"+name+"="+a.SHA256) } - if pauseDigest != "" { - lines = append(lines, "pause_image="+pauseDigest) - } slices.Sort(lines) sum := sha256.Sum256([]byte(strings.Join(lines, "\n"))) return "sha256:" + hex.EncodeToString(sum[:]) @@ -475,10 +443,7 @@ func (s *workerSandboxSyncer) syncPod(ctx context.Context, pod *corev1.Pod) erro if err != nil { return err } - hashed, err := hashVersions(ctx, plan) - if err != nil { - return err - } + hashed := hashVersions(plan) // Resolvable, and on disk, before the file can let the worker register // any of them. if err := s.index.registerRuntimes(plan.config, hashed); err != nil { @@ -501,7 +466,7 @@ func (s *workerSandboxSyncer) syncPod(ctx context.Context, pod *corev1.Pod) erro // Only now, with the file naming its defaults: a sync that failed above // left the file, and so the directories, as they were. For a gVisor // default the build fetched the release for its probe, so the prewarm - // is links and the pause image; a micro-VM default downloads here. + // is links; a micro-VM default downloads here. if s.prewarm != nil { var defaults []string for _, v := range writtenVersions(hashed, want) { diff --git a/cmd/atelet/worker_sandbox_versions_test.go b/cmd/atelet/worker_sandbox_versions_test.go index 7941cc1915..ae66b36659 100644 --- a/cmd/atelet/worker_sandbox_versions_test.go +++ b/cmd/atelet/worker_sandbox_versions_test.go @@ -37,21 +37,14 @@ import ( "go.ate.dev/substrate/pkg/api/v1alpha1" ) -// pauseDigest is the digest of the pause image of multiVersionConfig's -// version with the given name. -func pauseDigest(name string) string { - return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte("pause-"+name))) -} - // multiVersionConfig is a gVisor config whose default version "v2" is listed // after an Enabled "v1", a Disabled "v0", and an Enabled "other-arch" with no // assets for this node. func multiVersionConfig() *v1alpha1.SandboxConfig { version := func(name, sha string, state *v1alpha1.SandboxVersionState) v1alpha1.SandboxVersionConfig { return v1alpha1.SandboxVersionConfig{ - Name: name, - State: state, - PauseImage: "registry.k8s.io/pause@" + pauseDigest(name), + Name: name, + State: state, Assets: map[string]map[string]v1alpha1.AssetFile{ runtime.GOARCH: {gvisorAssetName: {URL: "gs://bucket/" + name, SHA256: sha}}, }, @@ -85,7 +78,6 @@ func TestRecordForArch(t *testing.T) { assets: map[string]map[string]v1alpha1.AssetFile{runtime.GOARCH: {gvisorAssetName: {URL: "gs://bucket/gvisor", SHA256: sha}}}, want: &sandboxAssetsRecord{ SandboxClass: string(v1alpha1.SandboxClassGvisor), - PauseImage: "registry.k8s.io/pause@sha256:abc", Assets: map[string]assetEntry{gvisorAssetName: {URL: "gs://bucket/gvisor", SHA256: sha}}, }, }, { @@ -109,35 +101,26 @@ func TestRecordForArch(t *testing.T) { func TestBundleHash(t *testing.T) { gvisor := map[string]assetEntry{gvisorAssetName: {URL: "gs://bucket/gvisor", SHA256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f"}} - pause := "sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" // Snapshots record the hash, so a change to any of these values strands // every snapshot taken before it. tests := []struct { - name string - rec *sandboxAssetsRecord - pauseDigest string - want string + name string + rec *sandboxAssetsRecord + want string }{{ - name: "gvisor asset and pause image", - rec: &sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: "registry.k8s.io/pause:3.10.2@" + pause, Assets: gvisor}, - pauseDigest: pause, - want: "sha256:7197896bb429f0b824accbc635a38c9ed5666c160070b833bbdc6e036ccd4f76", + name: "gvisor asset", + rec: &sandboxAssetsRecord{SandboxClass: "gvisor", Assets: gvisor}, + want: "sha256:f86e6f2bd8e25ac31a22e51e6e57e0eb6741eb0b0df77082989cc1e40f906fe1", }, { // Only the digests count, not the version's other fields. - name: "another URL and pause image ref", - rec: &sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: "mirror.example.com/pause@" + pause, Assets: map[string]assetEntry{gvisorAssetName: {URL: "gs://other/gvisor", SHA256: gvisor[gvisorAssetName].SHA256}}}, - pauseDigest: pause, - want: "sha256:7197896bb429f0b824accbc635a38c9ed5666c160070b833bbdc6e036ccd4f76", - }, { - name: "no pause image", - rec: &sandboxAssetsRecord{SandboxClass: "microvm", Assets: gvisor}, + name: "another URL and class", + rec: &sandboxAssetsRecord{SandboxClass: "microvm", Assets: map[string]assetEntry{gvisorAssetName: {URL: "gs://other/gvisor", SHA256: gvisor[gvisorAssetName].SHA256}}}, want: "sha256:f86e6f2bd8e25ac31a22e51e6e57e0eb6741eb0b0df77082989cc1e40f906fe1", }, { - name: "the asset under another name", - rec: &sandboxAssetsRecord{SandboxClass: "gvisor", Assets: map[string]assetEntry{"renamed": gvisor[gvisorAssetName]}}, - pauseDigest: pause, - want: "sha256:04bf18376ca0c7227ae9fa65675127b1045d7de33f05ea5d2375eb11a1a8e62e", + name: "the asset under another name", + rec: &sandboxAssetsRecord{SandboxClass: "gvisor", Assets: map[string]assetEntry{"renamed": gvisor[gvisorAssetName]}}, + want: "sha256:510948cc43e084528a7c76c1627c5bfb2d140be0b620f33282c03fc2debd65af", }, { name: "several assets", rec: &sandboxAssetsRecord{SandboxClass: "microvm", Assets: map[string]assetEntry{ @@ -148,7 +131,7 @@ func TestBundleHash(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := bundleHash(tt.rec, tt.pauseDigest); got != tt.want { + if got := bundleHash(tt.rec); got != tt.want { t.Errorf("bundleHash() = %q, want %q", got, tt.want) } }) @@ -245,48 +228,6 @@ func TestWorkerSandboxResolverResolve(t *testing.T) { } } -func TestVersionBundleHash(t *testing.T) { - gvisor := map[string]assetEntry{gvisorAssetName: {URL: "gs://bucket/gvisor", SHA256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f"}} - pause := "sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" - - tests := []struct { - name string - pauseImage string - want string - wantErr bool - }{{ - name: "pause image pinned by digest", - pauseImage: "registry.k8s.io/pause:3.10.2@" + pause, - want: bundleHash(&sandboxAssetsRecord{Assets: gvisor}, pause), - }, { - name: "pause image pinned by digest only", - pauseImage: "gcr.io/gke-release/pause@" + pause, - want: bundleHash(&sandboxAssetsRecord{Assets: gvisor}, pause), - }, { - name: "no pause image", - want: bundleHash(&sandboxAssetsRecord{Assets: gvisor}, ""), - }, { - // The CRD rejects this shape. - name: "pause image by tag", - pauseImage: "registry.k8s.io/pause:3.10.2", - wantErr: true, - }} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := versionBundleHash(&sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: tt.pauseImage, Assets: gvisor}) - if (err != nil) != tt.wantErr { - t.Fatalf("versionBundleHash() error = %v, want error %v", err, tt.wantErr) - } - if err != nil && !errors.Is(err, errPermanent) { - t.Errorf("versionBundleHash() error = %v, want errPermanent", err) - } - if got != tt.want { - t.Errorf("versionBundleHash() = %q, want %q", got, tt.want) - } - }) - } -} - // wantRuntime is the runtime buildFile builds from multiVersionConfig's // version with the given name when runsc reports testCPUFeatures() ("3"). func wantRuntime(name string) sandboxruntimes.Attributes { @@ -298,7 +239,7 @@ func wantRuntimeWithFeatures(name, cpuFeatures string) sandboxruntimes.Attribute return sandboxruntimes.Attributes{ hardware.AttrArchitecture: runtime.GOARCH, hardware.AttrCPUFeatures: cpuFeatures, - hardware.AttrSandboxBundleHash: bundleHash(rec, pauseDigest(name)), + hardware.AttrSandboxBundleHash: bundleHash(rec), } } @@ -316,11 +257,7 @@ func planAndBuild(s *workerSandboxSyncer, pod *corev1.Pod) (*sandboxruntimes.Fil if err != nil { return nil, err } - hashed, err := hashVersions(context.Background(), plan) - if err != nil { - return nil, err - } - return s.buildFile(context.Background(), plan, hashed) + return s.buildFile(context.Background(), plan, hashVersions(plan)) } func TestPlanAndBuild(t *testing.T) { @@ -331,7 +268,6 @@ func TestPlanAndBuild(t *testing.T) { // sameAsDefault has v1 run the default version's binaries. sameAsDefault := multiVersionConfig() - sameAsDefault.Spec.Versions[1].PauseImage = sameAsDefault.Spec.Versions[3].PauseImage sameAsDefault.Spec.Versions[1].Assets = sameAsDefault.Spec.Versions[3].Assets // v1Twice adds an Enabled "v3" running v1's binaries. @@ -340,13 +276,6 @@ func TestPlanAndBuild(t *testing.T) { v3.Name = "v3" v1Twice.Spec.Versions = append(v1Twice.Spec.Versions, v3) - // The CRD rejects a pause image not pinned by digest. - tagged := func(i int) *v1alpha1.SandboxConfig { - cfg := multiVersionConfig() - cfg.Spec.Versions[i].PauseImage = "registry.k8s.io/pause:3.10.2" - return cfg - } - unsupportedClass := multiVersionConfig() unsupportedClass.Spec.SandboxClass = "unknown" @@ -381,16 +310,6 @@ func TestPlanAndBuild(t *testing.T) { cfg: defaultNoArch, wantErr: true, wantPermanent: true, - }, { - // No worker of the pool could start actors on this node. - name: "default version pause image not pinned by digest", - cfg: tagged(3), - wantErr: true, - wantPermanent: true, - }, { - name: "non-default version pause image not pinned by digest is left out", - cfg: tagged(1), - want: onlyDefault, }, { name: "unsupported sandbox class", cfg: unsupportedClass, @@ -454,7 +373,7 @@ func TestPlanAndBuildMicroVM(t *testing.T) { hardware.AttrArchitecture: runtime.GOARCH, hardware.AttrCPUVendor: testCPUVendor, hardware.AttrCPUModel: testCPUModel, - hardware.AttrSandboxBundleHash: bundleHash(rec, ""), + hardware.AttrSandboxBundleHash: bundleHash(rec), } } @@ -534,11 +453,11 @@ func TestGVisorCPUFeaturesMemoizationAndErrors(t *testing.T) { t.Run("memoizes runsc cpu-features by asset digest across versions and syncs", func(t *testing.T) { cfg := multiVersionConfig() // Add a third version "v3" that shares v1's gvisor asset digest ("sha-v1") - // but has its own pause image digest, so it has a distinct bundle hash - // while reusing the same runsc binary. + // but has one more asset, so it has a distinct bundle hash while it + // uses the same runsc binary. v3 := *cfg.Spec.Versions[1].DeepCopy() v3.Name = "v3" - v3.PauseImage = "registry.k8s.io/pause@" + pauseDigest("v3") + v3.Assets[runtime.GOARCH]["extra"] = v1alpha1.AssetFile{URL: "gs://bucket/extra", SHA256: "sha-extra"} cfg.Spec.Versions = append(cfg.Spec.Versions, v3) s := newTestSyncer(t, resolvedPool("pool", "gvisor-release"), cfg) @@ -560,7 +479,7 @@ func TestGVisorCPUFeaturesMemoizationAndErrors(t *testing.T) { if err != nil { t.Fatalf("first planAndBuild() error = %v", err) } - v3Rec := &sandboxAssetsRecord{Assets: map[string]assetEntry{gvisorAssetName: {SHA256: "sha-v1"}}} + v3Rec := &sandboxAssetsRecord{Assets: map[string]assetEntry{gvisorAssetName: {SHA256: "sha-v1"}, "extra": {SHA256: "sha-extra"}}} want := &sandboxruntimes.File{ DefaultRuntime: wantRuntimeWithFeatures("v2", "3"), RestorableRuntimes: []sandboxruntimes.Attributes{ @@ -568,7 +487,7 @@ func TestGVisorCPUFeaturesMemoizationAndErrors(t *testing.T) { { hardware.AttrArchitecture: runtime.GOARCH, hardware.AttrCPUFeatures: "1", - hardware.AttrSandboxBundleHash: bundleHash(v3Rec, pauseDigest("v3")), + hardware.AttrSandboxBundleHash: bundleHash(v3Rec), }, }, } diff --git a/cmd/ateworker-gvisor/ckptcgroup.go b/cmd/ateworker-gvisor/ckptcgroup.go index d3f332c985..e5e26eefa9 100644 --- a/cmd/ateworker-gvisor/ckptcgroup.go +++ b/cmd/ateworker-gvisor/ckptcgroup.go @@ -35,8 +35,8 @@ import ( "golang.org/x/sys/unix" ) -// During checkpoint, moving the sentry from -_pause into a sibling -// -_ckpt cgroup (memory.max = "max", cpu.max copied from -_pause) +// During checkpoint, moving the sentry from -_sandbox into a sibling +// -_ckpt cgroup (memory.max = "max", cpu.max copied from -_sandbox) // prevents newly written pages.img cache from evicting clean pages or // OOM-killing the sentry under the actor's memory.max. Removing -_ckpt // after sandbox teardown reparents the cache to the worker scope for fast @@ -45,7 +45,7 @@ import ( // POSIX_FADV_DONTNEED once background restore page loading completes. const ( - // checkpointCgroupSuffix uses a leading underscore like _pause so it cannot + // checkpointCgroupSuffix uses a leading underscore like _sandbox so it cannot // collide with a DNS-label application container leaf (-). checkpointCgroupSuffix = "-_ckpt" ) @@ -60,8 +60,8 @@ func checkpointCgroupDir(root, actorUID string) string { return filepath.Join(root, actorUID+checkpointCgroupSuffix) } -// isolateCheckpointCache moves the actor's sentry from its pause leaf into a -// sibling -_ckpt cgroup, preserving the pause leaf's cpu.max so checkpoint +// isolateCheckpointCache moves the actor's sentry from its sandbox leaf into a +// sibling -_ckpt cgroup, preserving the sandbox leaf's cpu.max so checkpoint // serialization remains capped to the actor's CPU limit. func isolateCheckpointCache(root string, actorDirs *ateworkerpb.ActorDirs, actorUID string) (*checkpointCgroup, error) { pid, err := readSandboxPID(actorDirs) @@ -69,7 +69,7 @@ func isolateCheckpointCache(root string, actorDirs *ateworkerpb.ActorDirs, actor return nil, err } - leaf := filepath.Join(root, ocispec.GVisorCgroupLeaf(actorUID, ocispec.PauseContainer)) + leaf := filepath.Join(root, ocispec.GVisorCgroupLeaf(actorUID, ocispec.SandboxID)) if !cgroupHasPID(leaf, pid) { return nil, fmt.Errorf("sandbox pid %d is not in cgroup %q", pid, leaf) } @@ -90,7 +90,7 @@ func isolateCheckpointCache(root string, actorDirs *ateworkerpb.ActorDirs, actor } func readSandboxPID(actorDirs *ateworkerpb.ActorDirs) (int, error) { - pidFile := pidFilePath(actorDirs, ocispec.PauseContainer) + pidFile := pidFilePath(actorDirs, ocispec.SandboxID) b, err := os.ReadFile(pidFile) if err != nil { return 0, fmt.Errorf("while reading sandbox pid file %q: %w", pidFile, err) @@ -120,7 +120,7 @@ func writeCgroupPID(dir string, pid int) error { return nil } -// restore moves the sentry back into its pause leaf on checkpoint failure. It +// restore moves the sentry back into its sandbox leaf on checkpoint failure. It // returns nil once the sentry has exited; any other failure leaves the sentry // alive in the checkpoint cgroup, so the caller must terminate it. func (c *checkpointCgroup) restore() error { @@ -151,7 +151,7 @@ func (c *checkpointCgroup) remove() error { // checkpointIsolated isolates the sentry into a sibling cgroup (when the // worker's cgroup is delegated), runs checkpoint, and tears down the workload // before removing the sibling cgroup so its page cache reparents cleanly. If -// checkpoint fails, it moves the sentry back into its pause leaf (or terminates +// checkpoint fails, it moves the sentry back into its sandbox leaf (or terminates // the workload if that rollback fails). func checkpointIsolated( ctx context.Context, @@ -177,9 +177,9 @@ func checkpointIsolated( } err := ckptCgroup.remove() if err != nil { - // If terminate left the sentry alive, move it back to its pause leaf. + // If terminate left the sentry alive, move it back to its sandbox leaf. if rerr := ckptCgroup.restore(); rerr != nil { - slog.WarnContext(ctx, "Failed to move the sentry back into its pause leaf", + slog.WarnContext(ctx, "Failed to move the sentry back into its sandbox leaf", slog.String("actorUID", actorUID), slog.Any("err", rerr)) } else { err = ckptCgroup.remove() @@ -193,7 +193,7 @@ func checkpointIsolated( if err := checkpoint(ctx); err != nil { if rerr := ckptCgroup.restore(); rerr != nil { - slog.WarnContext(ctx, "Failed to move the sentry back into its pause leaf; terminating workload", + slog.WarnContext(ctx, "Failed to move the sentry back into its sandbox leaf; terminating workload", slog.String("actorUID", actorUID), slog.Any("err", rerr)) } else { terminateOnExit = false diff --git a/cmd/ateworker-gvisor/ckptcgroup_test.go b/cmd/ateworker-gvisor/ckptcgroup_test.go index 7cb2c83ac9..dfca0796e9 100644 --- a/cmd/ateworker-gvisor/ckptcgroup_test.go +++ b/cmd/ateworker-gvisor/ckptcgroup_test.go @@ -50,14 +50,14 @@ func newCkptFixture(t *testing.T, sandboxPID string) ckptFixture { CheckpointDir: t.TempDir(), }, } - f.leaf = filepath.Join(f.root, ocispec.GVisorCgroupLeaf(testActorUID, ocispec.PauseContainer)) + f.leaf = filepath.Join(f.root, ocispec.GVisorCgroupLeaf(testActorUID, ocispec.SandboxID)) mustMkdir(t, f.leaf) mustWrite(t, filepath.Join(f.leaf, "cgroup.procs"), "100\n"+sandboxPID+"\n") mustWrite(t, filepath.Join(f.leaf, "memory.max"), "805306368\n") mustWrite(t, filepath.Join(f.leaf, "cpu.max"), "200000 100000\n") if sandboxPID != "" { - pidFile := pidFilePath(f.actorDirs, ocispec.PauseContainer) + pidFile := pidFilePath(f.actorDirs, ocispec.SandboxID) mustMkdir(t, filepath.Dir(pidFile)) mustWrite(t, pidFile, sandboxPID+"\n") } @@ -69,7 +69,7 @@ func newCkptFixture(t *testing.T, sandboxPID string) ckptFixture { return err } // Emulate cgroup2 migration out of the sibling checkpoint cgroup so rmdir - // succeeds once the sentry moves back into its pause leaf. + // succeeds once the sentry moves back into its sandbox leaf. ckptDir := checkpointCgroupDir(f.root, testActorUID) if dir != ckptDir { _ = os.Remove(filepath.Join(ckptDir, "cgroup.procs")) @@ -125,7 +125,7 @@ func TestIsolateCheckpointCacheMovesOnlyTheSentry(t *testing.T) { t.Fatalf("restore: %v", err) } if got := mustRead(t, filepath.Join(f.leaf, "cgroup.procs")); got != "101" { - t.Errorf("pause leaf cgroup.procs after restore = %q, want 101", got) + t.Errorf("sandbox leaf cgroup.procs after restore = %q, want 101", got) } if err := c.remove(); err != nil { t.Fatalf("remove: %v", err) @@ -136,25 +136,25 @@ func TestIsolateCheckpointCacheWithoutPIDFile(t *testing.T) { f := newCkptFixture(t, "") if _, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID); err == nil { - t.Fatal("isolateCheckpointCache succeeded with no pause pid file") + t.Fatal("isolateCheckpointCache succeeded with no sandbox pid file") } if _, err := os.Stat(checkpointCgroupDir(f.root, testActorUID)); !os.IsNotExist(err) { t.Errorf("checkpoint cgroup left behind: stat err = %v", err) } } -func TestIsolateCheckpointCacheRejectsPIDOutsidePauseLeaf(t *testing.T) { +func TestIsolateCheckpointCacheRejectsPIDOutsideSandboxLeaf(t *testing.T) { f := newCkptFixture(t, "101") mustWrite(t, filepath.Join(f.leaf, "cgroup.procs"), "100\n") if _, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID); err == nil { - t.Fatal("isolateCheckpointCache succeeded with the pid outside the pause leaf") + t.Fatal("isolateCheckpointCache succeeded with the pid outside the sandbox leaf") } if _, err := os.Stat(checkpointCgroupDir(f.root, testActorUID)); !os.IsNotExist(err) { t.Errorf("checkpoint cgroup created for a reused pid: stat err = %v", err) } if got := mustRead(t, filepath.Join(f.leaf, "cgroup.procs")); got != "100\n" { - t.Errorf("pause leaf cgroup.procs = %q, want unchanged \"100\\n\"", got) + t.Errorf("sandbox leaf cgroup.procs = %q, want unchanged \"100\\n\"", got) } } @@ -185,14 +185,14 @@ func TestCheckpointCgroupRestoreSkipsExitedSentry(t *testing.T) { } // If the sentry exited (no longer in ckpt cgroup.procs) and its PID was - // recycled elsewhere, restore must not move it into the pause leaf. + // recycled elsewhere, restore must not move it into the sandbox leaf. mustWrite(t, filepath.Join(c.dir, "cgroup.procs"), "") mustWrite(t, filepath.Join(f.leaf, "cgroup.procs"), "100\n") if err := c.restore(); err != nil { t.Fatalf("restore with exited sentry: %v", err) } if got := mustRead(t, filepath.Join(f.leaf, "cgroup.procs")); got != "100\n" { - t.Errorf("pause leaf cgroup.procs = %q, want unchanged \"100\\n\"", got) + t.Errorf("sandbox leaf cgroup.procs = %q, want unchanged \"100\\n\"", got) } // If the sentry exits between cgroupHasPID and movePID (ESRCH), restore returns nil. @@ -296,7 +296,7 @@ func TestCheckpointIsolated(t *testing.T) { t.Error("terminate called after successful cgroup rollback") } if got := mustRead(t, filepath.Join(f.leaf, "cgroup.procs")); got != "101" { - t.Errorf("pause leaf cgroup.procs = %q, want 101", got) + t.Errorf("sandbox leaf cgroup.procs = %q, want 101", got) } if _, err := os.Stat(ckptDir); !os.IsNotExist(err) { t.Errorf("checkpoint cgroup left behind: stat err = %v", err) @@ -332,7 +332,7 @@ func TestCheckpointIsolated(t *testing.T) { } }) - t.Run("terminates workload if checkpoint fails and the pause leaf is gone", func(t *testing.T) { + t.Run("terminates workload if checkpoint fails and the sandbox leaf is gone", func(t *testing.T) { f := newCkptFixture(t, "101") ckptDir := checkpointCgroupDir(f.root, testActorUID) var terminated bool @@ -356,14 +356,14 @@ func TestCheckpointIsolated(t *testing.T) { t.Fatal("checkpointIsolated succeeded on checkpoint error") } if !terminated { - t.Error("terminate was not called with the pause leaf gone; sentry would stay in unlimited cgroup") + t.Error("terminate was not called with the sandbox leaf gone; sentry would stay in unlimited cgroup") } if _, err := os.Stat(ckptDir); !os.IsNotExist(err) { t.Errorf("checkpoint cgroup left behind: stat err = %v", err) } }) - t.Run("moves the sentry back into its pause leaf when terminate leaves it running", func(t *testing.T) { + t.Run("moves the sentry back into its sandbox leaf when terminate leaves it running", func(t *testing.T) { f := newCkptFixture(t, "101") ckptDir := checkpointCgroupDir(f.root, testActorUID) @@ -380,7 +380,7 @@ func TestCheckpointIsolated(t *testing.T) { t.Fatalf("checkpointIsolated: %v", err) } if got := mustRead(t, filepath.Join(f.leaf, "cgroup.procs")); got != "101" { - t.Errorf("pause leaf cgroup.procs = %q, want the live sentry 101 back under its memory.max", got) + t.Errorf("sandbox leaf cgroup.procs = %q, want the live sentry 101 back under its memory.max", got) } if _, err := os.Stat(ckptDir); !os.IsNotExist(err) { t.Errorf("checkpoint cgroup left behind: stat err = %v", err) @@ -397,7 +397,7 @@ func TestRemoveStaleCheckpointCgroups(t *testing.T) { // A non-empty directory stands in for a cgroup that still has processes: // rmdir fails on both. mustWrite(t, filepath.Join(busy, "cgroup.procs"), strconv.Itoa(42)) - leaf := filepath.Join(root, ocispec.GVisorCgroupLeaf(testActorUID, ocispec.PauseContainer)) + leaf := filepath.Join(root, ocispec.GVisorCgroupLeaf(testActorUID, ocispec.SandboxID)) mustMkdir(t, leaf) removeStaleCheckpointCgroups(context.Background(), root) @@ -409,7 +409,7 @@ func TestRemoveStaleCheckpointCgroups(t *testing.T) { t.Errorf("busy checkpoint cgroup removed: %v", err) } if _, err := os.Stat(leaf); err != nil { - t.Errorf("pause leaf removed: %v", err) + t.Errorf("sandbox leaf removed: %v", err) } } diff --git a/cmd/ateworker-gvisor/cleanup_test.go b/cmd/ateworker-gvisor/cleanup_test.go index 655f765872..746f93ee78 100644 --- a/cmd/ateworker-gvisor/cleanup_test.go +++ b/cmd/ateworker-gvisor/cleanup_test.go @@ -83,14 +83,14 @@ func assertCalls(t *testing.T, f *fakeRunsc, want ...string) { var appContainers = []*ateworkerpb.Container{{Name: "app"}} func TestCleanupContainers_DeletesEveryContainerRunscKnows(t *testing.T) { - f := newFakeRunsc("app", "_pause") + f := newFakeRunsc("app", "_sandbox") if err := cleanupContainers(context.Background(), f, appContainers); err != nil { t.Fatalf("cleanupContainers: %v", err) } // Happy path: no `runsc list` at all. - assertCalls(t, f, "state app", "state _pause", "delete app", "delete _pause") + assertCalls(t, f, "state app", "state _sandbox", "delete app", "delete _sandbox") if len(f.containers) != 0 { t.Errorf("containers survived cleanup: %v", slices.Sorted(maps.Keys(f.containers))) } @@ -106,7 +106,7 @@ func TestCleanupContainers_SkipsContainersRunscHasNoRecordOf(t *testing.T) { t.Fatalf("cleanupContainers: %v", err) } - assertCalls(t, f, "state app", "state _pause", "list", "delete app") + assertCalls(t, f, "state app", "state _sandbox", "list", "delete app") } func TestCleanupContainers_SucceedsWhenEverythingIsAlreadyGone(t *testing.T) { @@ -116,10 +116,10 @@ func TestCleanupContainers_SucceedsWhenEverythingIsAlreadyGone(t *testing.T) { t.Fatalf("cleanupContainers: %v", err) } - assertCalls(t, f, "state app", "list", "state _pause", "list") + assertCalls(t, f, "state app", "list", "state _sandbox", "list") } -// The pause container is the sandbox: it must outlive the deletes of the +// The sandbox must outlive the deletes of the // containers inside it, so stopping the actor leaves it to cleanup. func TestStopContainersLeavesTheSandboxToCleanup(t *testing.T) { f := newFakeRunsc() diff --git a/cmd/ateworker-gvisor/crash_test.go b/cmd/ateworker-gvisor/crash_test.go index 151fd9a16b..dcd80d54ca 100644 --- a/cmd/ateworker-gvisor/crash_test.go +++ b/cmd/ateworker-gvisor/crash_test.go @@ -51,7 +51,7 @@ func exitingSandbox(t *testing.T) *ateworkerpb.ActorDirs { if err := os.MkdirAll(pidFileDir(actorDirs), 0o700); err != nil { t.Fatal(err) } - if err := os.WriteFile(pidFilePath(actorDirs, ocispec.PauseContainer), []byte(strconv.Itoa(cmd.Process.Pid)), 0o600); err != nil { + if err := os.WriteFile(pidFilePath(actorDirs, ocispec.SandboxID), []byte(strconv.Itoa(cmd.Process.Pid)), 0o600); err != nil { t.Fatal(err) } return actorDirs diff --git a/cmd/ateworker-gvisor/main.go b/cmd/ateworker-gvisor/main.go index 9c201942aa..8481a61ff5 100644 --- a/cmd/ateworker-gvisor/main.go +++ b/cmd/ateworker-gvisor/main.go @@ -629,7 +629,7 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run // Contract with atelet: // // * Correct runsc version is downloaded and placed on disk. - // * All OCI bundles are set up, including for the pause container. + // * The OCI bundles of the application containers are set up. egress, err := s.tunnel.PrepareEgress(ctx, workerstats.ActorAttributionFromRequest(req), req.GetEgressGateway()) if err != nil { @@ -671,31 +671,27 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run } } }() - // Create and start pause container. The bundle rootfs is composed here — - // an overlay of the node's cached image layers plus the bundle's private - // upper — because mounting is ateworker's job (atelet runs with no - // capabilities); runsc's gofer resolves the mount in this pod's mount - // namespace. - if err := imagecache.SetupBundleRootfs(ociBundlePath(req.GetActorDirs(), ocispec.PauseContainer)); err != nil { - return nil, fmt.Errorf("while composing pause rootfs: %w", err) - } - containersToDelete = append(containersToDelete, ocispec.PauseContainer) - // Attribute runsc CLI output and the pause container's stdio to the actor. - pw, err := s.actorLogger.StartJSONLogPipe(attribution, ocispec.PauseContainer) + containersToDelete = append(containersToDelete, ocispec.SandboxID) + // Attribute the sandbox's runsc CLI output to the actor. + pw, err := s.actorLogger.StartJSONLogPipe(attribution, ocispec.SandboxID) if err != nil { - return nil, fmt.Errorf("while starting pause log pipe: %w", err) + return nil, fmt.Errorf("while starting sandbox log pipe: %w", err) } defer pw.Close() - if err := rcmd.cmdCreate(ctx, pw, ocispec.PauseContainer, nil); err != nil { - return nil, fmt.Errorf("while creating pause container: %w", err) + if err := rcmd.cmdCreate(ctx, pw, ocispec.SandboxID, nil); err != nil { + return nil, fmt.Errorf("while creating the sandbox: %w", err) } s.pinSandbox(ctx, hosted, req.GetActorDirs()) - if err := rcmd.cmdStart(ctx, pw, ocispec.PauseContainer); err != nil { - return nil, fmt.Errorf("while starting pause container: %w", err) + if err := rcmd.cmdStart(ctx, pw, ocispec.SandboxID); err != nil { + return nil, fmt.Errorf("while starting the sandbox: %w", err) } // Create and start each application container, each with its own log pipe so // every line is tagged with the originating container (ate.actor.container.name). + // The bundle rootfs is composed here, as an overlay of the node's cached + // image layers plus the bundle's private upper. atelet runs with no + // capabilities, so ateworker mounts it, and runsc's gofer resolves the + // mount in this pod's mount namespace. for _, ac := range req.GetSpec().GetContainers() { pw, err := s.actorLogger.StartJSONLogPipe(attribution, ac.GetName()) if err != nil { @@ -831,7 +827,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke return nil, fmt.Errorf("no durable-dir volumes found for VOLUMES snapshot") } checkpointFn = func(ctx context.Context) error { - err := rcmd.cmdFsCheckpoint(ctx, ocispec.PauseContainer, checkpointPath, durablePaths) + err := rcmd.cmdFsCheckpoint(ctx, ocispec.SandboxID, checkpointPath, durablePaths) timing.fsCheckpoint = lap(&tLast) if err != nil { return fmt.Errorf("while checkpointing durable-dir volumes: %w", err) @@ -840,7 +836,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke } case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: checkpointFn = func(ctx context.Context) error { - err := rcmd.cmdFsCheckpoint(ctx, ocispec.PauseContainer, checkpointPath, fsPaths) + err := rcmd.cmdFsCheckpoint(ctx, ocispec.SandboxID, checkpointPath, fsPaths) timing.fsCheckpoint = lap(&tLast) if err != nil { return fmt.Errorf("while checkpointing container filesystems: %w", err) @@ -848,9 +844,9 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke return nil } case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: - // Checkpoint pause container (root of the sandbox), splitting container - // rootfs overlays into the rootfs bundle and durable-dir volumes into - // /fs when present. + // Checkpoint the sandbox, splitting container rootfs overlays into the + // rootfs bundle and durable-dir volumes into /fs when + // present. checkpointFn = func(ctx context.Context) error { var sandbox *sandboxProcess if hosted != nil { @@ -867,7 +863,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke defer sandbox.Close() } } - err := rcmd.cmdCheckpoint(ctx, ocispec.PauseContainer, checkpointPath, fsPaths) + err := rcmd.cmdCheckpoint(ctx, ocispec.SandboxID, checkpointPath, fsPaths) if err != nil { // the actor may stay RUNNING, so its sandbox dying is a crash // again, including if this checkpoint stopped it @@ -875,7 +871,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke hosted.stopping.Store(false) } timing.checkpoint = lap(&tLast) - return fmt.Errorf("while checkpointing pause: %w", err) + return fmt.Errorf("while checkpointing the sandbox: %w", err) } // the checkpoint stops the sandbox, runsc delete fails until it is reaped if sandbox != nil { @@ -936,10 +932,9 @@ func listSnapshotFiles(dir string) ([]string, error) { return files, nil } -// stopContainers stops the actor's application containers. The pause container -// is left running: it is the sandbox, and cleanupContainers deletes the others -// through it before deleting it last. Killing it first leaves the sentry a -// zombie until reaped, which runsc delete mistakes for a live sandbox. +// stopContainers stops the actor's application containers. The sandbox is left +// running: cleanupContainers deletes the others through it before deleting it +// last. runsc kill does not apply to a sandbox with no root container. func stopContainers(ctx context.Context, rcmd containerRuntime, containers []*ateworkerpb.Container) { for _, ctr := range containers { _ = rcmd.cmdKill(ctx, ctr.GetName(), "SIGKILL") @@ -948,12 +943,12 @@ func stopContainers(ctx context.Context, rcmd containerRuntime, containers []*at } func cleanupContainers(ctx context.Context, rcmd containerRuntime, containers []*ateworkerpb.Container) error { - // Application containers first, the pause (root) container last. + // Application containers first, the sandbox last. names := make([]string, 0, len(containers)+1) for _, ctr := range containers { names = append(names, ctr.GetName()) } - names = append(names, ocispec.PauseContainer) + names = append(names, ocispec.SandboxID) // Check state of all containers to mimic containerd. // Without this, `runsc delete` occasionally throws an error. @@ -1048,7 +1043,7 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb // Contract with atelet: // // * Correct runsc version is downloaded and placed on disk. - // * All OCI bundles are set up, including for the pause container. + // * The OCI bundles of the application containers are set up. // * Checkpoint downloaded and placed on disk egress, err := s.tunnel.PrepareEgress(ctx, attribution, req.GetEgressGateway()) @@ -1095,49 +1090,40 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb return nil, err } - // Compose the pause rootfs before create (see RunWorkload). runsc restore - // only needs the rootfs to hold the correct content; whether it came from - // an untar or an overlay of cached layers is transparent to it. - err = imagecache.SetupBundleRootfs(ociBundlePath(req.GetActorDirs(), ocispec.PauseContainer)) - timing.pauseRootfs = lap(&tLast) + // Attribute the sandbox's runsc CLI output to the actor. + pw, err := s.actorLogger.StartJSONLogPipe(attribution, ocispec.SandboxID) if err != nil { - return nil, fmt.Errorf("while composing pause rootfs: %w", err) - } - - // Attribute runsc CLI output and the pause container's stdio to the actor. - pw, err := s.actorLogger.StartJSONLogPipe(attribution, ocispec.PauseContainer) - if err != nil { - return nil, fmt.Errorf("while starting pause log pipe: %w", err) + return nil, fmt.Errorf("while starting sandbox log pipe: %w", err) } defer pw.Close() switch fidelity { case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: - // Create and start pause container (cold boot with filesystem checkpoint bundles restored) - containersToDelete = append(containersToDelete, ocispec.PauseContainer) - err := rcmd.cmdCreate(ctx, pw, ocispec.PauseContainer, restoreArgs) - timing.pauseCreate = lap(&tLast) + // Create and start the sandbox (cold boot with filesystem checkpoint bundles restored) + containersToDelete = append(containersToDelete, ocispec.SandboxID) + err := rcmd.cmdCreate(ctx, pw, ocispec.SandboxID, restoreArgs) + timing.sandboxCreate = lap(&tLast) if err != nil { - return nil, fmt.Errorf("while creating pause container: %w", err) + return nil, fmt.Errorf("while creating the sandbox: %w", err) } s.pinSandbox(ctx, hosted, req.GetActorDirs()) - err = rcmd.cmdStart(ctx, pw, ocispec.PauseContainer) - timing.pauseRestore = lap(&tLast) + err = rcmd.cmdStart(ctx, pw, ocispec.SandboxID) + timing.sandboxRestore = lap(&tLast) if err != nil { - return nil, fmt.Errorf("while starting pause container: %w", err) + return nil, fmt.Errorf("while starting the sandbox: %w", err) } case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: - // Create and restore pause container (restores process state and filesystem checkpoint bundles) - containersToDelete = append(containersToDelete, ocispec.PauseContainer) - err := rcmd.cmdCreate(ctx, pw, ocispec.PauseContainer, restoreArgs) - timing.pauseCreate = lap(&tLast) + // Create and restore the sandbox (restores process state and filesystem checkpoint bundles) + containersToDelete = append(containersToDelete, ocispec.SandboxID) + err := rcmd.cmdCreate(ctx, pw, ocispec.SandboxID, restoreArgs) + timing.sandboxCreate = lap(&tLast) if err != nil { - return nil, fmt.Errorf("while creating pause container: %w", err) + return nil, fmt.Errorf("while creating the sandbox: %w", err) } s.pinSandbox(ctx, hosted, req.GetActorDirs()) - err = rcmd.cmdRestore(ctx, pw, ocispec.PauseContainer, checkpointDir) - timing.pauseRestore = lap(&tLast) + err = rcmd.cmdRestore(ctx, pw, ocispec.SandboxID, checkpointDir) + timing.sandboxRestore = lap(&tLast) if err != nil { - return nil, fmt.Errorf("while restoring pause container: %w", err) + return nil, fmt.Errorf("while restoring the sandbox: %w", err) } } @@ -1191,7 +1177,7 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb } dropActorCheckpointCacheAsync(hosted.ctx, req.GetActorUid(), checkpointDir, req.GetPreserveRestoreDir(), func(waitCtx context.Context) error { if fidelity == ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { - if err := rcmd.cmdWaitRestore(waitCtx, ocispec.PauseContainer); err != nil { + if err := rcmd.cmdWaitRestore(waitCtx, ocispec.SandboxID); err != nil { return err } } diff --git a/cmd/ateworker-gvisor/main_test.go b/cmd/ateworker-gvisor/main_test.go index 84b4c5d2ac..71b7500d9a 100644 --- a/cmd/ateworker-gvisor/main_test.go +++ b/cmd/ateworker-gvisor/main_test.go @@ -162,9 +162,9 @@ func validTerminateRequest() *ateworkerpb.TerminateWorkloadRequest { return &ateworkerpb.TerminateWorkloadRequest{Atespace: testActor.Ref.Atespace, ActorName: testActor.Ref.Name, ActorUid: testActor.UID, Spec: testSpec(), ActorDirs: testDirs()} } -// TestPauseContainerLogAttribution covers the actor and _pause labels on runsc +// TestSandboxLogAttribution covers the actor and _sandbox labels on runsc // CLI output, JSON field preservation, and rejection of forged attribution. -func TestPauseContainerLogAttribution(t *testing.T) { +func TestSandboxLogAttribution(t *testing.T) { attribution := resources.ActorAttribution{ Ref: resources.ActorRef{Atespace: "default", Name: "act-1"}, UID: "uid-1", @@ -174,7 +174,7 @@ func TestPauseContainerLogAttribution(t *testing.T) { var buf syncLogBuffer logger := actorlog.NewActorLogger(&buf, false) - pw, err := logger.StartJSONLogPipe(attribution, ocispec.PauseContainer) + pw, err := logger.StartJSONLogPipe(attribution, ocispec.SandboxID) if err != nil { t.Fatalf("StartJSONLogPipe: %v", err) } @@ -184,7 +184,7 @@ func TestPauseContainerLogAttribution(t *testing.T) { if _, err := fmt.Fprintln(pw, jsonLine); err != nil { t.Fatalf("write json line: %v", err) } - const plainLine = "I1008 03:40:00.000000 1234 main.go:123] Args: [runsc --alsologtostderr create _pause]" + const plainLine = "I1008 03:40:00.000000 1234 main.go:123] Args: [runsc --alsologtostderr create _sandbox]" if _, err := fmt.Fprintln(pw, plainLine); err != nil { t.Fatalf("write plain line: %v", err) } @@ -220,8 +220,8 @@ func TestPauseContainerLogAttribution(t *testing.T) { if got := labels[actorName]; got != "act-1" { t.Errorf("%s label %s = %v, want act-1 (the forged value must not survive)", name, actorName, got) } - if got := labels[containerName]; got != ocispec.PauseContainer { - t.Errorf("%s label %s = %v, want %q", name, containerName, got, ocispec.PauseContainer) + if got := labels[containerName]; got != ocispec.SandboxID { + t.Errorf("%s label %s = %v, want %q", name, containerName, got, ocispec.SandboxID) } } } diff --git a/cmd/ateworker-gvisor/phaselog.go b/cmd/ateworker-gvisor/phaselog.go index 78ab0f2740..1413099695 100644 --- a/cmd/ateworker-gvisor/phaselog.go +++ b/cmd/ateworker-gvisor/phaselog.go @@ -33,17 +33,16 @@ import ( // over the application containers, which are restored one after another; the // container count rides along on the record so a reader can divide. const ( - phasePrep = "prep" - phaseEgressPrepare = "egress_prepare" - phaseNetSetup = "net_setup" - phasePauseRootfs = "pause_rootfs" - phasePauseCreate = "pause_create" - phasePauseRestore = "pause_restore" - phaseAppRootfs = "app_rootfs" - phaseAppCreate = "app_create" - phaseAppRestore = "app_restore" - phaseWakeupProbe = "wakeup_probe" - phaseActivate = "activate" + phasePrep = "prep" + phaseEgressPrepare = "egress_prepare" + phaseNetSetup = "net_setup" + phaseSandboxCreate = "sandbox_create" + phaseSandboxRestore = "sandbox_restore" + phaseAppRootfs = "app_rootfs" + phaseAppCreate = "app_create" + phaseAppRestore = "app_restore" + phaseWakeupProbe = "wakeup_probe" + phaseActivate = "activate" phaseCheckpoint = "checkpoint" phaseFSCheckpoint = "fs_checkpoint" @@ -57,13 +56,13 @@ const ( const containerCountKey = "ate.actor.container.count" // restoreTiming is the elapsed time of each RestoreWorkload phase. A phase -// left at zero never ran. Under VOLUMES and ROOTFS fidelity, pauseRestore and +// left at zero never ran. Under VOLUMES and ROOTFS fidelity, sandboxRestore and // appRestore time the cold start that stands in for the restore. type restoreTiming struct { - prep, egressPrepare, netSetup time.Duration - pauseRootfs, pauseCreate, pauseRestore time.Duration - appRootfs, appCreate, appRestore time.Duration - wakeupProbe, activate, total time.Duration + prep, egressPrepare, netSetup time.Duration + sandboxCreate, sandboxRestore time.Duration + appRootfs, appCreate, appRestore time.Duration + wakeupProbe, activate, total time.Duration } func (t restoreTiming) phases() []phaselog.Phase { @@ -71,9 +70,8 @@ func (t restoreTiming) phases() []phaselog.Phase { {Name: phasePrep, D: t.prep}, {Name: phaseEgressPrepare, D: t.egressPrepare}, {Name: phaseNetSetup, D: t.netSetup}, - {Name: phasePauseRootfs, D: t.pauseRootfs}, - {Name: phasePauseCreate, D: t.pauseCreate}, - {Name: phasePauseRestore, D: t.pauseRestore}, + {Name: phaseSandboxCreate, D: t.sandboxCreate}, + {Name: phaseSandboxRestore, D: t.sandboxRestore}, {Name: phaseAppRootfs, D: t.appRootfs}, {Name: phaseAppCreate, D: t.appCreate}, {Name: phaseAppRestore, D: t.appRestore}, diff --git a/cmd/ateworker-gvisor/phaselog_test.go b/cmd/ateworker-gvisor/phaselog_test.go index 11efbe3973..ac6356962a 100644 --- a/cmd/ateworker-gvisor/phaselog_test.go +++ b/cmd/ateworker-gvisor/phaselog_test.go @@ -48,18 +48,17 @@ func TestRestoreTimingPhases(t *testing.T) { t.Parallel() timing := restoreTiming{ - prep: 10 * time.Millisecond, - egressPrepare: 120 * time.Millisecond, - netSetup: 30 * time.Millisecond, - pauseRootfs: 5 * time.Millisecond, - pauseCreate: 200 * time.Millisecond, - pauseRestore: 900 * time.Millisecond, - appRootfs: 15 * time.Millisecond, - appCreate: 50 * time.Millisecond, - appRestore: 400 * time.Millisecond, - wakeupProbe: 250 * time.Millisecond, - activate: 2 * time.Millisecond, - total: 2 * time.Second, + prep: 10 * time.Millisecond, + egressPrepare: 120 * time.Millisecond, + netSetup: 30 * time.Millisecond, + sandboxCreate: 200 * time.Millisecond, + sandboxRestore: 900 * time.Millisecond, + appRootfs: 15 * time.Millisecond, + appCreate: 50 * time.Millisecond, + appRestore: 400 * time.Millisecond, + wakeupProbe: 250 * time.Millisecond, + activate: 2 * time.Millisecond, + total: 2 * time.Second, } attrs := phaselog.SnapshotPhaseAttrs(resources.ActorAttribution{UID: "uid-abc"}, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, phaselog.RestoreDurationKey, nil, timing.phases()) @@ -75,19 +74,18 @@ func TestRestoreTimingPhases(t *testing.T) { rec := renderPhaseRecord(t, attrs) for k, want := range map[string]float64{ - "ateworker.actor.restore.duration.prep": 0.01, - "ateworker.actor.restore.duration.egress_prepare": 0.12, - "ateworker.actor.restore.duration.net_setup": 0.03, - "ateworker.actor.restore.duration.pause_rootfs": 0.005, - "ateworker.actor.restore.duration.pause_create": 0.2, - "ateworker.actor.restore.duration.pause_restore": 0.9, - "ateworker.actor.restore.duration.app_rootfs": 0.015, - "ateworker.actor.restore.duration.app_create": 0.05, - "ateworker.actor.restore.duration.app_restore": 0.4, - "ateworker.actor.restore.duration.wakeup_probe": 0.25, - "ateworker.actor.restore.duration.activate": 0.002, - "ateworker.actor.restore.duration.total": 2, - "ate.actor.container.count": 2, + "ateworker.actor.restore.duration.prep": 0.01, + "ateworker.actor.restore.duration.egress_prepare": 0.12, + "ateworker.actor.restore.duration.net_setup": 0.03, + "ateworker.actor.restore.duration.sandbox_create": 0.2, + "ateworker.actor.restore.duration.sandbox_restore": 0.9, + "ateworker.actor.restore.duration.app_rootfs": 0.015, + "ateworker.actor.restore.duration.app_create": 0.05, + "ateworker.actor.restore.duration.app_restore": 0.4, + "ateworker.actor.restore.duration.wakeup_probe": 0.25, + "ateworker.actor.restore.duration.activate": 0.002, + "ateworker.actor.restore.duration.total": 2, + "ate.actor.container.count": 2, } { got, ok := rec[k].(float64) if !ok { diff --git a/cmd/ateworker-gvisor/runsc.go b/cmd/ateworker-gvisor/runsc.go index f41e6e7daf..18faacc6ec 100644 --- a/cmd/ateworker-gvisor/runsc.go +++ b/cmd/ateworker-gvisor/runsc.go @@ -27,6 +27,7 @@ import ( "slices" "strings" + "github.com/opencontainers/runtime-spec/specs-go" "go.ate.dev/substrate/internal/ocispec" "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/internal/sizing" @@ -82,27 +83,33 @@ func durableVolumeNames(spec *ateworkerpb.WorkloadSpec) []string { return slices.Compact(names) } -// pauseContainer is the sandbox root container. It only reaps, so it gets no -// capabilities and mounts nothing. -func pauseContainer() *ateworkerpb.Container { - return &ateworkerpb.Container{ - Name: ocispec.PauseContainer, - ContainerSpec: &ateworkerpb.ContainerSpec{Args: []string{"/pause"}}, +// noRootContainerArgs returns the flag that tells runsc the sandbox has no +// root container. runsc refuses the flag for any other container. +func noRootContainerArgs(containerName string) []string { + if containerName == ocispec.SandboxID { + return []string{"-no-root-container"} } + return nil } // writeSpec builds the container's OCI spec, shapes it for gVisor, and saves it // into the container's bundle. func (r *runsc) writeSpec(containerName string) error { - container := pauseContainer() - if containerName != ocispec.PauseContainer { + bundlePath := ociBundlePath(r.actorDirs, containerName) + var spec *specs.Spec + if containerName == ocispec.SandboxID { + // atelet makes only the application containers' bundles. + if err := os.MkdirAll(bundlePath, 0o700); err != nil { + return fmt.Errorf("while creating bundle %q: %w", bundlePath, err) + } + spec = ocispec.BuildSandbox(r.actorUID) + } else { i := slices.IndexFunc(r.containers, func(c *ateworkerpb.Container) bool { return c.GetName() == containerName }) if i < 0 { return fmt.Errorf("container %q is not in the workload spec", containerName) } - container = r.containers[i] + spec = ocispec.Build(ocispec.Options{ActorUID: r.actorUID, ActorDirs: r.actorDirs, Container: r.containers[i]}) } - spec := ocispec.Build(ocispec.Options{ActorUID: r.actorUID, ActorDirs: r.actorDirs, Container: container}) ocispec.ShapeGVisor(spec, ocispec.GVisorOptions{ ActorUID: r.actorUID, ActorDirs: r.actorDirs, @@ -110,7 +117,18 @@ func (r *runsc) writeSpec(containerName string) error { DurableVolumes: r.durableVolumes, Size: r.size, }) - return ocispec.Save(ociBundlePath(r.actorDirs, containerName), spec) + return ocispec.Save(bundlePath, spec) +} + +// createArgs builds the argv for `runsc create `. Factored out so +// the argument construction can be unit-tested without executing runsc. +func (r *runsc) createArgs(containerName string, additionalArgs []string) []string { + args := append(noRootContainerArgs(containerName), + "-bundle", ociBundlePath(r.actorDirs, containerName), + "-pid-file", pidFilePath(r.actorDirs, containerName), + ) + args = append(args, additionalArgs...) + return r.buildArgs("create", containerName, args...) } func (r *runsc) cmdCreate(ctx context.Context, out io.Writer, containerName string, additionalArgs []string) error { @@ -120,18 +138,7 @@ func (r *runsc) cmdCreate(ctx context.Context, out io.Writer, containerName stri return fmt.Errorf("while writing the OCI spec for %q: %w", containerName, err) } - args := make([]string, 0, 4+len(additionalArgs)) - args = append(args, - "-bundle", ociBundlePath(r.actorDirs, containerName), - "-pid-file", pidFilePath(r.actorDirs, containerName), - ) - args = append(args, additionalArgs...) - - cmd := exec.CommandContext( - ctx, - r.path, - r.buildArgs("create", containerName, args...)..., - ) + cmd := exec.CommandContext(ctx, r.path, r.createArgs(containerName, additionalArgs)...) cmd.Stdout = out cmd.Stderr = out @@ -210,15 +217,16 @@ func (r *runsc) cmdFsCheckpoint(ctx context.Context, containerName, checkpointPa // restoreArgs builds the argv for `runsc restore `. Factored out so // the argument construction can be unit-tested without executing runsc. func (r *runsc) restoreArgs(containerName, checkpointPath string) []string { - return r.buildArgs("restore", containerName, + args := append(noRootContainerArgs(containerName), "-bundle", ociBundlePath(r.actorDirs, containerName), "-image-path", checkpointPath, "-pid-file", pidFilePath(r.actorDirs, containerName), ) + return r.buildArgs("restore", containerName, args...) } -// We take a checkpoint only of the root container of the sandbox, but we need -// to call restore on each container, using the same checkpoint. +// We take a checkpoint only of the sandbox, but we need to call restore on each +// container, using the same checkpoint. func (r *runsc) cmdRestore(ctx context.Context, out io.Writer, containerName, checkpointPath string) error { slog.InfoContext(ctx, "About to run runsc restore", slog.String("container", containerName)) diff --git a/cmd/ateworker-gvisor/runsc_test.go b/cmd/ateworker-gvisor/runsc_test.go index badafe71be..16e50620ec 100644 --- a/cmd/ateworker-gvisor/runsc_test.go +++ b/cmd/ateworker-gvisor/runsc_test.go @@ -40,8 +40,8 @@ var testActorDirs = &ateworkerpb.ActorDirs{ OciBundleDir: "/node/actors/test-actor-123/bundle", } -// writeSpec builds a workload container's spec from the request and the pause -// container's from ateworker's own description. +// writeSpec builds a workload container's spec from the request, and a +// sandbox spec with no process and no rootfs. It makes the sandbox bundle. func TestWriteSpec(t *testing.T) { root := t.TempDir() actorDirs := &ateworkerpb.ActorDirs{RootDir: root, OciBundleDir: filepath.Join(root, "bundles")} @@ -50,18 +50,18 @@ func TestWriteSpec(t *testing.T) { actorDirs: actorDirs, containers: []*ateworkerpb.Container{{Name: "app", ContainerSpec: &ateworkerpb.ContainerSpec{Args: []string{"/app", "serve"}}}}, } - for name, wantArgs := range map[string][]string{ - "app": {"/app", "serve"}, - ocispec.PauseContainer: {"/pause"}, - } { - bundle := ociBundlePath(actorDirs, name) - if err := os.MkdirAll(bundle, 0o700); err != nil { - t.Fatal(err) - } + // atelet makes the application bundle; writeSpec does not make it. + if err := r.writeSpec("app"); err == nil { + t.Error("writeSpec(app) = nil without the bundle atelet makes") + } + if err := os.MkdirAll(ociBundlePath(actorDirs, "app"), 0o700); err != nil { + t.Fatal(err) + } + for _, name := range []string{"app", ocispec.SandboxID} { if err := r.writeSpec(name); err != nil { t.Fatalf("writeSpec(%q) = %v", name, err) } - b, err := os.ReadFile(filepath.Join(bundle, "config.json")) + b, err := os.ReadFile(filepath.Join(ociBundlePath(actorDirs, name), "config.json")) if err != nil { t.Fatal(err) } @@ -69,14 +69,17 @@ func TestWriteSpec(t *testing.T) { if err := json.Unmarshal(b, &spec); err != nil { t.Fatal(err) } - if !slices.Equal(spec.Process.Args, wantArgs) { - t.Errorf("%s args = %v, want %v", name, spec.Process.Args, wantArgs) - } if got := spec.Annotations["io.kubernetes.cri.container-name"]; got != name { t.Errorf("%s container-name annotation = %q, want the spec shaped for gVisor", name, got) } - if name == ocispec.PauseContainer && len(spec.Process.Capabilities.Bounding) != 0 { - t.Errorf("pause capabilities = %v, want none", spec.Process.Capabilities.Bounding) + if name == ocispec.SandboxID { + if spec.Process != nil || spec.Root != nil || spec.Mounts != nil { + t.Errorf("sandbox spec has process %v, root %v, mounts %v; want none", spec.Process, spec.Root, spec.Mounts) + } + continue + } + if want := []string{"/app", "serve"}; spec.Process == nil || !slices.Equal(spec.Process.Args, want) { + t.Errorf("%s process = %+v, want args %v", name, spec.Process, want) } } if err := r.writeSpec("missing"); err == nil { @@ -215,7 +218,7 @@ func TestCheckpointArgs(t *testing.T) { } checkpointDir := "/node/actors/test-actor-123/checkpoints/snap-1" - got := r.checkpointArgs(ocispec.PauseContainer, checkpointDir, []string{"rootfs/=app:/", "fs/=app:/var/data"}) + got := r.checkpointArgs(ocispec.SandboxID, checkpointDir, []string{"rootfs/=app:/", "fs/=app:/var/data"}) want := []string{ "-global1", "-global2", @@ -224,7 +227,7 @@ func TestCheckpointArgs(t *testing.T) { "-image-path", checkpointDir, "-fs-checkpoint-paths", "rootfs/=app:/", "-fs-checkpoint-paths", "fs/=app:/var/data", - ocispec.PauseContainer, + ocispec.SandboxID, } if !reflect.DeepEqual(got, want) { @@ -241,7 +244,7 @@ func TestFsCheckpointArgs(t *testing.T) { } checkpointDir := "/node/actors/test-actor-123/checkpoints/snap-1" - got := r.fsCheckpointArgs(ocispec.PauseContainer, checkpointDir, []string{"rootfs/=app:/", "fs/=app:/var/data"}) + got := r.fsCheckpointArgs(ocispec.SandboxID, checkpointDir, []string{"rootfs/=app:/", "fs/=app:/var/data"}) want := []string{ "-global1", "-global2", @@ -250,7 +253,7 @@ func TestFsCheckpointArgs(t *testing.T) { "-image-path", checkpointDir, "-path", "rootfs/=app:/", "-path", "fs/=app:/var/data", - ocispec.PauseContainer, + ocispec.SandboxID, } if !reflect.DeepEqual(got, want) { @@ -258,32 +261,98 @@ func TestFsCheckpointArgs(t *testing.T) { } } -func TestRestoreArgs(t *testing.T) { +// Only the sandbox gets -no-root-container: runsc refuses it for any other +// container. +func TestCreateArgs(t *testing.T) { r := &runsc{ path: "/usr/bin/runsc", actorUID: "test-actor-123", actorDirs: testActorDirs, flags: mustFlagStore(t), } - checkpointDir := "/node/actors/test-actor-123/checkpoints/snap-1" + for _, tc := range []struct { + container string + extra []string + want []string + }{{ + container: ocispec.SandboxID, + extra: []string{"-fs-restore-image-path", "/snap/fs"}, + want: []string{ + "-global1", + "-global2", + "-root", "/node/actors/test-actor-123/runsc-state", + "create", + "-no-root-container", + "-bundle", "/node/actors/test-actor-123/bundle/" + ocispec.SandboxID, + "-pid-file", "/node/actors/test-actor-123/pidfiles/" + ocispec.SandboxID + ".pid", + "-fs-restore-image-path", "/snap/fs", + ocispec.SandboxID, + }, + }, { + container: "app", + want: []string{ + "-global1", + "-global2", + "-root", "/node/actors/test-actor-123/runsc-state", + "create", + "-bundle", "/node/actors/test-actor-123/bundle/app", + "-pid-file", "/node/actors/test-actor-123/pidfiles/app.pid", + "app", + }, + }} { + if got := r.createArgs(tc.container, tc.extra); !reflect.DeepEqual(got, tc.want) { + t.Errorf("createArgs(%q) = %v, want %v", tc.container, got, tc.want) + } + } +} - got := r.restoreArgs(ocispec.PauseContainer, checkpointDir) - want := []string{ - "-global1", - "-global2", - "-restore-global", - "-root", "/node/actors/test-actor-123/runsc-state", - "restore", - "-restore-sub1", - "-restore-sub2", - "-bundle", "/node/actors/test-actor-123/bundle/" + ocispec.PauseContainer, - "-image-path", checkpointDir, - "-pid-file", "/node/actors/test-actor-123/pidfiles/" + ocispec.PauseContainer + ".pid", - ocispec.PauseContainer, +func TestRestoreArgs(t *testing.T) { + r := &runsc{ + path: "/usr/bin/runsc", + actorUID: "test-actor-123", + actorDirs: testActorDirs, + flags: mustFlagStore(t), } + checkpointDir := "/node/actors/test-actor-123/checkpoints/snap-1" - if !reflect.DeepEqual(got, want) { - t.Errorf("restoreArgs() = %v, want %v", got, want) + for _, tc := range []struct { + container string + want []string + }{{ + container: ocispec.SandboxID, + want: []string{ + "-global1", + "-global2", + "-restore-global", + "-root", "/node/actors/test-actor-123/runsc-state", + "restore", + "-restore-sub1", + "-restore-sub2", + "-no-root-container", + "-bundle", "/node/actors/test-actor-123/bundle/" + ocispec.SandboxID, + "-image-path", checkpointDir, + "-pid-file", "/node/actors/test-actor-123/pidfiles/" + ocispec.SandboxID + ".pid", + ocispec.SandboxID, + }, + }, { + container: "app", + want: []string{ + "-global1", + "-global2", + "-restore-global", + "-root", "/node/actors/test-actor-123/runsc-state", + "restore", + "-restore-sub1", + "-restore-sub2", + "-bundle", "/node/actors/test-actor-123/bundle/app", + "-image-path", checkpointDir, + "-pid-file", "/node/actors/test-actor-123/pidfiles/app.pid", + "app", + }, + }} { + if got := r.restoreArgs(tc.container, checkpointDir); !reflect.DeepEqual(got, tc.want) { + t.Errorf("restoreArgs(%q) = %v, want %v", tc.container, got, tc.want) + } } } diff --git a/cmd/ateworker-gvisor/sandboxexit.go b/cmd/ateworker-gvisor/sandboxexit.go index 6bbe327812..aedb0d5689 100644 --- a/cmd/ateworker-gvisor/sandboxexit.go +++ b/cmd/ateworker-gvisor/sandboxexit.go @@ -56,10 +56,10 @@ func openSandboxProcess(actorDirs *ateworkerpb.ActorDirs) (*sandboxProcess, erro return &sandboxProcess{pid: pid, pidfd: fd}, nil } -// sandboxPid reads the sandbox's pid from the pause container's pid file, -// which runsc writes for the sandbox. +// sandboxPid reads the sandbox's pid from the pid file runsc writes for the +// sandbox. func sandboxPid(actorDirs *ateworkerpb.ActorDirs) (int, error) { - b, err := os.ReadFile(pidFilePath(actorDirs, ocispec.PauseContainer)) + b, err := os.ReadFile(pidFilePath(actorDirs, ocispec.SandboxID)) if err != nil { return 0, err } diff --git a/cmd/ateworker-gvisor/sandboxexit_test.go b/cmd/ateworker-gvisor/sandboxexit_test.go index a2067de117..016e18cd6f 100644 --- a/cmd/ateworker-gvisor/sandboxexit_test.go +++ b/cmd/ateworker-gvisor/sandboxexit_test.go @@ -44,7 +44,7 @@ func startFakeSandbox(t *testing.T) (*exec.Cmd, *sandboxProcess) { if err := os.MkdirAll(pidFileDir(actorDirs), 0o700); err != nil { t.Fatal(err) } - pidFile := pidFilePath(actorDirs, ocispec.PauseContainer) + pidFile := pidFilePath(actorDirs, ocispec.SandboxID) if err := os.WriteFile(pidFile, []byte(strconv.Itoa(cmd.Process.Pid)), 0o600); err != nil { t.Fatal(err) } diff --git a/cmd/ateworker-gvisor/stats.go b/cmd/ateworker-gvisor/stats.go index dce914cd1d..d312cdd622 100644 --- a/cmd/ateworker-gvisor/stats.go +++ b/cmd/ateworker-gvisor/stats.go @@ -44,10 +44,10 @@ const defaultCgroupRoot = "/sys/fs/cgroup" // sandbox as a whole. // // gVisor runs every container of a sandbox as threads inside one host process, -// the sentry. runsc starts that process from the root container's create and -// from inside that container's cgroup — container.createRoot wraps the sandbox -// and gofer spawn in cgroup.RunInCgroup — so the sentry lands in the leaf of -// the pause container, the first one RunWorkload and RestoreWorkload create. +// the sentry. runsc starts that process from the sandbox's create, inside the +// sandbox's cgroup (container.createRoot wraps the sandbox spawn in +// cgroup.RunInCgroup), so the sentry lands in the sandbox's leaf. RunWorkload +// and RestoreWorkload create the sandbox first. // // The leaf is a direct child of the delegated scope rather than of ateworker's own // cgroup, because runsc resolves cgroupsPath against the parent of the cgroup @@ -72,11 +72,9 @@ const defaultCgroupRoot = "/sys/fs/cgroup" // // What the leaf holds besides the actor's own work: the sentry's own overhead // (its Go heap, page tables, netstack) and the gofers. Process listings taken -// on a live node in #161 put runsc-sandbox and both gofers — the pause -// container's and the actor container's — in the pause cgroup. Those runs -// predate #496, so they establish the leaf name and the fact that everything -// lands in one leaf, not the absolute path, which #496's delegation moved under -// the pod scope. +// on a live node in #161 put runsc-sandbox and the gofers in the sandbox's +// cgroup leaf. Those runs predate #496 and the rename to _sandbox, so they +// establish only that everything lands in one leaf. // // So this measures the sandbox, not the actor's processes in isolation, which // is what the proto means by "the unit of measurement is the SANDBOX". Sizing @@ -87,7 +85,7 @@ const defaultCgroupRoot = "/sys/fs/cgroup" // accounting, which the proto already says is not reported here. // // ocispec.GVisorCgroupLeaf supplies the leaf name for both shaping and stats. -const sandboxCgroupContainer = ocispec.PauseContainer +const sandboxCgroupContainer = ocispec.SandboxID // GetWorkloadStats implements ateworkerpb.Worker/GetWorkloadStats. // diff --git a/cmd/ateworker-gvisor/stats_test.go b/cmd/ateworker-gvisor/stats_test.go index 7a211acc7c..9fd8cb5ea5 100644 --- a/cmd/ateworker-gvisor/stats_test.go +++ b/cmd/ateworker-gvisor/stats_test.go @@ -635,7 +635,7 @@ func TestSweepUsageDeadSandboxConcurrentSweeps(t *testing.T) { } // drainedCgroup is the sandbox leaf after a drain has killed the app -// containers: the sentry and gofers stay in the pause leaf, so it is still +// containers: the sentry and gofers stay in the sandbox leaf, so it is still // populated. var drainedCgroup = map[string]string{ "memory.current": "31457280\n", @@ -671,18 +671,18 @@ func TestDeadSandboxCheckDuringDrain(t *testing.T) { } } -// TestDrainNeverKillsPauseContainer: gracefulShutdown kills the names +// TestDrainNeverKillsTheSandbox: gracefulShutdown kills the names // containerNames takes from the spec. atelet requires container names to be -// DNS-1123 labels (k8s-short-name) and the pause container's name is not one, +// DNS-1123 labels (k8s-short-name) and the sandbox ID is not one, // so no spec can name it and the drain cannot stop the sandbox itself. -func TestDrainNeverKillsPauseContainer(t *testing.T) { - if resources.IsValidResourceName(ocispec.PauseContainer) { - t.Fatalf("%q is a valid container name; a spec could then put it on the drain's kill list", ocispec.PauseContainer) +func TestDrainNeverKillsTheSandbox(t *testing.T) { + if resources.IsValidResourceName(ocispec.SandboxID) { + t.Fatalf("%q is a valid container name; a spec could then put it on the drain's kill list", ocispec.SandboxID) } spec := []*ateworkerpb.Container{{Name: "app"}, {Name: "sidecar"}} for _, name := range containerNames(spec) { - if name == ocispec.PauseContainer { - t.Fatalf("containerNames(%v) includes %q", spec, ocispec.PauseContainer) + if name == ocispec.SandboxID { + t.Fatalf("containerNames(%v) includes %q", spec, ocispec.SandboxID) } } } diff --git a/cmd/kubectl-ate/internal/cmd/actor.go b/cmd/kubectl-ate/internal/cmd/actor.go index 1304adb860..c1fcd70d42 100644 --- a/cmd/kubectl-ate/internal/cmd/actor.go +++ b/cmd/kubectl-ate/internal/cmd/actor.go @@ -747,7 +747,7 @@ func (f logLineFilter) matches(emitter resources.ActorRef, containerName string) } } if f.container == "" { - return containerName != ocispec.PauseContainer + return containerName != ocispec.SandboxID } return containerName == f.container } @@ -869,7 +869,7 @@ func init() { logsActorsCmd.Flags().BoolVarP(&logsActorFollowFlag, "follow", "f", false, "stream logs continuously") logsActorsCmd.Flags().StringVarP(&logsActorAtespaceFlag, "atespace", "a", "", "the actor's atespace") - logsActorsCmd.Flags().StringVarP(&logsActorContainerFlag, "container", "c", "", "show only logs from a named container; use _pause for runsc diagnostics") + logsActorsCmd.Flags().StringVarP(&logsActorContainerFlag, "container", "c", "", "show only logs from a named container; use _sandbox for runsc diagnostics") logsActorsCmd.Flags().StringVar(&logsActorSourceFlag, "source", string(logSourceAll), "which lines to show: all, containers (container output, no lifecycle events), or lifecycle (only the actor's lifecycle events)") logsCmd.AddCommand(logsActorsCmd) } diff --git a/cmd/kubectl-ate/internal/cmd/actor_test.go b/cmd/kubectl-ate/internal/cmd/actor_test.go index 4c5db51c3e..f7b72b79a1 100644 --- a/cmd/kubectl-ate/internal/cmd/actor_test.go +++ b/cmd/kubectl-ate/internal/cmd/actor_test.go @@ -523,22 +523,22 @@ func TestFilterAndDisplayLogLine(t *testing.T) { wantOutput: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"hi"}`, }, { - name: "no filter excludes pause container diagnostics", - line: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _pause]","labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"_pause"}}`, + name: "no filter excludes sandbox diagnostics", + line: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _sandbox]","labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"_sandbox"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: false, }, { - name: "explicit pause filter shows runsc diagnostics", - line: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _pause]","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"_pause"}}`, + name: "explicit sandbox filter shows runsc diagnostics", + line: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _sandbox]","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"_sandbox"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, - container: "_pause", + container: "_sandbox", wantMatched: true, - wantOutput: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _pause]"}`, + wantOutput: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _sandbox]"}`, }, { - name: "application filter excludes pause container diagnostics", - line: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _pause]","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"_pause"}}`, + name: "application filter excludes sandbox diagnostics", + line: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _sandbox]","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"_sandbox"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, container: "counter", wantMatched: false, @@ -785,9 +785,9 @@ func TestLogsActorRunner_Run_OneShot_SourceFilter(t *testing.T) { startedLine := `{"time":"2026-05-16T01:03:37Z","message":"Actor started","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123"}}` counterLine := `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"from counter","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123","ate.actor.container.name":"counter"}}` sidecarLine := `{"time":"2026-05-16T01:03:39Z","level":"info","msg":"from sidecar","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123","ate.actor.container.name":"sidecar"}}` - pauseLine := `{"time":"2026-05-16T01:03:39Z","message":"Args: [runsc create _pause]","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123","ate.actor.container.name":"_pause"}}` + sandboxLine := `{"time":"2026-05-16T01:03:39Z","message":"Args: [runsc create _sandbox]","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123","ate.actor.container.name":"_sandbox"}}` checkpointingLine := `{"time":"2026-05-16T01:03:40Z","message":"Actor checkpointing","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123"}}` - stream := strings.Join([]string{startedLine, counterLine, sidecarLine, pauseLine, checkpointingLine}, "\n") + "\n" + stream := strings.Join([]string{startedLine, counterLine, sidecarLine, sandboxLine, checkpointingLine}, "\n") + "\n" tests := []struct { name string @@ -800,7 +800,7 @@ func TestLogsActorRunner_Run_OneShot_SourceFilter(t *testing.T) { wantOutput: []string{"Actor started", "from counter", "from sidecar", "Actor checkpointing"}, }, { - name: "source=all excludes pause diagnostics", + name: "source=all excludes sandbox diagnostics", source: "all", wantOutput: []string{"Actor started", "from counter", "from sidecar", "Actor checkpointing"}, }, @@ -810,15 +810,15 @@ func TestLogsActorRunner_Run_OneShot_SourceFilter(t *testing.T) { wantOutput: []string{"from counter"}, }, { - name: "explicit pause container filter", - container: "_pause", - wantOutput: []string{"Args: [runsc create _pause]"}, + name: "explicit sandbox filter", + container: "_sandbox", + wantOutput: []string{"Args: [runsc create _sandbox]"}, }, { - name: "source=containers with explicit pause filter", + name: "source=containers with explicit sandbox filter", source: "containers", - container: "_pause", - wantOutput: []string{"Args: [runsc create _pause]"}, + container: "_sandbox", + wantOutput: []string{"Args: [runsc create _sandbox]"}, }, { name: "source=containers drops lifecycle events", diff --git a/docs/api-guide.md b/docs/api-guide.md index 1e465a5baf..cab8815a76 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -104,7 +104,7 @@ The `ActorTemplate` defines the code, environment, and state-management policies | `volumes` | `[]Volume` | Optional. Volumes the containers may mount, each a `durableDir`, an `external` CSI volume source (see [CSI Volumes Guide](csi-volumes.md)), or a `systemInfo` volume (see [SystemInfo Volumes](#systeminfo-volumes)). Every declared volume must be mounted by at least one container. A `durableDir` is created `0777`, as Kubernetes creates an emptyDir, so a container running as any user can write it. A snapshot restores it with the mode it had, so the container's own chmod survives suspend and resume. | | `resources` | `*ResourceRequirements` | Required. Declares each actor's compute size via `limits`, which must set both `cpuMillicores` and `memoryBytes` — see [Sandbox Right-Sizing](#sandbox-right-sizing-resources). Immutable, like the rest of the template. | -The sandbox runtime and its binaries (e.g. the gVisor `runsc` binary and `pauseImage`, or a micro-VM kernel/firmware/config) are configured on the [`WorkerPool`](#1-workerpool-the-compute-foundation) via its `sandboxClasses` and the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) it resolves, rather than on the `ActorTemplate`. An `ActorTemplate` selects its `WorkerPool`s via `workerSelector` (AND'd with the actor's `worker_selector`). When an actor runs on a worker, the worker's active `SandboxRuntime` is stamped onto the actor's assignment and snapshots so subsequent resumes prefer runtime-compatible workers. +The sandbox runtime and its binaries (e.g. the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) are configured on the [`WorkerPool`](#1-workerpool-the-compute-foundation) via its `sandboxClasses` and the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) it resolves, rather than on the `ActorTemplate`. An `ActorTemplate` selects its `WorkerPool`s via `workerSelector` (AND'd with the actor's `worker_selector`). When an actor runs on a worker, the worker's active `SandboxRuntime` is stamped onto the actor's assignment and snapshots so subsequent resumes prefer runtime-compatible workers. ### Sandbox Right-Sizing (`resources`) @@ -433,7 +433,7 @@ One consequence worth planning for: **a published snapshot is read from the ates ## 3. SandboxConfig: The Sandbox Itself -`SandboxConfig` is a **cluster-scoped** resource that decouples the sandbox — its binaries (the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) and the `pauseImage` that holds the sandbox's namespaces — from the workload definition in the `ActorTemplate`. A `WorkerPool`'s `sandboxClasses` entry resolves its `SandboxConfig` (either by explicit `configRef` or via the class default), and `atelet` provisions each worker pod with the enabled versions of that config, using `defaultVersion` for cold boots and keeping other enabled versions in `restorableRuntimes` so existing snapshots can still restore across upgrades. +`SandboxConfig` is a **cluster-scoped** resource that decouples the sandbox binaries (the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) from the workload definition in the `ActorTemplate`. A `WorkerPool`'s `sandboxClasses` entry resolves its `SandboxConfig` (either by explicit `configRef` or via the class default), and `atelet` provisions each worker pod with the enabled versions of that config, using `defaultVersion` for cold boots and keeping other enabled versions in `restorableRuntimes` so existing snapshots can still restore across upgrades. ### Specification (`SandboxConfigSpec`) @@ -449,7 +449,6 @@ Each entry of `versions` (`SandboxVersionConfig`): | :--- | :--- | :--- | | `name` | `string` | **Required.** Identifies the version within the config. A DNS label (lower-case alphanumerics and `-`, at most 63 characters). | | `state` | `string` | Optional. `Enabled` (default) or `Disabled`. The version named by `defaultVersion` cannot be `Disabled`. | -| `pauseImage` | `string` | **Required for `gvisor`; not allowed for `microvm`**, which runs no pause container. The image for the sandbox's root container (e.g. `registry.k8s.io/pause`, or `gcr.io/gke-release/pause` on GKE). Must include a digest (`...@sha256:...`) — it is recorded in each snapshot's manifest so a restore rebuilds the sandbox from the same image. | | `assets` | `map[arch]map[name]AssetFile` | Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A micro-VM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair; `url` is at most 2048 bytes. A `ValidatingAdmissionPolicy` enforces each class's required assets on every version. atelet holds no storage credentials: it fetches every URL through the node's object-store plugin (`AssetProvider`), with the plugin's credentials. Which schemes work depends on the plugin; the in-tree one serves only its storage backend's scheme, `gs://` on GCS and `s3://` on S3, as `/`, and rejects other schemes without reading them. A `gs://` URL the plugin cannot serve (it has no such object, rejects the URL, does not implement `AssetProvider`, is unreachable, or fails with an error it does not recognize, such as GCS refusing its credentials for a public bucket) is then read anonymously, which serves public buckets such as `gs://gvisor`; any other refusal by the plugin is final. The plugin returns an object only if its content matches `sha256`, and refuses any URL under a snapshot location (an `ActorTemplate`'s `snapshotConfig.location` followed by `/atespaces/`), so assets may share the snapshot bucket but must not be stored there. atelet verifies the sha256 again before using an asset. Its logs and errors show an asset URL as `scheme://host/path`. | A cluster-wide gVisor `SandboxConfig` (`gvisor-default`, annotated as the class default) is installed with the platform, so gVisor `WorkerPool`s use it automatically without any extra setup. @@ -493,7 +492,6 @@ spec: versions: - name: v4 state: Enabled - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" assets: amd64: gvisor: diff --git a/docs/architecture.md b/docs/architecture.md index 7b40dbe937..7b0d97be15 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -335,7 +335,7 @@ The node-level subsystem manages the physical execution of sandboxes and the mov ### Sandbox Classes -A `WorkerPool` selects a **sandbox class** (`spec.sandboxClasses[].name`), and each class has a matching `ateworker` herder image. The sandbox binaries themselves are not baked into the worker image — they, and the pause image holding the sandbox's namespaces, come at runtime from the cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) the `WorkerPool` resolves for its class (an explicit `configRef`, else the class default) and are pinned into each snapshot's manifest so restores stay reproducible across runtime upgrades. +A `WorkerPool` selects a **sandbox class** (`spec.sandboxClasses[].name`), and each class has a matching `ateworker` herder image. The sandbox binaries themselves are not baked into the worker image. They come at runtime from the cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) the `WorkerPool` resolves for its class (an explicit `configRef`, else the class default). Each snapshot records the sandbox runtime it was taken on, and a memory restore runs on a worker that still offers that runtime, so restores stay reproducible across runtime upgrades. * **gVisor** (`ateworker-gvisor`, the default): Runs the workload under `runsc` for kernel-level sandboxing. Suspend and resume leverage gVisor's native checkpoint/restore of the sandboxed process tree. `DurableDir` volumes and each container's rootfs overlay are saved as separate filesystem checkpoint bundles (`fs/` and `rootfs/`). A `MEMORY` snapshot splits them out of the process checkpoint; `VOLUMES` and `ROOTFS` snapshots save only the bundles they need with `runsc fscheckpoint`, without memory, and restore them on a cold boot. diff --git a/docs/glossary.md b/docs/glossary.md index 69e800fb48..156bf42d7d 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -24,11 +24,10 @@ For how the pieces fit together, see the [Architecture](architecture.md) and - **SandboxConfig** (Kubernetes CRD): a cluster-scoped resource holding the sandbox binaries for one runtime family (the gVisor `runsc` binary, or a - micro-VM kernel/firmware/config), plus the pause image for the sandbox's - root container. An actor resolves its sandbox at first cold boot from the - config its `ActorTemplate` names (naming one is currently required; - per-class cluster defaults are planned), so one config pins the runtime - version for many templates. + micro-VM kernel/firmware/config). An actor resolves its sandbox at first cold + boot from the config its `ActorTemplate` names (naming one is currently + required; per-class cluster defaults are planned), so one config pins the + runtime version for many templates. ## Records (dynamic state, in the control-plane store) diff --git a/docs/observability.md b/docs/observability.md index 60fa15aae8..15f872c60e 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -12,7 +12,7 @@ To make underlying infrastructure transitions transparent, Agent Substrate estab * `ate.actor.uid`: Server-assigned UID of the actor, unique to the lifetime of an actor. * `ate.template.name`: The name of the actor's ActorTemplate (e.g., `counter`). * `ate.template.atespace`: The atespace of the actor's ActorTemplate (e.g., `ate-demo-counter`). -* `ate.actor.container.name`: The name of the container within the actor that produced the log line (e.g., `counter`), so a multi-container actor's logs can be demultiplexed by container. Absent on the synthetic lifecycle records (`Actor starting`, `Actor restored`, …): those are about the actor, so no container produced them. The `runsc create`, `start`, and `restore` CLI output and the pause container's stdio are wrapped under `_pause`. This does not capture sentry or gofer output, which runsc directs to `/dev/null` when debug logging is disabled. +* `ate.actor.container.name`: The name of the container within the actor that produced the log line (e.g., `counter`), so a multi-container actor's logs can be demultiplexed by container. Absent on the synthetic lifecycle records (`Actor starting`, `Actor restored`, …): those are about the actor, so no container produced them. The `runsc create`, `start`, and `restore` CLI output for the sandbox is wrapped under `_sandbox`. This does not capture sentry or gofer output, which runsc directs to `/dev/null` when debug logging is disabled. Currently, Agent Substrate automatically wraps container output and injects these metadata labels into **container logs**. For metrics and distributed tracing, Agent Substrate provides foundational system telemetry and head-sampled request tracing (10% by default, 1% at the router), with roadmap plans to fully integrate actor-level correlation. @@ -68,19 +68,19 @@ Actor is currently running on pod ate-demo-counter/counter-ab123-x4y5z ``` #### Example 4: Filtering by Container or Source -An actor can run several containers. By default application container logs and synthetic lifecycle events (`Actor starting`, `Actor started`, `Actor checkpointing`, `Actor checkpointed`, `Actor restoring`, `Actor restored`, `Actor terminated`) are shown. The `_pause` logs are excluded so runsc startup diagnostics do not mix with application output. `--container` (short form `-c`) restricts the output to the named container's logs: +An actor can run several containers. By default application container logs and synthetic lifecycle events (`Actor starting`, `Actor started`, `Actor checkpointing`, `Actor checkpointed`, `Actor restoring`, `Actor restored`, `Actor terminated`) are shown. The `_sandbox` logs are excluded so runsc startup diagnostics do not mix with application output. `--container` (short form `-c`) restricts the output to the named container's logs: ```bash kubectl ate logs actor -a -c ``` -To view the pause container's runsc diagnostics explicitly: +To view the sandbox's runsc diagnostics explicitly: ```bash -kubectl ate logs actor -a -c _pause +kubectl ate logs actor -a -c _sandbox ``` -`--source` selects a class of lines instead of a container. `containers` shows application container output with the lifecycle events removed; `_pause` remains excluded unless selected with `-c _pause`. `lifecycle` shows only the lifecycle events, which is the quickest way to see how often an actor has been suspended and restored on its current worker (a migration moves the actor to another pod, whose log holds none of the earlier workers' events): +`--source` selects a class of lines instead of a container. `containers` shows application container output with the lifecycle events removed; `_sandbox` remains excluded unless selected with `-c _sandbox`. `lifecycle` shows only the lifecycle events, which is the quickest way to see how often an actor has been suspended and restored on its current worker (a migration moves the actor to another pod, whose log holds none of the earlier workers' events): ```bash $ kubectl ate logs actor test -a demo --source=lifecycle diff --git a/docs/sandbox-asset-cache.md b/docs/sandbox-asset-cache.md index f1e5131aad..13df476b5a 100644 --- a/docs/sandbox-asset-cache.md +++ b/docs/sandbox-asset-cache.md @@ -38,7 +38,7 @@ invalidates: changed content arrives under a new key. 1. ate-api-server places the actor on a worker and names, in `Run` or `Restore`, the **sandbox runtime** that worker registered. Its `sandbox_bundle_hash` attribute is the hash of one `SandboxConfig` - version's assets and pause image. + version's assets. 2. `atelet` resolves that hash in its **runtime index** to the asset record the hash was computed from. A hash the index does not hold fails the request with `FailedPrecondition`: the actor reached a node whose workers @@ -74,8 +74,8 @@ pool runs, hashes every enabled version, and: says; - makes the worker's `default-runtime/` directory match the file's **default** runtimes, and hands each to the **prewarmer**, which fetches - its assets into the cache, links them into the runtime's directory, and - pulls its pause image, so the first actor on the node finds a warm cache. + its assets into the cache and links them into the runtime's directory, so + the first actor on the node finds a warm cache. A `SandboxConfig` or `WorkerPool` change registers the affected pools' runtimes at once and rewrites their files after a jitter of up to thirty @@ -104,8 +104,8 @@ which directories exist. Nothing counts references and nothing prunes. permanent `runsc` probe failure does, leaves the file and so the directories as they were, with the previous default still linked and still advertised. The prewarm starts late by only a little: a gVisor default's - release is fetched by the build's probe, so the prewarm is links and the - pause image pull, and a micro-VM build takes milliseconds. + release is fetched by the build's probe, so the prewarm is links, and a + micro-VM build takes milliseconds. - The prewarm fills what exists and creates nothing. It checks that its directory is there, stages into it, and on an error checks again: a directory gone, after a later sync or the pod's removal, means the item diff --git a/internal/ateworkervalidation/validation_test.go b/internal/ateworkervalidation/validation_test.go index 5eaed65bbb..53520f9f93 100644 --- a/internal/ateworkervalidation/validation_test.go +++ b/internal/ateworkervalidation/validation_test.go @@ -437,7 +437,7 @@ func TestValidateContainer(t *testing.T) { want: field.ErrorList{field.Required(field.NewPath("name"), "")}, }, { name: "invalid name: the sandbox's own container", - obj: valid(func(c *ateworkerpb.Container) { c.Name = "_pause" }), + obj: valid(func(c *ateworkerpb.Container) { c.Name = "_sandbox" }), want: field.ErrorList{field.Invalid(field.NewPath("name"), nil, "").WithOrigin("format=k8s-short-name")}, }, { name: "too many external mounts", diff --git a/internal/hardware/hardware.go b/internal/hardware/hardware.go index ff858cf033..af796bcabf 100644 --- a/internal/hardware/hardware.go +++ b/internal/hardware/hardware.go @@ -31,8 +31,7 @@ const ( AttrArchitecture = "architecture" // AttrSandboxBundleHash identifies the sandbox runtime bundle: a - // sha256 over a SandboxConfig version's asset digests and pause image - // digest, computed by atelet. + // sha256 over a SandboxConfig version's asset digests, computed by atelet. AttrSandboxBundleHash = "sandbox_bundle_hash" // AttrCPUFeatures is the set of CPU features `runsc cpu-features` reports diff --git a/internal/imagecache/imagecache.go b/internal/imagecache/imagecache.go index 9afaa78ab9..630dc81f0f 100644 --- a/internal/imagecache/imagecache.go +++ b/internal/imagecache/imagecache.go @@ -148,10 +148,9 @@ var ( // sharedRegistries are the well-known communal registries whose rate limits // are shared across all anonymous clients. Everything not listed here is -// assumed dedicated: fleet-scale pulls realistically hit either the pause -// image (registry.k8s.io) or the customer's own registry, and enumerating -// the small stable set of communal hosts beats guessing at every private -// registry vendor. +// assumed dedicated: fleet-scale pulls realistically hit the customer's own +// registry, and enumerating the small stable set of communal hosts beats +// guessing at every private registry vendor. var sharedRegistries = map[string]bool{ "registry.k8s.io": true, // Pulls only ever present index.docker.io here: name.ParseReference diff --git a/internal/ocispec/gvisor.go b/internal/ocispec/gvisor.go index c532a0c67f..7d80fdd92c 100644 --- a/internal/ocispec/gvisor.go +++ b/internal/ocispec/gvisor.go @@ -22,17 +22,18 @@ import ( "go.ate.dev/substrate/internal/sizing" ) -// PauseContainer is the name of the sandbox root container. The underscore +// SandboxID is the runsc ID of the sandbox. runsc creates it with +// -no-root-container, so it has no process and no rootfs. The underscore // keeps it outside the k8s-short-name an ActorTemplate container // name is drawn from, so no actor container can collide with it. -const PauseContainer = "_pause" +const SandboxID = "_sandbox" // GVisorOptions describes the gVisor-specific context of one actor container. type GVisorOptions struct { ActorUID string ActorDirs *ateworkerpb.ActorDirs ContainerName string - // DurableVolumes are declared on the sandbox (pause) spec only. + // DurableVolumes are declared on the sandbox spec only. DurableVolumes []string // Size sizes the container's cgroup leaf. Only gVisor applies it; a micro-VM // container's limits come from its own declared resources (see sizing). @@ -51,7 +52,7 @@ func ShapeGVisor(spec *specs.Spec, o GVisorOptions) { spec.Annotations = make(map[string]string) } spec.Annotations["io.kubernetes.cri.container-name"] = o.ContainerName - if o.ContainerName == PauseContainer { + if o.ContainerName == SandboxID { spec.Annotations["io.kubernetes.cri.container-type"] = "sandbox" for _, v := range o.DurableVolumes { spec.Annotations["dev.gvisor.spec.mount."+v+".type"] = "tmpfs" @@ -60,7 +61,7 @@ func ShapeGVisor(spec *specs.Spec, o GVisorOptions) { } } else { spec.Annotations["io.kubernetes.cri.container-type"] = "container" - spec.Annotations["io.kubernetes.cri.sandbox-id"] = PauseContainer + spec.Annotations["io.kubernetes.cri.sandbox-id"] = SandboxID } if spec.Linux == nil { diff --git a/internal/ocispec/gvisor_test.go b/internal/ocispec/gvisor_test.go index 6d0ff33566..1a68788ccd 100644 --- a/internal/ocispec/gvisor_test.go +++ b/internal/ocispec/gvisor_test.go @@ -28,8 +28,8 @@ func TestShapeGVisorGivesEachActorItsOwnCgroupLeaf(t *testing.T) { return spec.Linux.CgroupsPath } - first := shape("uid-a", PauseContainer) - second := shape("uid-b", PauseContainer) + first := shape("uid-a", SandboxID) + second := shape("uid-b", SandboxID) if first == second { t.Errorf("two actors share the cgroup leaf %q", first) } @@ -41,15 +41,15 @@ func TestShapeGVisorGivesEachActorItsOwnCgroupLeaf(t *testing.T) { t.Errorf("cgroupsPath %q contains a colon", first) } - if a, b := shape("uid-a", PauseContainer), shape("uid-a", "app"); a == b { + if a, b := shape("uid-a", SandboxID), shape("uid-a", "app"); a == b { t.Errorf("two containers of one actor share the leaf %q", a) } } func TestGVisorCgroupLeafMatchesTheShapedPath(t *testing.T) { spec := &specs.Spec{} - ShapeGVisor(spec, GVisorOptions{ActorUID: "uid-a", ContainerName: PauseContainer}) - if want := "/" + GVisorCgroupLeaf("uid-a", PauseContainer); spec.Linux.CgroupsPath != want { + ShapeGVisor(spec, GVisorOptions{ActorUID: "uid-a", ContainerName: SandboxID}) + if want := "/" + GVisorCgroupLeaf("uid-a", SandboxID); spec.Linux.CgroupsPath != want { t.Errorf("shaped cgroupsPath = %q, want %q", spec.Linux.CgroupsPath, want) } } diff --git a/internal/ocispec/ocispec.go b/internal/ocispec/ocispec.go index 14e76e852f..7a59103e94 100644 --- a/internal/ocispec/ocispec.go +++ b/internal/ocispec/ocispec.go @@ -77,6 +77,27 @@ func ociResources(r *ateworkerpb.ResourceRequirements) *specs.LinuxResources { return out } +// BuildSandbox returns the OCI spec of a sandbox with no root container: the +// hostname and namespaces, and no process, rootfs, or mounts. +func BuildSandbox(actorUID string) *specs.Spec { + return &specs.Spec{ + Hostname: Hostname, + Linux: &specs.Linux{Namespaces: namespaces(actorUID)}, + } +} + +// namespaces returns the namespaces of an actor's sandbox and containers. The +// network namespace is the actor's. +func namespaces(actorUID string) []specs.LinuxNamespace { + return []specs.LinuxNamespace{ + {Type: "pid"}, + {Type: "network", Path: nodepath.ActorNetNSPath(actorUID)}, + {Type: "ipc"}, + {Type: "uts"}, + {Type: "mount"}, + } +} + // Build returns a runtime-neutral OCI spec for an actor container. func Build(o Options) *specs.Spec { container := o.Container @@ -138,25 +159,8 @@ func Build(o Options) *specs.Spec { }, }, Linux: &specs.Linux{ - Namespaces: []specs.LinuxNamespace{ - { - Type: "pid", - }, - { - Type: "network", - Path: nodepath.ActorNetNSPath(o.ActorUID), - }, - { - Type: "ipc", - }, - { - Type: "uts", - }, - { - Type: "mount", - }, - }, - Resources: ociResources(containerSpec.GetResources()), + Namespaces: namespaces(o.ActorUID), + Resources: ociResources(containerSpec.GetResources()), }, } diff --git a/internal/ocispec/ocispec_test.go b/internal/ocispec/ocispec_test.go index 1d5438f2ca..4b4e7fb101 100644 --- a/internal/ocispec/ocispec_test.go +++ b/internal/ocispec/ocispec_test.go @@ -137,9 +137,9 @@ func TestBuild_Capabilities(t *testing.T) { } } -// The pause container gets no capabilities. -func TestBuild_NoCapabilitiesForPause(t *testing.T) { - spec := Build(Options{Container: &ateworkerpb.Container{ContainerSpec: &ateworkerpb.ContainerSpec{Args: []string{"/pause"}}}}) +// A container that declares no capabilities gets none. +func TestBuild_NoCapabilities(t *testing.T) { + spec := Build(Options{Container: &ateworkerpb.Container{ContainerSpec: &ateworkerpb.ContainerSpec{Args: []string{"/app"}}}}) caps := spec.Process.Capabilities if caps == nil { diff --git a/internal/ocispec/parity_test.go b/internal/ocispec/parity_test.go index e341285761..f886892087 100644 --- a/internal/ocispec/parity_test.go +++ b/internal/ocispec/parity_test.go @@ -16,6 +16,7 @@ package ocispec import ( "encoding/json" + "slices" "strings" "testing" @@ -140,15 +141,25 @@ func TestShapeMicroVM_UnstagedSourceIsAnError(t *testing.T) { // Shaping a spec twice does not accumulate mounts. func TestShapeGVisor_Idempotent(t *testing.T) { spec := Build(parityOptions) - o := GVisorOptions{ActorUID: testActorUID, ActorDirs: parityActorDirs, ContainerName: PauseContainer, DurableVolumes: []string{"data"}, Size: paritySize} + o := GVisorOptions{ActorUID: testActorUID, ActorDirs: parityActorDirs, ContainerName: "app", Size: paritySize} ShapeGVisor(spec, o) first := len(spec.Mounts) ShapeGVisor(spec, o) if len(spec.Mounts) != first { t.Errorf("mount count = %d after a second shaping, want %d", len(spec.Mounts), first) } +} + +// The sandbox spec has no process, rootfs, or mounts, and declares the +// durable-dir mount hints. +func TestShapeGVisor_Sandbox(t *testing.T) { + spec := BuildSandbox(testActorUID) + ShapeGVisor(spec, GVisorOptions{ActorUID: testActorUID, ActorDirs: parityActorDirs, ContainerName: SandboxID, DurableVolumes: []string{"data"}, Size: paritySize}) + if spec.Process != nil || spec.Root != nil || spec.Mounts != nil { + t.Errorf("sandbox spec has process %v, root %v, mounts %v; want none", spec.Process, spec.Root, spec.Mounts) + } if got := spec.Annotations["io.kubernetes.cri.container-type"]; got != "sandbox" { - t.Errorf("pause container-type = %q, want sandbox", got) + t.Errorf("sandbox container-type = %q, want sandbox", got) } if got := spec.Annotations["dev.gvisor.spec.mount.data.type"]; got != "tmpfs" { t.Errorf("durable-dir mount type annotation = %q, want tmpfs", got) @@ -163,11 +174,25 @@ func TestShapeGVisor_Idempotent(t *testing.T) { // gVisor sizes the container's cgroup leaf from the actor-level limits: one // sentry backs every container, so the sandbox cgroup is the only one that -// binds. +// binds. The sandbox spec keeps its limits, cgroup, and namespaces. func TestShapeGVisor_AppliesTheActorSize(t *testing.T) { - spec := Build(parityOptions) - ShapeGVisor(spec, GVisorOptions{ActorUID: testActorUID, ContainerName: "app", Size: paritySize}) - if got := *spec.Linux.Resources.Memory.Limit; got != paritySize.MemoryBytes { - t.Errorf("memory limit = %d, want %d", got, paritySize.MemoryBytes) + for _, name := range []string{"app", SandboxID} { + spec := Build(parityOptions) + if name == SandboxID { + spec = BuildSandbox(testActorUID) + } + ShapeGVisor(spec, GVisorOptions{ActorUID: testActorUID, ContainerName: name, Size: paritySize}) + if got := *spec.Linux.Resources.Memory.Limit; got != paritySize.MemoryBytes { + t.Errorf("%s memory limit = %d, want %d", name, got, paritySize.MemoryBytes) + } + if want := "/" + GVisorCgroupLeaf(testActorUID, name); spec.Linux.CgroupsPath != want { + t.Errorf("%s cgroupsPath = %q, want %q", name, spec.Linux.CgroupsPath, want) + } + if !slices.ContainsFunc(spec.Linux.Namespaces, func(ns specs.LinuxNamespace) bool { return ns.Type == "network" && ns.Path != "" }) { + t.Errorf("%s namespaces = %v, want the actor's network namespace", name, spec.Linux.Namespaces) + } + if spec.Hostname == "" { + t.Errorf("%s hostname is empty", name) + } } } diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index fc781298ee..d7d8ba3e49 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -1682,9 +1682,9 @@ type RunRequest struct { // +k8s:optional Resources *ResourceRequirements `protobuf:"bytes,10,opt,name=resources,proto3" json:"resources,omitempty"` // The runtime the assigned worker runs this actor under, from - // WorkerAssignment.sandbox_runtime. atelet resolves the pause image and - // sandbox binaries through its advertised-runtime index and records the - // runtime for the actor so a later Checkpoint pins the snapshot to it. + // WorkerAssignment.sandbox_runtime. atelet resolves the sandbox binaries + // through its advertised-runtime index and records the runtime for the + // actor so a later Checkpoint pins the snapshot to it. // // +k8s:required SandboxRuntime *SandboxRuntime `protobuf:"bytes,12,opt,name=sandbox_runtime,json=sandboxRuntime,proto3" json:"sandbox_runtime,omitempty"` @@ -3600,10 +3600,10 @@ type RestoreRequest struct { // +k8s:optional Resources *ResourceRequirements `protobuf:"bytes,10,opt,name=resources,proto3" json:"resources,omitempty"` // The runtime the assigned worker restores this actor under, from - // WorkerAssignment.sandbox_runtime. atelet resolves the pause image and - // sandbox binaries through its advertised-runtime index and records the - // runtime for the actor so the next Checkpoint pins the snapshot to what ran - // this activation, not what the restored snapshot was taken under. + // WorkerAssignment.sandbox_runtime. atelet resolves the sandbox binaries + // through its advertised-runtime index and records the runtime for the + // actor so the next Checkpoint pins the snapshot to what ran this + // activation, not what the restored snapshot was taken under. // // +k8s:required SandboxRuntime *SandboxRuntime `protobuf:"bytes,12,opt,name=sandbox_runtime,json=sandboxRuntime,proto3" json:"sandbox_runtime,omitempty"` diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index b1ab598586..24e17cef7e 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -485,9 +485,9 @@ message RunRequest { ResourceRequirements resources = 10; // The runtime the assigned worker runs this actor under, from - // WorkerAssignment.sandbox_runtime. atelet resolves the pause image and - // sandbox binaries through its advertised-runtime index and records the - // runtime for the actor so a later Checkpoint pins the snapshot to it. + // WorkerAssignment.sandbox_runtime. atelet resolves the sandbox binaries + // through its advertised-runtime index and records the runtime for the + // actor so a later Checkpoint pins the snapshot to it. // // +k8s:required SandboxRuntime sandbox_runtime = 12; @@ -1101,10 +1101,10 @@ message RestoreRequest { ResourceRequirements resources = 10; // The runtime the assigned worker restores this actor under, from - // WorkerAssignment.sandbox_runtime. atelet resolves the pause image and - // sandbox binaries through its advertised-runtime index and records the - // runtime for the actor so the next Checkpoint pins the snapshot to what ran - // this activation, not what the restored snapshot was taken under. + // WorkerAssignment.sandbox_runtime. atelet resolves the sandbox binaries + // through its advertised-runtime index and records the runtime for the + // actor so the next Checkpoint pins the snapshot to what ran this + // activation, not what the restored snapshot was taken under. // // +k8s:required SandboxRuntime sandbox_runtime = 12; diff --git a/internal/proto/ateworkerpb/ateworker.pb.go b/internal/proto/ateworkerpb/ateworker.pb.go index 322b7d1648..01670638ff 100644 --- a/internal/proto/ateworkerpb/ateworker.pb.go +++ b/internal/proto/ateworkerpb/ateworker.pb.go @@ -721,8 +721,8 @@ func (x *WorkloadSpec) GetContainers() []*Container { // +k8s:customValidation # mount paths are unique and do not nest across the four lists type Container struct { state protoimpl.MessageState `protogen:"open.v1"` - // The sandbox's own container, ocispec.PauseContainer, is not a short - // name, so the format keeps a workload container from taking its bundle. + // The sandbox ID, ocispec.SandboxID, is not a short name, so the format + // keeps a workload container from taking its bundle. // // +k8s:required // +k8s:format=k8s-short-name diff --git a/internal/proto/ateworkerpb/ateworker.proto b/internal/proto/ateworkerpb/ateworker.proto index b31556c3d7..fa23e6725a 100644 --- a/internal/proto/ateworkerpb/ateworker.proto +++ b/internal/proto/ateworkerpb/ateworker.proto @@ -273,8 +273,8 @@ message WorkloadSpec { // // +k8s:customValidation # mount paths are unique and do not nest across the four lists message Container { - // The sandbox's own container, ocispec.PauseContainer, is not a short - // name, so the format keeps a workload container from taking its bundle. + // The sandbox ID, ocispec.SandboxID, is not a short name, so the format + // keeps a workload container from taking its bundle. // // +k8s:required // +k8s:format=k8s-short-name diff --git a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml index cdb8a6512d..9e0c24cfc0 100644 --- a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml +++ b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml @@ -92,8 +92,8 @@ spec: keyed by name. items: description: |- - SandboxVersionConfig is one version of a SandboxConfig: the exact pause image - and asset set a sandbox boots with. + SandboxVersionConfig is one version of a SandboxConfig: the asset set a + sandbox boots with. properties: assets: additionalProperties: @@ -143,26 +143,6 @@ spec: minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string - pauseImage: - description: |- - pauseImage is the container image used as the root sandbox container. - It holds the sandbox's namespaces and runs no workload code, so it is an - implementation detail of the sandbox rather than something actor authors - choose. Required for gvisor; not allowed for microvm, which runs no pause - container. It is captured in the snapshot manifest alongside the sandbox - binaries, so a restore always re-creates the sandbox from the same image - the snapshot was taken with. - - Typically, set it to [1] for on-gcp, and [2] for off-gcp - - - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da - - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 - maxLength: 512 - minLength: 1 - type: string - x-kubernetes-validations: - - message: All images must include a digest - rule: self.contains('@') state: default: Enabled description: |- @@ -191,11 +171,6 @@ spec: - message: the version named by defaultVersion must not be Disabled rule: '!self.versions.exists(v, v.name == self.defaultVersion && has(v.state) && v.state == ''Disabled'')' - - message: pauseImage is required on every version for gvisor and not - allowed for other sandbox classes - rule: 'self.sandboxClass == ''gvisor'' ? self.versions.all(v, has(v.pauseImage) - && size(v.pauseImage) > 0) : self.versions.all(v, !has(v.pauseImage) - || size(v.pauseImage) == 0)' required: - spec type: object diff --git a/manifests/ate-install/sandboxconfig-gvisor.yaml b/manifests/ate-install/sandboxconfig-gvisor.yaml index 908877e29c..99ef0fcc6a 100644 --- a/manifests/ate-install/sandboxconfig-gvisor.yaml +++ b/manifests/ate-install/sandboxconfig-gvisor.yaml @@ -36,12 +36,10 @@ spec: # gVisor nightly 2026-10-17, the first to include google/gvisor@b79256080dec # (save and restore multiple filesystem checkpoint bundles). ateworker-gvisor # uses it to save container rootfs overlays (rootfs/) and durable-dir volumes - # (fs/) as separate bundles. + # (fs/) as separate bundles, and to run the sandbox with no root container + # (google/gvisor#15551). - name: v4 state: Enabled - # The root sandbox container's image. On GCP, prefer the in-project mirror - # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da. - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" assets: amd64: gvisor: diff --git a/pkg/api/v1alpha1/sandboxconfig_types.go b/pkg/api/v1alpha1/sandboxconfig_types.go index 3672affbff..3c5276c418 100644 --- a/pkg/api/v1alpha1/sandboxconfig_types.go +++ b/pkg/api/v1alpha1/sandboxconfig_types.go @@ -96,8 +96,8 @@ const ( SandboxVersionStateDisabled SandboxVersionState = "Disabled" ) -// SandboxVersionConfig is one version of a SandboxConfig: the exact pause image -// and asset set a sandbox boots with. +// SandboxVersionConfig is one version of a SandboxConfig: the asset set a +// sandbox boots with. type SandboxVersionConfig struct { // name identifies this version within its SandboxConfig. It is a DNS label // (lower-case alphanumerics and '-'). @@ -115,28 +115,6 @@ type SandboxVersionConfig struct { // +kubebuilder:default=Enabled State *SandboxVersionState `json:"state,omitempty"` - // TODO: drop PauseImage once gVisor can run without a pause container: - // https://github.com/google/gvisor/pull/13981 - - // pauseImage is the container image used as the root sandbox container. - // It holds the sandbox's namespaces and runs no workload code, so it is an - // implementation detail of the sandbox rather than something actor authors - // choose. Required for gvisor; not allowed for microvm, which runs no pause - // container. It is captured in the snapshot manifest alongside the sandbox - // binaries, so a restore always re-creates the sandbox from the same image - // the snapshot was taken with. - // - // Typically, set it to [1] for on-gcp, and [2] for off-gcp - // - // - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da - // - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 - // - // +optional - // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:MaxLength=512 - // +kubebuilder:validation:XValidation:rule="self.contains('@')",message="All images must include a digest" - PauseImage string `json:"pauseImage,omitempty"` - // assets is the set of files atelet fetches for this runtime, keyed first by // architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The // asset names are interpreted by the sandbox backend: gVisor expects a @@ -154,7 +132,6 @@ type SandboxVersionConfig struct { // // +kubebuilder:validation:XValidation:rule="self.versions.exists(v, v.name == self.defaultVersion)",message="defaultVersion must name an entry in versions" // +kubebuilder:validation:XValidation:rule="!self.versions.exists(v, v.name == self.defaultVersion && has(v.state) && v.state == 'Disabled')",message="the version named by defaultVersion must not be Disabled" -// +kubebuilder:validation:XValidation:rule="self.sandboxClass == 'gvisor' ? self.versions.all(v, has(v.pauseImage) && size(v.pauseImage) > 0) : self.versions.all(v, !has(v.pauseImage) || size(v.pauseImage) == 0)",message="pauseImage is required on every version for gvisor and not allowed for other sandbox classes" type SandboxConfigSpec struct { // sandboxClass is the sandbox runtime family this config applies to. A // WorkerPool only uses SandboxConfigs whose sandboxClass matches the entry diff --git a/pkg/api/v1alpha1/sandboxconfig_validation_test.go b/pkg/api/v1alpha1/sandboxconfig_validation_test.go index 5336a93296..6860130a6a 100644 --- a/pkg/api/v1alpha1/sandboxconfig_validation_test.go +++ b/pkg/api/v1alpha1/sandboxconfig_validation_test.go @@ -40,15 +40,9 @@ const gvisorAssetMsg = "'gvisor' (release tarball) asset" // the install — loaded here so the test guards the policy we actually ship. const vapManifestPath = "../../../manifests/ate-install/sandboxconfig-validation.yaml" -const validPauseImage = "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" - -// sandboxVersion returns a version with a pause image iff the class needs one. -func sandboxVersion(name string, class SandboxClass, assets map[string]map[string]AssetFile) SandboxVersionConfig { - v := SandboxVersionConfig{Name: name, Assets: assets} - if class == SandboxClassGvisor { - v.PauseImage = validPauseImage - } - return v +// sandboxVersion returns a version with the given name and assets. +func sandboxVersion(name string, assets map[string]map[string]AssetFile) SandboxVersionConfig { + return SandboxVersionConfig{Name: name, Assets: assets} } // sandboxConfig returns a config with a single default version "v1". @@ -58,18 +52,11 @@ func sandboxConfig(name string, class SandboxClass, assets map[string]map[string Spec: SandboxConfigSpec{ SandboxClass: class, DefaultVersion: "v1", - Versions: []SandboxVersionConfig{sandboxVersion("v1", class, assets)}, + Versions: []SandboxVersionConfig{sandboxVersion("v1", assets)}, }, } } -// withPauseImage overrides the pause image of the first version on an -// otherwise-valid config. -func withPauseImage(sc *SandboxConfig, image string) *SandboxConfig { - sc.Spec.Versions[0].PauseImage = image - return sc -} - // withVersions appends versions to a config. func withVersions(sc *SandboxConfig, versions ...SandboxVersionConfig) *SandboxConfig { sc.Spec.Versions = append(sc.Spec.Versions, versions...) @@ -223,29 +210,14 @@ func TestSandboxConfigValidation(t *testing.T) { sc: sandboxConfig("bad-sha", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": {URL: "gs://bucket/gvisor.tar.bz2", SHA256: "deadbeef"}}}), wantErr: true, errMsg: "sha256", - }, { - name: "gvisor missing pauseImage", - sc: withPauseImage(sandboxConfig("bad-no-pause", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}}), ""), - wantErr: true, - errMsg: "pauseImage is required on every version for gvisor", - }, { - name: "microvm with pauseImage", - sc: withPauseImage(sandboxConfig("bad-microvm-pause", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": microVMAssets()}), validPauseImage), - wantErr: true, - errMsg: "pauseImage is required on every version for gvisor", - }, { - name: "unpinned pauseImage", - sc: withPauseImage(sandboxConfig("bad-unpinned-pause", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}}), "registry.k8s.io/pause:3.10.2"), - wantErr: true, - errMsg: "All images must include a digest", }, { name: "valid gvisor with two versions", - sc: withVersions(sandboxConfig("ok-two-versions", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", SandboxClassGvisor, gvisorAssets())), + sc: withVersions(sandboxConfig("ok-two-versions", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", gvisorAssets())), wantErr: false, }, { name: "valid non-default disabled version", sc: withVersions(sandboxConfig("ok-disabled-old", SandboxClassGvisor, gvisorAssets()), func() SandboxVersionConfig { - v := sandboxVersion("v0", SandboxClassGvisor, gvisorAssets()) + v := sandboxVersion("v0", gvisorAssets()) v.State = ptr.To(SandboxVersionStateDisabled) return v }()), @@ -271,7 +243,7 @@ func TestSandboxConfigValidation(t *testing.T) { errMsg: "spec.versions", }, { name: "duplicate version names", - sc: withVersions(sandboxConfig("bad-dup-versions", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v1", SandboxClassGvisor, gvisorAssets())), + sc: withVersions(sandboxConfig("bad-dup-versions", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v1", gvisorAssets())), wantErr: true, errMsg: "Duplicate value", }, { @@ -279,7 +251,7 @@ func TestSandboxConfigValidation(t *testing.T) { sc: func() *SandboxConfig { sc := sandboxConfig("bad-many-versions", SandboxClassGvisor, gvisorAssets()) for i := range 16 { - sc.Spec.Versions = append(sc.Spec.Versions, sandboxVersion(fmt.Sprintf("extra-%d", i), SandboxClassGvisor, gvisorAssets())) + sc.Spec.Versions = append(sc.Spec.Versions, sandboxVersion(fmt.Sprintf("extra-%d", i), gvisorAssets())) } return sc }(), @@ -295,32 +267,14 @@ func TestSandboxConfigValidation(t *testing.T) { }(), wantErr: true, errMsg: "spec.versions[0].name", - }, { - name: "gvisor second version missing pauseImage", - sc: withVersions(sandboxConfig("bad-v2-no-pause", SandboxClassGvisor, gvisorAssets()), func() SandboxVersionConfig { - v := sandboxVersion("v2", SandboxClassGvisor, gvisorAssets()) - v.PauseImage = "" - return v - }()), - wantErr: true, - errMsg: "pauseImage is required on every version for gvisor", - }, { - name: "microvm second version with pauseImage", - sc: withVersions(sandboxConfig("bad-microvm-v2-pause", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": microVMAssets()}), func() SandboxVersionConfig { - v := sandboxVersion("v2", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": microVMAssets()}) - v.PauseImage = validPauseImage - return v - }()), - wantErr: true, - errMsg: "pauseImage is required on every version for gvisor", }, { name: "gvisor second version missing the gvisor asset", - sc: withVersions(sandboxConfig("bad-v2-no-gvisor", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"other": otherAsset()}})), + sc: withVersions(sandboxConfig("bad-v2-no-gvisor", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", map[string]map[string]AssetFile{"amd64": {"other": otherAsset()}})), wantErr: true, errMsg: gvisorAssetMsg, }, { name: "microvm second version missing an asset", - sc: withVersions(sandboxConfig("bad-microvm-v2", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": microVMAssets()}), sandboxVersion("v2", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": func() map[string]AssetFile { + sc: withVersions(sandboxConfig("bad-microvm-v2", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": microVMAssets()}), sandboxVersion("v2", map[string]map[string]AssetFile{"amd64": func() map[string]AssetFile { m := microVMAssets() delete(m, "kata-kernel") return m @@ -367,7 +321,7 @@ func TestSandboxConfigSandboxClassDefault(t *testing.T) { func TestSandboxConfigUpdateRemovingDefaultVersion(t *testing.T) { ctx := t.Context() - sc := withVersions(sandboxConfig("update-remove-default", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", SandboxClassGvisor, gvisorAssets())) + sc := withVersions(sandboxConfig("update-remove-default", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", gvisorAssets())) if err := k8sClient.Create(ctx, sc); err != nil { t.Fatalf("Create() unexpected error: %v", err) } From 0279fe3dd37284c652ed0f30e47d6f6784501989 Mon Sep 17 00:00:00 2001 From: Chuang Wang Date: Wed, 21 Oct 2026 00:46:05 +0000 Subject: [PATCH 33/63] atunnel uses less memory for idle egress connections (#2908) Fixes #2759 Each idle egress tunnel held about 157 KiB and 5 goroutines: a 32 KiB io.Copy buffer per direction in copyBothWays, and a 32 KiB buffer held by the HTTP/2 stream's reader goroutine while it waited. At a 1 GiB limit, atunnel was OOM-killed at around 6,000 idle connections, which breaks egress for every actor on the worker. Changes: - copyBothWays waits for data with a 512 B buffer and borrows a pooled 32 KiB buffer only while data is moving. - h2StreamConn.WriteTo reads the response body directly, so the proxy doesn't start a reader goroutine per stream. With a read deadline set, it falls back to Read. benchmarking/atunnel, 1 CPU, 1 GiB: | | before | after | |---|---|---| | EgressIdleConns, per connection | 157 KiB, 5.1 goroutines | 29 KiB, 4.0 goroutines | | EgressIdleConns, 8,000 connections | OOM-killed | 226 MiB | | EgressTCP (average of 3 runs) | 311 MB/s | 302 MB/s | | EgressHTTP at 8,000 req/s | 1.00 core | 0.83 core | EgressTCP is about 3% slower because bulk copies do a few extra small reads. Most of the remaining 29 KiB is net/http's 16 KiB HTTP/2 send buffer, which this doesn't touch. - [x] Tests pass - [ ] Appropriate changes to documentation are included in the PR --- internal/atunnel/egress/client.go | 57 ++++++++ internal/atunnel/egress/client_test.go | 147 +++++++++++++++++++++ internal/atunnel/egress/egress.go | 14 +- internal/atunnel/egress/streamcopy.go | 108 +++++++++++++++ internal/atunnel/egress/streamcopy_test.go | 136 +++++++++++++++++++ 5 files changed, 459 insertions(+), 3 deletions(-) create mode 100644 internal/atunnel/egress/streamcopy.go create mode 100644 internal/atunnel/egress/streamcopy_test.go diff --git a/internal/atunnel/egress/client.go b/internal/atunnel/egress/client.go index 5c844c1f1a..d33db9167a 100644 --- a/internal/atunnel/egress/client.go +++ b/internal/atunnel/egress/client.go @@ -791,6 +791,56 @@ func (c *h2StreamConn) Read(p []byte) (int, error) { } } +// writeTo copies the stream to w until EOF or an error. +func (c *h2StreamConn) writeTo(w io.Writer) (int64, error) { + c.readMu.Lock() + if isClosedChan(c.closedCh) { + c.readMu.Unlock() + return 0, net.ErrClosed + } + var written int64 + if len(c.readPending) > 0 { + n, err := w.Write(c.readPending) + written += int64(n) + c.readPending = c.readPending[n:] + if err != nil { + c.readMu.Unlock() + return written, err + } + } + if err := c.readErr; err != nil { + c.readMu.Unlock() + if err == io.EOF { + return written, nil + } + return written, err + } + // A pending Read or a read deadline needs Read's background reader. + if c.readInFlight || c.readDeadline.isSet() { + c.readMu.Unlock() + n, err := (&streamCopy{dest: w, src: c}).do() + return written + n, err + } + defer c.readMu.Unlock() + n, err := (&streamCopy{dest: w, src: h2StreamBody{c}}).do() + return written + n, err +} + +// h2StreamBody reads an h2StreamConn's response body directly. The caller holds +// readMu. +type h2StreamBody struct{ c *h2StreamConn } + +func (b h2StreamBody) Read(p []byte) (int, error) { + n, err := b.c.respBody.Read(p) + if err != nil { + if isClosedChan(b.c.closedCh) { + err = net.ErrClosed + } + b.c.readErr = err + } + return n, err +} + func (c *h2StreamConn) Write(p []byte) (int, error) { c.writeMu.Lock() defer c.writeMu.Unlock() @@ -975,6 +1025,13 @@ func (d *streamDeadline) set(t time.Time) { } } +// isSet reports whether a deadline is pending or has passed. +func (d *streamDeadline) isSet() bool { + d.mu.Lock() + defer d.mu.Unlock() + return d.timer != nil || isClosedChan(d.cancel) +} + func (d *streamDeadline) wait() chan struct{} { d.mu.Lock() defer d.mu.Unlock() diff --git a/internal/atunnel/egress/client_test.go b/internal/atunnel/egress/client_test.go index 8054cc31e0..ee164a6d17 100644 --- a/internal/atunnel/egress/client_test.go +++ b/internal/atunnel/egress/client_test.go @@ -833,3 +833,150 @@ func issueDNSCertificate(t *testing.T, ca *testCA, dnsName string) tls.Certifica } return cert } + +// serveTestH2EchoGateway serves an HTTP/2 CONNECT gateway that echoes each +// stream back until the client ends it. +func serveTestH2EchoGateway(t *testing.T, ca *testCA) string { + t.Helper() + address, _ := serveTestH2ConnectGateway(t, ca, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + rc := http.NewResponseController(w) + _ = rc.Flush() + buf := make([]byte, 8<<10) + for { + n, err := r.Body.Read(buf) + if n > 0 { + if _, werr := w.Write(buf[:n]); werr != nil { + return + } + _ = rc.Flush() + } + if err != nil { + return + } + } + })) + return address +} + +func TestCopyBothWaysHTTP2Stream(t *testing.T) { + ca := newTestCA(t) + client := newTestClient(t, ca, withDialer(dialFixedAddress(serveTestH2EchoGateway(t, ca)))) + upstream, err := client.DialContext(context.Background(), "192.0.2.10:443") + if err != nil { + t.Fatal(err) + } + defer upstream.Close() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + app, err := net.Dial("tcp", listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer app.Close() + downstream, err := listener.Accept() + if err != nil { + t.Fatal(err) + } + defer downstream.Close() + + copied := make(chan struct{}) + go func() { + copyBothWays(downstream, upstream) + close(copied) + }() + + // Mix write sizes around streamCopy's buffer sizes, then end the stream. + var sent []byte + for i, size := range []int{1, streamCopySmallBufSize, streamCopySmallBufSize + 1, streamCopyBufSize, 3 << 20} { + chunk := make([]byte, size) + for j := range chunk { + chunk[j] = byte(i + j) + } + sent = append(sent, chunk...) + } + writeErr := make(chan error, 1) + go func() { + _, err := app.Write(sent) + if err == nil { + err = app.(*net.TCPConn).CloseWrite() + } + writeErr <- err + }() + + got, err := io.ReadAll(app) + if err != nil { + t.Fatalf("reading echo: %v", err) + } + if err := receiveWithin(t, writeErr, "write to finish"); err != nil { + t.Fatalf("writing: %v", err) + } + if len(got) != len(sent) || string(got) != string(sent) { + t.Fatalf("echo returned %d bytes, want the %d sent", len(got), len(sent)) + } + receiveWithin(t, copied, "copyBothWays to return") +} + +func TestH2StreamConnWriteToStopsOnClose(t *testing.T) { + ca := newTestCA(t) + client := newTestClient(t, ca, withDialer(dialFixedAddress(serveTestH2EchoGateway(t, ca)))) + conn, err := client.DialContext(context.Background(), "192.0.2.10:443") + if err != nil { + t.Fatal(err) + } + stream := conn.(*h2StreamConn) + + done := make(chan error, 1) + go func() { + _, err := stream.writeTo(io.Discard) + done <- err + }() + time.Sleep(20 * time.Millisecond) + _ = conn.Close() + if err := receiveWithin(t, done, "writeTo to return"); !errors.Is(err, net.ErrClosed) { + t.Fatalf("writeTo after Close = %v, want net.ErrClosed", err) + } +} + +func TestH2StreamConnWriteToHonorsReadDeadline(t *testing.T) { + ca := newTestCA(t) + releaseServerSend := make(chan struct{}) + gatewayAddress, _ := serveTestH2ConnectGateway(t, ca, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _ = http.NewResponseController(w).Flush() + <-releaseServerSend + _, _ = io.WriteString(w, "delayed") + })) + client := newTestClient(t, ca, withDialer(dialFixedAddress(gatewayAddress))) + conn, err := client.DialContext(context.Background(), "192.0.2.10:443") + if err != nil { + t.Fatal(err) + } + defer conn.Close() + stream := conn.(*h2StreamConn) + + if err := conn.SetReadDeadline(time.Now().Add(20 * time.Millisecond)); err != nil { + t.Fatal(err) + } + if _, err := stream.writeTo(io.Discard); !errors.Is(err, os.ErrDeadlineExceeded) { + t.Fatalf("writeTo error = %v, want os.ErrDeadlineExceeded", err) + } + + // The timed-out read is still in flight, so writeTo must collect its + // result instead of reading the body itself. + if err := conn.SetReadDeadline(time.Time{}); err != nil { + t.Fatal(err) + } + close(releaseServerSend) + var got strings.Builder + if _, err := stream.writeTo(&got); err != nil { + t.Fatalf("writeTo after clearing deadline: %v", err) + } + if got.String() != "delayed" { + t.Errorf("got %q, want delayed", got.String()) + } +} diff --git a/internal/atunnel/egress/egress.go b/internal/atunnel/egress/egress.go index c3d7017555..6f9c0f6d77 100644 --- a/internal/atunnel/egress/egress.go +++ b/internal/atunnel/egress/egress.go @@ -18,7 +18,6 @@ import ( "context" "errors" "fmt" - "io" "log/slog" "math/rand/v2" "net" @@ -328,12 +327,12 @@ func (e *Proxy) handle(downstream net.Conn, active *activation) { func copyBothWays(a, b net.Conn) { done := make(chan struct{}, 2) go func() { - _, _ = io.Copy(a, b) + _, _ = copyConn(a, b) closeWrite(a) done <- struct{}{} }() go func() { - _, _ = io.Copy(b, a) + _, _ = copyConn(b, a) closeWrite(b) done <- struct{}{} }() @@ -341,6 +340,15 @@ func copyBothWays(a, b net.Conn) { <-done } +// copyConn copies src to dst until EOF. Unlike io.Copy, it doesn't hold a 32 KiB +// buffer per direction while the tunnel is idle. +func copyConn(dst, src net.Conn) (int64, error) { + if stream, ok := src.(*h2StreamConn); ok { + return stream.writeTo(dst) + } + return (&streamCopy{dest: dst, src: src}).do() +} + func closeWrite(conn net.Conn) { if conn, ok := conn.(interface{ CloseWrite() error }); ok { _ = conn.CloseWrite() diff --git a/internal/atunnel/egress/streamcopy.go b/internal/atunnel/egress/streamcopy.go new file mode 100644 index 0000000000..a77d74bed5 --- /dev/null +++ b/internal/atunnel/egress/streamcopy.go @@ -0,0 +1,108 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egress + +import ( + "fmt" + "io" + "sync" +) + +const ( + // streamCopySmallBufSize is the buffer streamCopy holds while src is idle. + streamCopySmallBufSize = 512 + // streamCopyBufSize is the buffer streamCopy borrows while data is arriving. + streamCopyBufSize = 32 << 10 +) + +var streamCopyBufPool = sync.Pool{ + New: func() any { + buf := make([]byte, streamCopyBufSize) + return &buf + }, +} + +// streamCopy copies src to dest and counts the bytes read and written. +type streamCopy struct { + dest io.Writer + src io.Reader + + read uint64 + written uint64 +} + +// do copies src to dest until EOF or an error. Unlike io.Copy, it waits for +// data with a small buffer and borrows a large one from a pool only while data +// keeps arriving, so an idle stream holds little memory. Errors include the +// bytes read and written so far. +func (s *streamCopy) do() (written int64, err error) { + read := func(p []byte) (int, error) { + n, err := s.src.Read(p) + s.read += uint64(n) + return n, err + } + write := func(p []byte) error { + if len(p) == 0 { + return nil + } + n, err := s.dest.Write(p) + s.written += uint64(n) + if err == nil && n < len(p) { + err = io.ErrShortWrite + } + if err != nil { + return fmt.Errorf("egress: writing stream after %d bytes read, %d written: %w", s.read, s.written, err) + } + return nil + } + finish := func(readErr error) (int64, error) { + if readErr == io.EOF { + return int64(s.written), nil + } + return int64(s.written), fmt.Errorf("egress: reading stream after %d bytes read, %d written: %w", s.read, s.written, readErr) + } + + smallBuf := make([]byte, streamCopySmallBufSize) + for { + n, readErr := read(smallBuf) + if err := write(smallBuf[:n]); err != nil { + return int64(s.written), err + } + if readErr != nil { + return finish(readErr) + } + if n < len(smallBuf) { + continue + } + + // The read filled smallBuf, so more data is likely queued. Switch to + // the larger buffer for the next reads. + buf := streamCopyBufPool.Get().(*[]byte) + for { + n, readErr = read(*buf) + err = write((*buf)[:n]) + if err != nil || readErr != nil || n < len(*buf) { + break + } + } + streamCopyBufPool.Put(buf) + if err != nil { + return int64(s.written), err + } + if readErr != nil { + return finish(readErr) + } + } +} diff --git a/internal/atunnel/egress/streamcopy_test.go b/internal/atunnel/egress/streamcopy_test.go new file mode 100644 index 0000000000..7478927e9d --- /dev/null +++ b/internal/atunnel/egress/streamcopy_test.go @@ -0,0 +1,136 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egress + +import ( + "errors" + "fmt" + "io" + "strings" + "testing" +) + +// chunkReader returns its data in reads of the given sizes, cycling through +// them, so tests control what each Read sees. +type chunkReader struct { + data []byte + sizes []int + reads int +} + +func (r *chunkReader) Read(p []byte) (int, error) { + if len(r.data) == 0 { + return 0, io.EOF + } + n := min(r.sizes[r.reads%len(r.sizes)], len(p), len(r.data)) + r.reads++ + copy(p, r.data[:n]) + r.data = r.data[n:] + return n, nil +} + +func TestStreamCopyCopiesEverything(t *testing.T) { + data := make([]byte, 1<<20) + for i := range data { + data[i] = byte(i * 7) + } + for _, sizes := range [][]int{ + {1}, + {streamCopySmallBufSize - 1}, + {streamCopySmallBufSize}, + {streamCopySmallBufSize + 1}, + {streamCopySmallBufSize, 1, streamCopyBufSize, 3}, + {streamCopyBufSize}, + {1 << 20}, + } { + t.Run(fmt.Sprint(sizes), func(t *testing.T) { + var got strings.Builder + c := &streamCopy{dest: &got, src: &chunkReader{data: data, sizes: sizes}} + n, err := c.do() + if err != nil { + t.Fatalf("do: %v", err) + } + if n != int64(len(data)) || got.String() != string(data) { + t.Fatalf("do copied %d bytes (%d match), want %d", n, len(got.String()), len(data)) + } + if c.read != uint64(len(data)) || c.written != uint64(len(data)) { + t.Errorf("read, written = %d, %d; want %d for both", c.read, c.written, len(data)) + } + }) + } +} + +type errWriter struct{ err error } + +func (w errWriter) Write(p []byte) (int, error) { return len(p) / 2, w.err } + +func TestStreamCopyReturnsErrors(t *testing.T) { + readErr := errors.New("read failed") + _, err := (&streamCopy{dest: io.Discard, src: io.MultiReader(strings.NewReader("abc"), errReader{readErr})}).do() + if !errors.Is(err, readErr) || !strings.Contains(err.Error(), "reading stream after 3 bytes read, 3 written") { + t.Errorf("do with failing reader = %v, want %v after 3 bytes read and written", err, readErr) + } + writeErr := errors.New("write failed") + n, err := (&streamCopy{dest: errWriter{writeErr}, src: strings.NewReader("abcd")}).do() + if !errors.Is(err, writeErr) || n != 2 || !strings.Contains(err.Error(), "writing stream after 4 bytes read, 2 written") { + t.Errorf("do with failing writer = %d, %v; want 2, %v after 4 bytes read and 2 written", n, err, writeErr) + } + if _, err := (&streamCopy{dest: errWriter{}, src: strings.NewReader("abcd")}).do(); !errors.Is(err, io.ErrShortWrite) { + t.Errorf("do with short writer = %v, want io.ErrShortWrite", err) + } +} + +type errReader struct{ err error } + +func (r errReader) Read([]byte) (int, error) { return 0, r.err } + +// notifyWriter reports the size of each write to it. +type notifyWriter struct{ total chan int } + +func (w *notifyWriter) Write(p []byte) (int, error) { + w.total <- len(p) + return len(p), nil +} + +func TestStreamCopyDeliversWithoutWaitingForMoreData(t *testing.T) { + // A message that exactly fills a buffer must reach dst while src stays + // open and quiet, as when a client sends a request and awaits the reply. + for _, size := range []int{streamCopySmallBufSize, streamCopyBufSize} { + t.Run(fmt.Sprint(size), func(t *testing.T) { + pr, pw := io.Pipe() + dst := ¬ifyWriter{total: make(chan int, 16)} + done := make(chan error, 1) + go func() { + _, err := (&streamCopy{dest: dst, src: pr}).do() + done <- err + }() + + messages := []int{streamCopySmallBufSize} + if size == streamCopyBufSize { + messages = append(messages, streamCopyBufSize) + } + for _, n := range messages { + go func() { _, _ = pw.Write(make([]byte, n)) }() + if got := receiveWithin(t, dst.total, "message to be copied"); got != n { + t.Fatalf("copied %d bytes, want %d", got, n) + } + } + _ = pw.Close() + if err := receiveWithin(t, done, "do to return"); err != nil { + t.Fatalf("do: %v", err) + } + }) + } +} From 5d5dd7aea44e1f6e800ca62a6ce2658d21046ffd Mon Sep 17 00:00:00 2001 From: Taahir Ahmed Date: Wed, 21 Oct 2026 00:53:25 +0000 Subject: [PATCH 34/63] router authn/z part 4: Remove deprecated-insecure mode (#2977) We have now cut over all e2e tests so that they work in static-mtls mode. Remove deprecated-insecure mode from the atenet ingress router. Agent-gateway's config is still left supporting insecure ports, for followup from Solo. --- .github/workflows/pr-workflow.yaml | 5 - AGENTS.md | 2 +- cmd/ate-setup/internal/cmd/deploy.go | 2 +- cmd/ate-setup/internal/config/config.go | 14 - cmd/ate-setup/internal/config/config_test.go | 41 --- .../internal/config/coverage_test.go | 2 - cmd/ate-setup/internal/config/load.go | 13 - cmd/ate-setup/internal/config/options_test.go | 2 - cmd/ate-setup/internal/config/setting.go | 12 +- cmd/ate-setup/internal/steps/overlay.go | 8 - cmd/ate-setup/internal/steps/overlay_test.go | 82 +---- cmd/atenet/README.md | 4 +- cmd/atenet/internal/router/README.md | 62 +++- cmd/atenet/internal/router/cmd.go | 12 +- cmd/atenet/internal/router/config.go | 102 +----- cmd/atenet/internal/router/config_test.go | 133 ++----- cmd/atenet/internal/router/dashboard.html | 4 +- cmd/atenet/internal/router/dataplane.go | 33 +- .../router/ingressauthmanifest_test.go | 158 ++------ cmd/atenet/internal/router/status.go | 4 +- cmd/atenet/internal/router/status_test.go | 15 +- cmd/atenet/internal/router/xds.go | 201 +++------- cmd/atenet/internal/router/xds_test.go | 345 ++++++++---------- .../router/xds_upstream_connect_test.go | 17 +- docs/architecture.md | 9 +- docs/operator-install.md | 1 - docs/security.md | 2 +- internal/e2e/atenet_dataplane.go | 7 + internal/e2e/ingressauth.go | 34 -- internal/e2e/router_client.go | 68 ++-- internal/e2e/router_client_test.go | 4 +- .../suites/ingressauth/ingressauth_test.go | 64 +--- .../e2e/suites/networking/grpcingress_test.go | 33 +- .../ate-system-networkpolicies.yaml | 4 - manifests/ate-install/atenet-router.yaml | 24 +- .../agentgateway/kustomization.yaml | 33 ++ .../router-static-mtls/kustomization.yaml | 51 --- 37 files changed, 473 insertions(+), 1134 deletions(-) delete mode 100644 manifests/ate-install/components/router-static-mtls/kustomization.yaml diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 28154ee609..feb731ad7a 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -191,11 +191,6 @@ jobs: # --preview, so tests of a disabled gate skip themselves (see # internal/e2e/preview.go). ATE_PREVIEW: ${{ matrix.preview == 'on' && 'ALL' || '' }} - # Job-wide for the same reason, and E2E_INGRESS_AUTH_MODE has to match - # it so the suites dial the router's mTLS listeners. The agentgateway - # dataplane does not support static-mtls. - ATE_INGRESS_AUTH_MODE: ${{ matrix.dataplane == 'agentgateway' && 'deprecated-insecure' || 'static-mtls' }} - E2E_INGRESS_AUTH_MODE: ${{ matrix.dataplane == 'agentgateway' && 'deprecated-insecure' || 'static-mtls' }} # The suites' router client (internal/e2e/ingressauth.go). Only e2e # installs allowlist it, so the suites can reach actors through the # router's mTLS listeners. diff --git a/AGENTS.md b/AGENTS.md index 850c5accf6..826ff61712 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -155,4 +155,4 @@ Agent Substrate is built for zero-trust, multi-tenant execution of untrusted act - **Workload & Storage Isolation**: Actors run strictly inside `gVisor` (`cmd/ateworker-gvisor`) or `Cloud Hypervisor` microVM (`cmd/ateworker-microvm`) sandboxes; plain Linux containers are not supported. `atelet` scrubs actor directories across lifecycle transitions (`resetActorDirs`), and snapshots are stamped with `SandboxRuntime` metadata and transferred via `cmd/objectstore-plugin`. - **Component & Actor Identity (mTLS, JWTs, and JWKs)**: Inter-component communication uses mTLS backed by `cmd/podcertcontroller` (`podid`, `servicedns`, and `postgres` signers). Actor JWT signing (`internal/localjwtauthority`), verification (`internal/oidcjwt`), and key discovery (served via `cmd/ate-idp-server`) are built on the standard library's crypto packages. Extend those packages rather than adding `go-jose` or another JOSE library, so the code that decides whether a token is valid stays small and supports only the algorithms Substrate uses. - **Control-Plane Authentication & Authorization**: `ate-api-server` checks every Control RPC against an embedded OpenFGA model (`cmd/ateapi/internal/authz/model.fga`), always enforced. Every served unary RPC needs an entry in `defaultRPCPermissions` (`registry.go`): an RPC without one is denied, and a test fails the build. Access is granted per global scope and atespace through AccessPolicy (`kubectl ate access-policy`); the install's bootstrap owner is the shared `ate-client` ServiceAccount that `kubectl ate` uses. See `docs/authorization.md` for roles and known gaps. Callers are authenticated via `SPIFFE` mTLS or configured OIDC `JWTProvider`s (`cmd/ateapi/internal/apiauthn`). -- **Network Isolation, Ingress, & Egress**: `manifests/ate-install/ate-system-networkpolicies.yaml` enforces default-deny ingress across `ate-system`, and `cmd/atecontroller` reconciles per-`WorkerPool` `NetworkPolicy` objects. Actors run in private `/30` point-to-point veth network namespaces (`internal/ateworker/actornet`) with link-local blocking, policy-enforced DNS filtering (`internal/atunnel/dns`), and zero ambient credentials. `atenet-router --ingress-auth-mode=static-mtls` requires clients to present a `SPIFFE` certificate signed by `--ingress-client-ca-file` whose ID is in `--ingress-allowed-spiffe-ids` (`manifests/ate-install/atenet-router.yaml`). Outbound traffic traverses `atunnel` to `atenet-egress`, which enforces per-actor `EgressPolicy` and injects credentials out-of-band via `cmd/credential-provider/kubernetes-secrets` (`k8s-credential-provider`) or actor JWT minting. +- **Network Isolation, Ingress, & Egress**: `manifests/ate-install/ate-system-networkpolicies.yaml` enforces default-deny ingress across `ate-system`, and `cmd/atecontroller` reconciles per-`WorkerPool` `NetworkPolicy` objects. Actors run in private `/30` point-to-point veth network namespaces (`internal/ateworker/actornet`) with link-local blocking, policy-enforced DNS filtering (`internal/atunnel/dns`), and zero ambient credentials. Outbound traffic traverses `atunnel` to `atenet-egress`, which enforces per-actor `EgressPolicy` and injects credentials out-of-band via `cmd/credential-provider/kubernetes-secrets` (`k8s-credential-provider`) or actor JWT minting. diff --git a/cmd/ate-setup/internal/cmd/deploy.go b/cmd/ate-setup/internal/cmd/deploy.go index 45b35a3014..76d3f12ff0 100644 --- a/cmd/ate-setup/internal/cmd/deploy.go +++ b/cmd/ate-setup/internal/cmd/deploy.go @@ -60,7 +60,7 @@ ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING or the ATE_API_POSTGRES_CLOUDSQL_* select an external database. Shape the install with the global --atenet-dataplane, --cluster-size, ---cordon-control-plane, --fake-data-plane, and --ingress-auth-mode flags.`, +--cordon-control-plane, and --fake-data-plane flags.`, // Args runs before the root command loads the configuration, so only the // flag is readable here. Checking it keeps an unusable value from costing // a credential fetch, which is the common case; a value arriving from the diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index 1ee448d8f5..50df2171ec 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -50,11 +50,6 @@ const ( // the podcertificate controller's API rate limits to match. ClusterSizeSize0 = "size0" ClusterSizeSize10 = "size10" - - // Ingress router authentication modes; see atenet router's - // --ingress-auth-mode. - IngressAuthDeprecatedInsecure = "deprecated-insecure" - IngressAuthStaticMTLS = "static-mtls" ) // DefaultRolloutTimeout is the default wait timeout for workload rollouts. @@ -215,11 +210,6 @@ type Config struct { // (ATE_INSTALL_FAKE_DATA_PLANE), for control-plane benchmarks. FakeDataPlane bool - // IngressAuthMode is how the ingress router authenticates its clients - // (ATE_INGRESS_AUTH_MODE): IngressAuthDeprecatedInsecure or - // IngressAuthStaticMTLS. - IngressAuthMode string - // IngressAllowedSPIFFEIDs are the SPIFFE IDs the ingress router lets // reach actors over its TLS listeners (ATE_INGRESS_ALLOWED_SPIFFE_IDS). // See RequireIngressAllowedSPIFFEIDs. @@ -494,10 +484,6 @@ func (c *Config) ScriptEnv() []string { if c.FakeDataPlane { merged["ATE_INSTALL_FAKE_DATA_PLANE"] = "true" } - delete(merged, "ATE_INGRESS_AUTH_MODE") - if c.IngressAuthMode != "" && c.IngressAuthMode != IngressAuthDeprecatedInsecure { - merged["ATE_INGRESS_AUTH_MODE"] = c.IngressAuthMode - } delete(merged, "ATE_INGRESS_ALLOWED_SPIFFE_IDS") if len(c.IngressAllowedSPIFFEIDs) > 0 { merged["ATE_INGRESS_ALLOWED_SPIFFE_IDS"] = strings.Join(c.IngressAllowedSPIFFEIDs, ",") diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index fcfb93ccf4..d39d53ab7c 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -64,7 +64,6 @@ func loadEnv(t *testing.T) { "ATE_IMAGE_REPO", "ATE_IMAGE_TAG", "ATE_INGRESS_ALLOWED_SPIFFE_IDS", - "ATE_INGRESS_AUTH_MODE", "ATE_INSTALL_CLUSTER_SIZE", "ATE_INSTALL_CORDON_CONTROL_PLANE", "ATE_INSTALL_FAKE_DATA_PLANE", @@ -132,9 +131,6 @@ func TestLoadDefaults(t *testing.T) { if cfg.CordonControlPlane { t.Error("CordonControlPlane = true, want false") } - if cfg.IngressAuthMode != IngressAuthDeprecatedInsecure { - t.Errorf("IngressAuthMode = %q, want %q", cfg.IngressAuthMode, IngressAuthDeprecatedInsecure) - } } func TestLoadClusterSize(t *testing.T) { @@ -250,39 +246,6 @@ func TestLoadFakeDataPlane(t *testing.T) { }) } } -func TestLoadIngressAuthMode(t *testing.T) { - for _, tc := range []struct { - name string - opts Options - env string - want string - wantExport bool - }{ - {name: "flag static-mtls", opts: Options{IngressAuthMode: IngressAuthStaticMTLS}, want: IngressAuthStaticMTLS, wantExport: true}, - {name: "environment static-mtls", env: IngressAuthStaticMTLS, want: IngressAuthStaticMTLS, wantExport: true}, - {name: "environment deprecated-insecure", env: IngressAuthDeprecatedInsecure, want: IngressAuthDeprecatedInsecure}, - {name: "flag outranks environment", opts: Options{IngressAuthMode: IngressAuthDeprecatedInsecure}, env: IngressAuthStaticMTLS, want: IngressAuthDeprecatedInsecure}, - } { - t.Run(tc.name, func(t *testing.T) { - loadEnv(t) - if tc.env != "" { - t.Setenv("ATE_INGRESS_AUTH_MODE", tc.env) - } - cfg, err := Load(tc.opts) - if err != nil { - t.Fatalf("Load() error = %v", err) - } - if cfg.IngressAuthMode != tc.want { - t.Errorf("IngressAuthMode = %q, want %q", cfg.IngressAuthMode, tc.want) - } - got, exported := scriptEnvMap(t, cfg)["ATE_INGRESS_AUTH_MODE"] - if exported != tc.wantExport || (exported && got != tc.want) { - t.Errorf("ScriptEnv() ATE_INGRESS_AUTH_MODE = %q (exported %v), want %q (exported %v)", got, exported, tc.want, tc.wantExport) - - } - }) - } -} func TestLoadIngressAllowedSPIFFEIDs(t *testing.T) { const a, b = "spiffe://cluster.local/ns/a/sa/a", "spiffe://cluster.local/ns/b/sa/b" @@ -332,7 +295,6 @@ func TestRequireIngressAllowedSPIFFEIDs(t *testing.T) { wantErr bool }{ {name: "envoy without IDs", cfg: Config{Router: RouterEnvoy}, wantErr: true}, - {name: "envoy static-mtls without IDs", cfg: Config{Router: RouterEnvoy, IngressAuthMode: IngressAuthStaticMTLS}, wantErr: true}, {name: "envoy with IDs", cfg: Config{Router: RouterEnvoy, IngressAllowedSPIFFEIDs: []string{"spiffe://cluster.local/ns/a/sa/a"}}}, {name: "agentgateway without IDs", cfg: Config{Router: RouterAgentgateway}}, } { @@ -996,9 +958,6 @@ func TestLoadRejectsInvalidValues(t *testing.T) { {"kubernetes address without a port", Options{CredentialProvider: `{"name":"k8s.io","address":"secrets.ate-system.svc"}`}}, {"other provider without an address", Options{CredentialProvider: `{"name":"vault.example.com"}`}}, {"other provider address without a port", Options{CredentialProvider: `{"name":"vault.example.com","address":"vault.ate-system.svc"}`}}, - {"ingress auth mode", Options{IngressAuthMode: "mtls"}}, - {"old insecure ingress auth mode", Options{IngressAuthMode: "insecure"}}, - {"static-mtls agentgateway", Options{Router: RouterAgentgateway, IngressAuthMode: IngressAuthStaticMTLS}}, } { t.Run(tc.name, func(t *testing.T) { if _, err := Load(tc.opts); err == nil { diff --git a/cmd/ate-setup/internal/config/coverage_test.go b/cmd/ate-setup/internal/config/coverage_test.go index 23ad45729f..8f23e55ea5 100644 --- a/cmd/ate-setup/internal/config/coverage_test.go +++ b/cmd/ate-setup/internal/config/coverage_test.go @@ -90,8 +90,6 @@ func valueFor(key string) string { return RouterEnvoy case "clusterSize": return ClusterSizeSize0 - case "ingressAuthMode": - return IngressAuthStaticMTLS case "ingressAllowedSPIFFEIDs": return "spiffe://cluster.local/ns/a/sa/b" case "ateapi.postgres.cloudsql.ipType": diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index 03f2c7c8c4..1b31c4fac1 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -274,7 +274,6 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro ClusterSize: r.String("clusterSize"), CordonControlPlane: r.Bool("cordonControlPlane"), FakeDataPlane: r.Bool("fakeDataPlane.enabled"), - IngressAuthMode: r.String("ingressAuthMode"), IngressAllowedSPIFFEIDs: splitList(r.String("ingressAllowedSPIFFEIDs")), AdditionalEgressExtprocService: r.String("atenet.egress.additionalExtprocService"), CredentialProviderJSON: r.String("atenet.egress.credentialProvider"), @@ -411,18 +410,6 @@ func validateResolved(cfg *Config, r *Resolved) error { return &InvalidError{Value: size, Want: ClusterSizeSize0 + " or " + ClusterSizeSize10} } - authMode, _ := r.Value("ingressAuthMode") - switch cfg.IngressAuthMode { - case IngressAuthDeprecatedInsecure: - case IngressAuthStaticMTLS: - if cfg.Router != RouterEnvoy { - return &ConflictError{A: authMode, B: dataplane, - Why: IngressAuthStaticMTLS + " requires dataplane " + RouterEnvoy} - } - default: - return &InvalidError{Value: authMode, Want: IngressAuthDeprecatedInsecure + " or " + IngressAuthStaticMTLS} - } - // The router rejects a malformed ID at startup; catch it before the // rollout does. for _, id := range cfg.IngressAllowedSPIFFEIDs { diff --git a/cmd/ate-setup/internal/config/options_test.go b/cmd/ate-setup/internal/config/options_test.go index 4aac4df5ea..3e8fba4ffc 100644 --- a/cmd/ate-setup/internal/config/options_test.go +++ b/cmd/ate-setup/internal/config/options_test.go @@ -40,7 +40,6 @@ type Options struct { ClusterSize string CordonControlPlane bool FakeDataPlane bool - IngressAuthMode string AdditionalEgressExtprocService string CredentialProvider string OtlpEndpoint string @@ -89,7 +88,6 @@ func optionsFlagSet(opts Options) (*pflag.FlagSet, error) { {"atenet-dataplane", opts.Router}, {"rollout-timeout", opts.RolloutTimeout}, {"cluster-size", opts.ClusterSize}, - {"ingress-auth-mode", opts.IngressAuthMode}, {"experimental-additional-egress-extproc-service", opts.AdditionalEgressExtprocService}, {"credential-provider", opts.CredentialProvider}, {"otlp-endpoint", opts.OtlpEndpoint}, diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index d94a95a44f..f3e0117c7a 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -295,22 +295,14 @@ var Registry = []Setting{ "atelet, and fake-workersync in place of ate-controller. Actors do not exist; never " + "use it on a cluster that serves real actors", }, - { - Key: "ingressAuthMode", Env: "ATE_INGRESS_AUTH_MODE", Flag: "ingress-auth-mode", Kind: KindString, - Default: IngressAuthDeprecatedInsecure, - Usage: "How the ingress router authenticates clients: deprecated-insecure or static-mtls. Both check a " + - "client certificate on the router's TLS ports against the podidentity CA and --ingress-allowed-spiffe-ids; " + - "deprecated-insecure also lets through TLS clients without one and serves plaintext ports, while " + - "static-mtls requires the certificate and --atenet-dataplane=envoy", - }, { // Comma-separated, as the router's own flag is. The registry has no // list kind, so the value is carried as written and split where it is // used. Key: "ingressAllowedSPIFFEIDs", Env: "ATE_INGRESS_ALLOWED_SPIFFE_IDS", Flag: "ingress-allowed-spiffe-ids", Kind: KindString, - Usage: "SPIFFE IDs (exact match, comma-separated) of the clients the ingress router lets reach actors over " + - "its TLS ports; any listed client may reach any actor. Required to deploy the router with " + + Usage: "SPIFFE IDs (exact match, comma-separated) of the clients the ingress router lets reach actors; " + + "any listed client may reach any actor. Required to deploy the router with " + "--atenet-dataplane=envoy. The podidentity signer issues spiffe://cluster.local/ns//sa/", }, { diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index 396cd4d89c..8a2d6d3e36 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -48,11 +48,6 @@ const cordonControlPlaneComponent = installDir + "/components/cordon-control-pla // over the atelet and ate-controller applies under --fake-data-plane. const fakeDataPlaneComponent = installDir + "/components/fake-data-plane" -// routerStaticMTLSComponent is the kustomize component that switches the -// ingress router to --ingress-auth-mode=static-mtls, layered over every -// control plane apply under --ingress-auth-mode=static-mtls. -const routerStaticMTLSComponent = installDir + "/components/router-static-mtls" - // components lists the kustomize components every control plane apply is // composed with. Each patch targets its workloads by name, and kustomize leaves // a stream alone when nothing in it matches, so composing a manifest that @@ -62,9 +57,6 @@ func (e *Env) components() []string { if e.Cfg.FakeDataPlane { components = append(components, e.Cfg.Path(fakeDataPlaneComponent)) } - if e.Cfg.IngressAuthMode == config.IngressAuthStaticMTLS { - components = append(components, e.Cfg.Path(routerStaticMTLSComponent)) - } if e.Cfg.CordonControlPlane { components = append(components, e.Cfg.Path(cordonControlPlaneComponent)) } diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index 619022b42e..dd5202e5c9 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -767,79 +767,8 @@ func TestRenderFakeDataPlane(t *testing.T) { } } -// Under --ingress-auth-mode=static-mtls both paths that deliver the router -// -- the system bundle and the lone router redeploy -- have to carry the -// static-mtls flags, alone or alongside the cordon component. Whether the -// router accepts those flags is pinned on the router side, by -// TestRouterIngressAuthManifestsAreValid. -func TestRenderRouterStaticMTLS(t *testing.T) { - root := repoRoot(t) - for _, tc := range []struct { - name string - cfg config.Config - path func(e *Env) string - cordon bool - }{ - { - name: "base bundle", - cfg: config.Config{Router: config.RouterEnvoy}, - path: func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) }, - }, - { - name: "kind bundle", - cfg: config.Config{Router: config.RouterEnvoy, Kind: true}, - path: func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) }, - }, - { - name: "router file", - cfg: config.Config{Router: config.RouterEnvoy}, - path: func(e *Env) string { return e.Cfg.Manifest("atenet-router.yaml") }, - }, - { - name: "kind bundle with cordon", - cfg: config.Config{Router: config.RouterEnvoy, Kind: true}, - path: func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) }, - cordon: true, - }, - } { - t.Run(tc.name, func(t *testing.T) { - cfg := tc.cfg - cfg.Root = root - cfg.IngressAuthMode = config.IngressAuthStaticMTLS - cfg.CordonControlPlane = tc.cordon - e := &Env{Cfg: &cfg} - - rendered, err := e.render(tc.path(e)) - if err != nil { - t.Fatalf("render: %v", err) - } - args := routerArgs(t, rendered) - if !slices.Contains(args, "--ingress-auth-mode=static-mtls") { - t.Errorf("atenet-router args %v do not select static-mtls", args) - } - if tc.cordon { - if pinned, _ := pinnedWorkloads(t, rendered); pinned["atenet-router"] != "ate-control-plane" { - t.Errorf("composing both components dropped the cordon pinning (pinned: %v)", pinned) - } - } - t.Run("deprecated-insecure leaves the router alone", func(t *testing.T) { - e := &Env{Cfg: &config.Config{Root: root, Router: config.RouterEnvoy, IngressAuthMode: config.IngressAuthDeprecatedInsecure}} - rendered, err := e.render(e.Cfg.Manifest("atenet-router.yaml")) - if err != nil { - t.Fatalf("render: %v", err) - } - for _, arg := range routerArgs(t, rendered) { - if strings.HasPrefix(arg, "--ingress-auth-mode") { - t.Errorf("deprecated-insecure render sets %s", arg) - } - } - }) - }) - } -} - // Both paths that deliver the Envoy router pass it the install's allowlist, -// whatever the overlay and auth mode, and refuse to render without one. +// whatever the overlay, and refuse to render without one. func TestRenderRouterAllowedSPIFFEIDs(t *testing.T) { const a, b = "spiffe://cluster.local/ns/a/sa/a", "spiffe://cluster.local/ns/b/sa/b" src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"} @@ -857,8 +786,8 @@ func TestRenderRouterAllowedSPIFFEIDs(t *testing.T) { render: func(e *Env) ([]byte, error) { return e.renderSystemManifests(t.Context()) }, }, { - name: "kind bundle static-mtls", - cfg: config.Config{Kind: true, IngressAuthMode: config.IngressAuthStaticMTLS}, + name: "kind bundle", + cfg: config.Config{Kind: true}, render: func(e *Env) ([]byte, error) { return e.renderSystemManifests(t.Context()) }, }, { @@ -866,11 +795,6 @@ func TestRenderRouterAllowedSPIFFEIDs(t *testing.T) { cfg: config.Config{}, render: func(e *Env) ([]byte, error) { return e.renderAtenetRouterManifest(t.Context()) }, }, - { - name: "router file static-mtls", - cfg: config.Config{IngressAuthMode: config.IngressAuthStaticMTLS}, - render: func(e *Env) ([]byte, error) { return e.renderAtenetRouterManifest(t.Context()) }, - }, } { t.Run(tc.name, func(t *testing.T) { cfg := tc.cfg diff --git a/cmd/atenet/README.md b/cmd/atenet/README.md index d72b0f6a71..3abe9e1a0e 100644 --- a/cmd/atenet/README.md +++ b/cmd/atenet/README.md @@ -18,8 +18,8 @@ likely be split in the future for better scalability.) * Envoy * atenet router * Service will expose: - * Envoy port 80 and 443 for ordinary ingress traffic, plus a dedicated - CONNECT port (8081, or 8444 for TLS) for arbitrary-port ingress: a client + * Envoy port 443 for ordinary ingress traffic, plus a dedicated + CONNECT port (8444) for arbitrary-port ingress: a client reaches a port on the actor other than its default (80) with an HTTP `CONNECT` request naming the port in the authority. Only HTTP(S) traffic over the tunnel is supported today, not raw TCP. diff --git a/cmd/atenet/internal/router/README.md b/cmd/atenet/internal/router/README.md index 9200ad33f6..2d7076bfa7 100644 --- a/cmd/atenet/internal/router/README.md +++ b/cmd/atenet/internal/router/README.md @@ -34,13 +34,13 @@ Router has several responsibilities: [egress legs](#egress-legs). * Serves arbitrary-port ingress: a client reaches a port on the actor other than its default (80) by sending an HTTP CONNECT to - `:` on `--port-connect`/`--port-connect-tls`, rather than + `:` on `--port-connect-tls`, rather than naming the port some other way. Envoy terminates the CONNECT and reinjects the tunneled bytes into an internal listener that runs the same ext_proc path as ordinary traffic, so each request inside a long-lived tunnel still resumes the actor and re-routes independently if it moves workers. Only HTTP(S) traffic over the tunnel is supported today -- see `xds.go`'s - `connect_terminate`/`main_internal` listeners and + `connect_terminate_tls`/`main_internal` listeners and `ingress.Handler.HandleRequestHeaders`. ## packages @@ -204,12 +204,9 @@ its own static configuration because ingress and egress scale independently. ## Ingress client authentication and authorization -The ingress router is migrating to require clients to present a SPIFFE certificate. -* With `--ingress-auth-mode=static-mtls`, the ingress router no longer offers any - plaintext listeners, and all clients *must* present a SPIFFE certificate. -* With `--ingress-auth-mode=deprecated-insecure`, the ingress router offers both - plaintext and secure listeners. If a client presents a SPIFFE certificate, it - must be valid. +The Envoy ingress router serves only TLS listeners, and every client must +present a SPIFFE certificate. The agentgateway dataplane does not authenticate +clients yet. Configure the CA that is used to validate client certificates with the `--ingress-client-ca-file` flag, which points at a PEM file containing root @@ -221,6 +218,55 @@ to contact any actor. Richer authorization policies may be added post-GA. Use the `--ingress-allowed-spiffe-ids` flag to set the allowlist (comma-separated list). +### Common setups + +The default install sets `--ingress-client-ca-file` to the trust bundle of +substrate's pod identity signer, `podidentity.podcert.ate.dev/identity`. That +signer issues certificates with the SPIFFE ID +`spiffe://cluster.local/ns//sa/`. + +The router's serving certificate comes from the +`servicedns.podcert.ate.dev/identity` signer. Clients verify it against that +signer's ClusterTrustBundle, with server name +`atenet-router.ate-system.svc`. + +**Custom CA.** An installer can point `--ingress-client-ca-file` at a CA of +their own choosing. Clients then get certificates from that CA however the +installer provides them, and the allowlist holds the SPIFFE IDs it issues. + +**Client in the cluster, pod identity CA.** Mount a client certificate with a +`podCertificate` projected volume, and the router's serving CA with a +`clusterTrustBundle` projected volume: + +```yaml +volumes: +- name: router-client + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: router-ca.pem +``` + +Add the pod's SPIFFE ID to `--ingress-allowed-spiffe-ids`. + +**Client outside the cluster, pod identity CA.** A demo on your machine can +follow the e2e tests in `internal/e2e/ingressauth.go`. It creates a +PodCertificateRequest for the podidentity signer that names a pod, node and +ServiceAccount that don't exist. kube-apiserver checks those only for requests +from a kubelet, and the signer issues the SPIFFE ID of the named namespace and +ServiceAccount. The client keeps the private key, reads the chain from the +request's status, and reads the servicedns ClusterTrustBundle to verify the +router. Creating PodCertificateRequests needs RBAC for that resource in the +namespace. + ## status page The unauthenticated `/statusz` dashboard binds to `127.0.0.1:4040` by default diff --git a/cmd/atenet/internal/router/cmd.go b/cmd/atenet/internal/router/cmd.go index bf52fe9144..17620d57fe 100644 --- a/cmd/atenet/internal/router/cmd.go +++ b/cmd/atenet/internal/router/cmd.go @@ -60,8 +60,6 @@ func bindRouterFlags(fs *pflag.FlagSet, cfg *routerConfig) { fs.StringVar(&cfg.RouterServiceName, "router-service-name", installdefaults.RouterServiceName, "Service name of this atenet-router in the operations namespace. Override when the deployment renames the Service.") fs.StringVar(&cfg.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig configuration file") fs.StringVar(&cfg.AteapiAddr, "ateapi-address", "k8s:///api.ate-system.svc:443", "gRPC dial target for the cluster ateapi Control instance.") - fs.IntVar(&cfg.HttpPort, "port-http", 8080, "TCP port for workload traffic entering through the Envoy Router") - fs.IntVar(&cfg.ConnectPlainTextPort, "port-connect", 8081, "TCP port for CONNECT-tunneled traffic entering through the router dataplane") fs.IntVar(&cfg.ConnectTLSPort, "port-connect-tls", 8444, "TCP port for CONNECT-tunneled traffic entering through the router dataplane over TLS. --port-https also defaults to 8443, and both listeners are commonly enabled at once, so --port-connect-tls defaults to a different port (8444) rather than colliding with it") fs.IntVar(&cfg.XdsPort, "port-xds", 18000, "TCP port listening for the xDS dynamic Envoy connections") fs.IntVar(&cfg.ExtprocPort, "port-extproc", 50051, "Listen port for the External Processing (ext_proc) server the dataplane calls") @@ -73,14 +71,8 @@ func bindRouterFlags(fs *pflag.FlagSet, cfg *routerConfig) { fs.StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file.") // Flags for client authentication and authorization in ingress mode. - fs.StringVar(&cfg.IngressAuth.ClientCAFile, "ingress-client-ca-file", "", "PEM bundle of the CAs allowed to sign ingress client certificates. Envoy reads the file, so it must be mounted at this path in the Envoy container as well. Required whenever a TLS ingress listener is enabled") - fs.StringSliceVar(&cfg.IngressAuth.AllowedSPIFFEIDs, "ingress-allowed-spiffe-ids", nil, "SPIFFE IDs (exact match) allowed to connect to actors through the ingress dataplane. Comma-separated or repeated. Required whenever a TLS ingress listener is enabled") - fs.StringVar( - (*string)(&cfg.IngressAuth.Mode), - "ingress-auth-mode", - string(IngressAuthDeprecatedInsecure), - "Ingress authentication and authorization mode. `deprecated-insecure` (both http and https served, client certificates optional on https) or `static-mtls` (only https served, client certificates required).", - ) + fs.StringVar(&cfg.IngressAuth.ClientCAFile, "ingress-client-ca-file", "", "PEM bundle of the CAs allowed to sign ingress client certificates. Envoy reads the file, so it must be mounted at this path in the Envoy container as well. Required for the Envoy ingress dataplane") + fs.StringSliceVar(&cfg.IngressAuth.AllowedSPIFFEIDs, "ingress-allowed-spiffe-ids", nil, "SPIFFE IDs (exact match) allowed to connect to actors through the ingress dataplane. Comma-separated or repeated. Required for the Envoy ingress dataplane") fs.StringVar(&cfg.UpstreamCredentialBundlePath, "upstream-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "PEM credential bundle (cert+key) the router presents as the client cert when dialing the actor's atunnel ingress server over mTLS. Empty disables upstream mTLS (legacy plaintext pod-IP:80) and needs --upstream-connect=false.") fs.StringVar(&cfg.UpstreamTrustBundlePath, "upstream-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "PEM trust bundle used to validate the actor's atunnel ingress server certificate.") diff --git a/cmd/atenet/internal/router/config.go b/cmd/atenet/internal/router/config.go index 84c43f4c6f..5345caef7d 100644 --- a/cmd/atenet/internal/router/config.go +++ b/cmd/atenet/internal/router/config.go @@ -63,51 +63,23 @@ func (m Mode) ServesIngress() bool { return m == ModeIngress } // ServesEgress reports whether this mode answers egress CONNECTs. func (m Mode) ServesEgress() bool { return m == ModeEgress } -// IngressAuthMode selects how the ingress dataplane authenticates and -// authorizes the clients that connect to it. -type IngressAuthMode string - -const ( - // IngressAuthDeprecatedInsecure serves the plaintext ingress listeners, - // which accept every client and let any of them reach any actor, - // alongside the TLS listeners, which authenticate clients exactly as in - // IngressAuthStaticMTLS. It exists so clients can move to mTLS one at a - // time, and will be removed once they have. - IngressAuthDeprecatedInsecure IngressAuthMode = "deprecated-insecure" - // IngressAuthStaticMTLS requires every client to present a certificate - // that chains to the configured client CA bundle and carries one of the - // allowed SPIFFE IDs as a URI SAN. Envoy enforces both during the TLS - // handshake, so only the TLS listeners may be enabled. - IngressAuthStaticMTLS IngressAuthMode = "static-mtls" -) - // ingressAuthConfig holds the client authentication settings for the ingress -// dataplane's listeners. +// dataplane's listeners. Every client must present a certificate that chains +// to ClientCAFile and carries one of AllowedSPIFFEIDs as a URI SAN. Envoy +// enforces both during the TLS handshake. type ingressAuthConfig struct { - // Mode is the authentication mode. Empty means - // IngressAuthDeprecatedInsecure. - Mode IngressAuthMode // ClientCAFile is the PEM bundle of CAs allowed to sign client // certificates. Envoy reads it, so it must be mounted in the Envoy - // container at the same path. Required whenever a TLS listener is - // enabled. + // container at the same path. ClientCAFile string // AllowedSPIFFEIDs is the exact set of SPIFFE IDs allowed to connect to - // actors through the TLS listeners. Required whenever a TLS listener is - // enabled. + // actors. // // Entries are complete SPIFFE IDs, like // `spiffe://mytrustdomain.com/foo/bar/baz`. AllowedSPIFFEIDs []string } -func (c ingressAuthConfig) mode() IngressAuthMode { - if c.Mode == "" { - return IngressAuthDeprecatedInsecure - } - return c.Mode -} - // authConfig holds the router's mTLS settings for dialing ateapi. type authConfig struct { AteapiCAFile string @@ -150,7 +122,6 @@ type routerConfig struct { RouterServiceName string Kubeconfig string AteapiAddr string - HttpPort int XdsPort int ExtprocPort int ExtprocAddr string @@ -158,12 +129,10 @@ type routerConfig struct { StatusAddr string HealthInterval time.Duration HttpsPort int - // ConnectPlainTextPort and ConnectTLSPort are the plaintext and TLS - // listener ports for CONNECT-tunneled traffic. Non-positive disables the - // corresponding listener. - ConnectPlainTextPort int - ConnectTLSPort int - EnvoyCertPath string + // ConnectTLSPort is the listener port for CONNECT-tunneled traffic. + // Non-positive disables the listener. + ConnectTLSPort int + EnvoyCertPath string // UpstreamCredentialBundlePath is the router's podidentity credential bundle // (cert+key) presented as the client cert when dialing the actor's atunnel @@ -340,60 +309,25 @@ func (c routerConfig) validate() error { return nil } -// tlsIngressEnabled reports whether the Envoy dataplane builds at least one -// TLS ingress listener. -func (c routerConfig) tlsIngressEnabled() bool { - return c.EnvoyCertPath != "" && (c.HttpsPort > 0 || c.ConnectTLSPort > 0) -} - -// validateIngressAuth rejects a configuration that would leave a TLS listener -// without client authentication, a static-mtls configuration that would leave -// any path to an actor unauthenticated, and one the dataplane cannot enforce. +// validateIngressAuth rejects an Envoy ingress configuration with no TLS +// listener, or one whose clients cannot be authenticated. Agentgateway's +// listeners come from its static configuration, so the router has none to +// check. func (c routerConfig) validateIngressAuth() error { - mode := c.IngressAuth.mode() - switch mode { - case IngressAuthDeprecatedInsecure, IngressAuthStaticMTLS: - default: - return fmt.Errorf("--ingress-auth-mode must be %q or %q, got %q", IngressAuthDeprecatedInsecure, IngressAuthStaticMTLS, c.IngressAuth.Mode) - } - // The egress gateway never builds the ingress listeners. - if !c.Mode.ServesIngress() { + if !c.Mode.ServesIngress() || c.atenetRouter() != atenetRouterEnvoy { return nil } - if mode == IngressAuthDeprecatedInsecure { - // Agentgateway's listeners come from its static configuration, so - // the router has none to authenticate. - if c.atenetRouter() != atenetRouterEnvoy || !c.tlsIngressEnabled() { - return nil - } - return c.validateIngressClientAuth() - } - if c.atenetRouter() != atenetRouterEnvoy { - return fmt.Errorf("--ingress-auth-mode=%s requires --atenet-dataplane=%s: the %s dataplane is configured statically, not by the router", IngressAuthStaticMTLS, atenetRouterEnvoy, c.atenetRouter()) - } - if c.HttpPort > 0 { - return fmt.Errorf("--ingress-auth-mode=%s requires --port-http=0: the plaintext listener cannot authenticate clients", IngressAuthStaticMTLS) - } - if c.ConnectPlainTextPort > 0 { - return fmt.Errorf("--ingress-auth-mode=%s requires --port-connect=0: the plaintext listener cannot authenticate clients", IngressAuthStaticMTLS) - } if c.EnvoyCertPath == "" { - return fmt.Errorf("--ingress-auth-mode=%s requires --envoy-cert-path: without a serving certificate there is no TLS listener", IngressAuthStaticMTLS) + return fmt.Errorf("--envoy-cert-path is required: without a serving certificate there is no TLS listener") } if c.HttpsPort <= 0 && c.ConnectTLSPort <= 0 { - return fmt.Errorf("--ingress-auth-mode=%s requires --port-https or --port-connect-tls: there is no TLS listener", IngressAuthStaticMTLS) + return fmt.Errorf("--port-https or --port-connect-tls is required: there is no TLS listener") } - return c.validateIngressClientAuth() -} - -// validateIngressClientAuth checks the settings the TLS listeners authenticate -// clients with. -func (c routerConfig) validateIngressClientAuth() error { if c.IngressAuth.ClientCAFile == "" { - return fmt.Errorf("TLS ingress listeners (--port-https, --port-connect-tls) require --ingress-client-ca-file") + return fmt.Errorf("--ingress-client-ca-file is required") } if len(c.IngressAuth.AllowedSPIFFEIDs) == 0 { - return fmt.Errorf("TLS ingress listeners (--port-https, --port-connect-tls) require at least one --ingress-allowed-spiffe-ids entry") + return fmt.Errorf("at least one --ingress-allowed-spiffe-ids entry is required") } for _, id := range c.IngressAuth.AllowedSPIFFEIDs { if _, err := spiffeid.FromString(id); err != nil { diff --git a/cmd/atenet/internal/router/config_test.go b/cmd/atenet/internal/router/config_test.go index ece06c928b..d047cee62a 100644 --- a/cmd/atenet/internal/router/config_test.go +++ b/cmd/atenet/internal/router/config_test.go @@ -126,7 +126,13 @@ func TestRouterConfigValidate(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - err := tc.cfg.validate() + // TestRouterConfigValidateIngressAuth covers the ingress + // listener settings, so every row here gets valid ones. + cfg := tc.cfg + cfg.HttpsPort = 8443 + cfg.EnvoyCertPath = "/run/servicedns.podcert.ate.dev/credential-bundle.pem" + cfg.IngressAuth = validIngressAuthConfig() + err := cfg.validate() if tc.wantErr == "" { if err != nil { t.Fatalf("expected valid, got %v", err) @@ -140,20 +146,23 @@ func TestRouterConfigValidate(t *testing.T) { } } -// staticMTLSConfig returns a valid static-mtls ingress configuration for +func validIngressAuthConfig() ingressAuthConfig { + return ingressAuthConfig{ + ClientCAFile: "/run/podidentity.podcert.ate.dev/trust-bundle.pem", + AllowedSPIFFEIDs: []string{"spiffe://cluster.local/ns/demo/sa/client"}, + } +} + +// validIngressConfig returns a valid ingress configuration for // TestRouterConfigValidateIngressAuth to break one field at a time. -func staticMTLSConfig() routerConfig { +func validIngressConfig() routerConfig { return routerConfig{ Mode: ModeIngress, ParkedRequest: ingress.ParkedRequestConfig{Max: ingress.DefaultParkedRequestMax}, HttpsPort: 8443, ConnectTLSPort: 8444, EnvoyCertPath: "/run/servicedns.podcert.ate.dev/credential-bundle.pem", - IngressAuth: ingressAuthConfig{ - Mode: IngressAuthStaticMTLS, - ClientCAFile: "/run/podidentity.podcert.ate.dev/trust-bundle.pem", - AllowedSPIFFEIDs: []string{"spiffe://cluster.local/ns/demo/sa/client"}, - }, + IngressAuth: validIngressAuthConfig(), } } @@ -164,121 +173,37 @@ func TestRouterConfigValidateIngressAuth(t *testing.T) { wantErr string // substring; empty means valid }{ { - name: "static-mtls with only TLS listeners accepted", + name: "both TLS listeners accepted", mutate: func(*routerConfig) {}, }, { - name: "static-mtls with only the HTTPS listener accepted", + name: "only the HTTPS listener accepted", mutate: func(c *routerConfig) { c.ConnectTLSPort = 0 }, }, { - name: "static-mtls with only the CONNECT-TLS listener accepted", + name: "only the CONNECT-TLS listener accepted", mutate: func(c *routerConfig) { c.HttpsPort = 0 }, }, - { - name: "unknown mode rejected", - mutate: func(c *routerConfig) { c.IngressAuth.Mode = "mtls" }, - wantErr: "--ingress-auth-mode must be", - }, - { - name: "old insecure mode name rejected", - mutate: func(c *routerConfig) { c.IngressAuth.Mode = "insecure" }, - wantErr: "--ingress-auth-mode must be", - }, { name: "egress-only instance ignores the ingress settings", mutate: func(c *routerConfig) { c.Mode = ModeEgress - c.HttpPort = 8080 - c.IngressAuth.ClientCAFile = "" - c.IngressAuth.AllowedSPIFFEIDs = nil - }, - }, - { - name: "deprecated-insecure accepts plaintext listeners alongside mTLS ones", - mutate: func(c *routerConfig) { - c.IngressAuth.Mode = IngressAuthDeprecatedInsecure - c.HttpPort = 8080 - c.ConnectPlainTextPort = 8081 - }, - }, - { - name: "empty mode means deprecated-insecure", - mutate: func(c *routerConfig) { - c.IngressAuth.Mode = "" - c.HttpPort = 8080 - }, - }, - { - name: "deprecated-insecure with only plaintext listeners needs no client auth", - mutate: func(c *routerConfig) { - c.IngressAuth = ingressAuthConfig{Mode: IngressAuthDeprecatedInsecure, AllowedSPIFFEIDs: []string{"not a spiffe id"}} - c.HttpPort = 8080 - c.HttpsPort = 0 - c.ConnectTLSPort = 0 - }, - }, - { - name: "deprecated-insecure without a serving cert needs no client auth", - mutate: func(c *routerConfig) { - c.IngressAuth = ingressAuthConfig{Mode: IngressAuthDeprecatedInsecure} - c.HttpPort = 8080 c.EnvoyCertPath = "" + c.IngressAuth = ingressAuthConfig{} }, }, { - name: "deprecated-insecure on agentgateway needs no client auth", + name: "agentgateway ignores the ingress settings", mutate: func(c *routerConfig) { - c.IngressAuth = ingressAuthConfig{Mode: IngressAuthDeprecatedInsecure} c.AtenetRouter = string(atenetRouterAgentgateway) - c.HttpPort = 8080 - }, - }, - { - name: "deprecated-insecure TLS listener without client CA rejected", - mutate: func(c *routerConfig) { - c.IngressAuth.Mode = IngressAuthDeprecatedInsecure - c.IngressAuth.ClientCAFile = "" - c.HttpPort = 8080 - }, - wantErr: "require --ingress-client-ca-file", - }, - { - name: "deprecated-insecure TLS listener without allowlist rejected", - mutate: func(c *routerConfig) { - c.IngressAuth.Mode = IngressAuthDeprecatedInsecure - c.IngressAuth.AllowedSPIFFEIDs = nil - c.HttpPort = 8080 - }, - wantErr: "require at least one --ingress-allowed-spiffe-ids", - }, - { - name: "deprecated-insecure non-SPIFFE ID rejected", - mutate: func(c *routerConfig) { - c.IngressAuth.Mode = IngressAuthDeprecatedInsecure - c.IngressAuth.AllowedSPIFFEIDs = []string{"https://cluster.local/ns/demo/sa/client"} + c.EnvoyCertPath = "" + c.IngressAuth = ingressAuthConfig{} }, - wantErr: "is not a valid SPIFFE ID", - }, - { - name: "agentgateway rejected", - mutate: func(c *routerConfig) { c.AtenetRouter = string(atenetRouterAgentgateway) }, - wantErr: "requires --atenet-dataplane=envoy", - }, - { - name: "plaintext HTTP listener rejected", - mutate: func(c *routerConfig) { c.HttpPort = 8080 }, - wantErr: "requires --port-http=0", - }, - { - name: "plaintext CONNECT listener rejected", - mutate: func(c *routerConfig) { c.ConnectPlainTextPort = 8081 }, - wantErr: "requires --port-connect=0", }, { name: "missing serving cert rejected", mutate: func(c *routerConfig) { c.EnvoyCertPath = "" }, - wantErr: "requires --envoy-cert-path", + wantErr: "--envoy-cert-path is required", }, { name: "no TLS listener rejected", @@ -286,17 +211,17 @@ func TestRouterConfigValidateIngressAuth(t *testing.T) { c.HttpsPort = 0 c.ConnectTLSPort = 0 }, - wantErr: "requires --port-https or --port-connect-tls", + wantErr: "--port-https or --port-connect-tls is required", }, { name: "missing client CA rejected", mutate: func(c *routerConfig) { c.IngressAuth.ClientCAFile = "" }, - wantErr: "require --ingress-client-ca-file", + wantErr: "--ingress-client-ca-file is required", }, { name: "empty allowlist rejected", mutate: func(c *routerConfig) { c.IngressAuth.AllowedSPIFFEIDs = nil }, - wantErr: "require at least one --ingress-allowed-spiffe-ids", + wantErr: "at least one --ingress-allowed-spiffe-ids entry is required", }, { name: "non-SPIFFE ID rejected", @@ -315,7 +240,7 @@ func TestRouterConfigValidateIngressAuth(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - cfg := staticMTLSConfig() + cfg := validIngressConfig() tc.mutate(&cfg) err := cfg.validate() if tc.wantErr == "" { diff --git a/cmd/atenet/internal/router/dashboard.html b/cmd/atenet/internal/router/dashboard.html index 856f2224d7..9b4e38badc 100644 --- a/cmd/atenet/internal/router/dashboard.html +++ b/cmd/atenet/internal/router/dashboard.html @@ -257,8 +257,8 @@

atenet Router Status

Component Network Allocation