From a9d3ee063cb8bc1f4d7d588c822249080662e24d Mon Sep 17 00:00:00 2001 From: "Ryan Keith (from Dev Box)" Date: Fri, 17 Jul 2026 13:23:21 -0700 Subject: [PATCH] GCS: periodically update cgroup limits Signed-off-by: Ryan Keith (from Dev Box) --- cmd/gcs/main.go | 100 +++++++++++++++++++++++++++++++++++++++-- cmd/gcs/main_test.go | 105 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 201 insertions(+), 4 deletions(-) create mode 100644 cmd/gcs/main_test.go diff --git a/cmd/gcs/main.go b/cmd/gcs/main.go index fd9575ec3b..64a35f16b5 100644 --- a/cmd/gcs/main.go +++ b/cmd/gcs/main.go @@ -113,6 +113,92 @@ func readMemoryEvents(startTime time.Time, efdFile *os.File, cgName string, thre } } +type cgroupUpdater interface { + Update(resources *oci.LinuxResources) error +} + +// Sanity checks the memory limit value +func calculateContainersMemoryLimit(totalMemoryBytes, rootMemReserveBytes uint64) (int64, error) { + if totalMemoryBytes <= rootMemReserveBytes { + return 0, errors.Errorf("total memory %d must be greater than root memory reserve %d", totalMemoryBytes, rootMemReserveBytes) + } + + limit := totalMemoryBytes - rootMemReserveBytes + const maxInt64 = uint64(1<<63 - 1) + if limit > maxInt64 { + return 0, errors.Errorf("containers cgroup memory limit %d exceeds maximum supported value %d", limit, maxInt64) + } + return int64(limit), nil +} + +// Returns the total system memory in bytes. +func currentTotalMemoryBytes() (uint64, error) { + sinfo := syscall.Sysinfo_t{} + if err := syscall.Sysinfo(&sinfo); err != nil { + return 0, errors.Wrap(err, "failed to get sys info") + } + + totalMemoryBytes := uint64(sinfo.Totalram) + unit := uint64(sinfo.Unit) + + // If the unit is non-zero, multiply the total memory by the unit to get the actual memory in bytes. + if unit != 0 { + const maxUint64 = ^uint64(0) + if totalMemoryBytes > maxUint64/unit { + return 0, errors.Errorf("total memory %d with unit %d exceeds maximum supported value", totalMemoryBytes, unit) + } + totalMemoryBytes *= unit + } + return totalMemoryBytes, nil +} + +// Set the cgroup memory limits for the containers and virtual-pods cgroups (pass-through to setCGroupMemoryLimits) +func updateCgroupMemoryLimits(podControl cgroupUpdater, rootMemReserveBytes uint64, lastAppliedLimit int64) (int64, bool, error) { + totalMemoryBytes, err := currentTotalMemoryBytes() + if err != nil { + return lastAppliedLimit, false, err + } + return setCgroupMemoryLimits(podControl, totalMemoryBytes, rootMemReserveBytes, lastAppliedLimit) +} + +// Set the cgroup memory limits for the containers and virtual-pods cgroups if they've changed +func setCgroupMemoryLimits(podControl cgroupUpdater, totalMemoryBytes, rootMemReserveBytes uint64, lastAppliedLimit int64) (int64, bool, error) { + podLimit, err := calculateContainersMemoryLimit(totalMemoryBytes, rootMemReserveBytes) + if err != nil { + return lastAppliedLimit, false, err + } + if podLimit == lastAppliedLimit { + return lastAppliedLimit, false, nil + } + resources := &oci.LinuxResources{ + Memory: &oci.LinuxMemory{Limit: &podLimit}, + } + + if err := podControl.Update(resources); err != nil { + return lastAppliedLimit, false, errors.Wrap(err, "failed to update containers cgroup memory limit") + } + return podLimit, true, nil +} + +// Start a periodic goroutine to monitor and update the cgroup memory limits for the containers and virtual-pods cgroups +func monitorCgroupMemoryLimits(containersControl cgroupUpdater, rootMemReserveBytes uint64, initialLimit int64, interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + lastAppliedLimit := initialLimit + for range ticker.C { + podLimit, updated, err := updateCgroupMemoryLimits(containersControl, rootMemReserveBytes, lastAppliedLimit) + if err != nil { + logrus.WithError(err).Error("failed to refresh cgroup memory limits") + continue + } + if !updated { + continue + } + lastAppliedLimit = podLimit + logrus.WithField("memoryLimitBytes", podLimit).Debug("refreshed cgroup memory limits") + } +} + // runWithRestartMonitor starts a command with given args and waits for it to exit. If the // command exit code is non-zero the command is restarted with with some back off delay. // Any stdout or stderr of the command will be split into lines and written as a log with @@ -351,11 +437,14 @@ func main() { // // The gcs cgroup is not limited but an event will get logged if memory // usage exceeds 50 MB. - sinfo := syscall.Sysinfo_t{} - if err := syscall.Sysinfo(&sinfo); err != nil { - logrus.WithError(err).Fatal("failed to get sys info") + totalMemoryBytes, err := currentTotalMemoryBytes() + if err != nil { + logrus.WithError(err).Fatal("failed to get total system memory") + } + podsLimit, err := calculateContainersMemoryLimit(totalMemoryBytes, *rootMemReserveBytes) + if err != nil { + logrus.WithError(err).Fatal("failed to calculate pods cgroup memory limit") } - podsLimit := int64(sinfo.Totalram - *rootMemReserveBytes) podsControl, err := cgroup.NewManager("/pods", &oci.LinuxResources{ Memory: &oci.LinuxMemory{ Limit: &podsLimit, @@ -383,6 +472,9 @@ func main() { h := hcsv2.NewHost(rtime, tport, initialEnforcer, logWriter) + // Start periodic task to update cgroup memory limits for containers and virtual-pods cgroups + go monitorCgroupMemoryLimits(podsControl, *rootMemReserveBytes, podsLimit, 60*time.Second) + // During live migration the VM is frozen and only wakes up when the host // shim is ready, so the vsock port should be immediately available. We // use a tight retry interval instead of exponential backoff. diff --git a/cmd/gcs/main_test.go b/cmd/gcs/main_test.go new file mode 100644 index 0000000000..e29728fec7 --- /dev/null +++ b/cmd/gcs/main_test.go @@ -0,0 +1,105 @@ +//go:build linux +// +build linux + +package main + +import ( + "reflect" + "strings" + "testing" + + oci "github.com/opencontainers/runtime-spec/specs-go" +) + +func TestCalculateContainersMemoryLimit(t *testing.T) { + tests := []struct { + name string + total uint64 + reserve uint64 + want int64 + wantErrText string + }{ + {name: "subtracts reserve", total: 4096, reserve: 1024, want: 3072}, + {name: "rejects equal reserve", total: 1024, reserve: 1024, wantErrText: "must be greater"}, + {name: "rejects below reserve", total: 512, reserve: 1024, wantErrText: "must be greater"}, + {name: "rejects int64 overflow", total: uint64(1 << 63), wantErrText: "exceeds maximum"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := calculateContainersMemoryLimit(test.total, test.reserve) + if test.wantErrText != "" { + if err == nil || !strings.Contains(err.Error(), test.wantErrText) { + t.Fatalf("expected error containing %q, got %v", test.wantErrText, err) + } + return + } + if err != nil { + t.Fatalf("calculateContainersMemoryLimit returned error: %v", err) + } + if got != test.want { + t.Fatalf("calculateContainersMemoryLimit returned %d, want %d", got, test.want) + } + }) + } +} + +func TestSetCgroupMemoryLimitsUpdatesParentBeforeChild(t *testing.T) { + var updateOrder []string + podsControl := &testCgroupUpdater{name: "pods", updateOrder: &updateOrder} + + limit, updated, err := setCgroupMemoryLimits(podsControl, 4096, 1024, 2048) + if err != nil { + t.Fatalf("setCgroupMemoryLimits returned error: %v", err) + } + if !updated { + t.Fatal("setCgroupMemoryLimits did not report an update") + } + if limit != 3072 { + t.Fatalf("setCgroupMemoryLimits returned limit %d, want 3072", limit) + } + if want := []string{"pods"}; !reflect.DeepEqual(updateOrder, want) { + t.Fatalf("cgroups updated in order %v, want %v", updateOrder, want) + } + if podsControl.limit != 3072 { + t.Fatalf("cgroup limits are pods=%d, want 3072", podsControl.limit) + } +} + +func TestSetCgroupMemoryLimitsSkipsUnchangedLimit(t *testing.T) { + podsControl := &testCgroupUpdater{} + + limit, updated, err := setCgroupMemoryLimits(podsControl, 4096, 1024, 3072) + if err != nil { + t.Fatalf("setCgroupMemoryLimits returned error: %v", err) + } + if updated { + t.Fatal("setCgroupMemoryLimits reported an update for an unchanged limit") + } + if limit != 3072 { + t.Fatalf("setCgroupMemoryLimits returned limit %d, want 3072", limit) + } + if podsControl.updates != 0 { + t.Fatalf("unchanged limit updated cgroups: pods=%d", podsControl.updates) + } +} + +type testCgroupUpdater struct { + name string + limit int64 + updates int + updateErr error + updateOrder *[]string +} + +func (cgroup *testCgroupUpdater) Update(resources *oci.LinuxResources) error { + cgroup.updates++ + if cgroup.updateErr != nil { + return cgroup.updateErr + } + cgroup.limit = *resources.Memory.Limit + if cgroup.updateOrder != nil { + *cgroup.updateOrder = append(*cgroup.updateOrder, cgroup.name) + } + return nil +}