From c71873f127c118d77a098a86502f7ad60ed7c88c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 09:41:17 +0000 Subject: [PATCH] build(deps): bump github.com/KimMachineGun/automemlimit Bumps [github.com/KimMachineGun/automemlimit](https://github.com/KimMachineGun/automemlimit) from 0.7.5 to 1.0.0. - [Release notes](https://github.com/KimMachineGun/automemlimit/releases) - [Commits](https://github.com/KimMachineGun/automemlimit/compare/v0.7.5...v1.0.0) --- updated-dependencies: - dependency-name: github.com/KimMachineGun/automemlimit dependency-version: 1.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 +- .../automemlimit/memlimit/cgroups.go | 511 ++++++++++++------ .../automemlimit/memlimit/cgroups_linux.go | 28 +- .../memlimit/cgroups_unsupported.go | 15 +- .../automemlimit/memlimit/experiment.go | 59 -- .../automemlimit/memlimit/logger.go | 19 +- .../automemlimit/memlimit/memlimit.go | 244 +++------ .../automemlimit/memlimit/provider.go | 16 +- .../memlimit/{exp_system.go => system.go} | 2 +- vendor/modules.txt | 4 +- 11 files changed, 481 insertions(+), 423 deletions(-) delete mode 100644 vendor/github.com/KimMachineGun/automemlimit/memlimit/experiment.go rename vendor/github.com/KimMachineGun/automemlimit/memlimit/{exp_system.go => system.go} (77%) diff --git a/go.mod b/go.mod index 731d7c83..f17f6c2b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module kubegems.io/modelx go 1.26 require ( - github.com/KimMachineGun/automemlimit v0.7.5 + github.com/KimMachineGun/automemlimit v1.0.0 github.com/aws/aws-sdk-go-v2 v1.45.1 github.com/aws/aws-sdk-go-v2/config v1.33.1 github.com/aws/aws-sdk-go-v2/credentials v1.20.1 diff --git a/go.sum b/go.sum index 62c3723a..af2715e0 100644 --- a/go.sum +++ b/go.sum @@ -34,8 +34,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/KimMachineGun/automemlimit v0.7.5 h1:RkbaC0MwhjL1ZuBKunGDjE/ggwAX43DwZrJqVwyveTk= -github.com/KimMachineGun/automemlimit v0.7.5/go.mod h1:QZxpHaGOQoYvFhv/r4u3U0JTC2ZcOwbSr11UZF46UBM= +github.com/KimMachineGun/automemlimit v1.0.0 h1:+MqlvDE/pkJNjk1rU+O14QsH8k10nJAD0frB0lsxyvw= +github.com/KimMachineGun/automemlimit v1.0.0/go.mod h1:n+BSXxQWDFS1DKh67Rqo0lgTsowsg6x65ak5uyngML0= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups.go index 2f0a8404..dc70865e 100644 --- a/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups.go +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups.go @@ -13,19 +13,22 @@ import ( "strings" ) +const ( + procSelfMountInfoPath = "/proc/self/mountinfo" + procSelfCgroupPath = "/proc/self/cgroup" +) + var ( - // ErrNoCgroup is returned when the process is not in cgroup. + // ErrNoCgroup is returned when the process is not assigned to a cgroup. ErrNoCgroup = errors.New("process is not in cgroup") // ErrCgroupsNotSupported is returned when the system does not support cgroups. ErrCgroupsNotSupported = errors.New("cgroups is not supported on this system") ) -// fromCgroup retrieves the memory limit from the cgroup. -// The versionDetector function is used to detect the cgroup version from the mountinfo. -func fromCgroup(versionDetector func(mis []mountInfo) (bool, bool)) (uint64, error) { - mf, err := os.Open("/proc/self/mountinfo") +func fromCgroup(mountInfoPath, cgroupPath string) (uint64, error) { + mf, err := os.Open(mountInfoPath) if err != nil { - return 0, fmt.Errorf("failed to open /proc/self/mountinfo: %w", err) + return 0, fmt.Errorf("failed to open %s: %w", mountInfoPath, err) } defer mf.Close() @@ -34,14 +37,9 @@ func fromCgroup(versionDetector func(mis []mountInfo) (bool, bool)) (uint64, err return 0, fmt.Errorf("failed to parse mountinfo: %w", err) } - v1, v2 := versionDetector(mis) - if !(v1 || v2) { - return 0, ErrNoCgroup - } - - cf, err := os.Open("/proc/self/cgroup") + cf, err := os.Open(cgroupPath) if err != nil { - return 0, fmt.Errorf("failed to open /proc/self/cgroup: %w", err) + return 0, fmt.Errorf("failed to open %s: %w", cgroupPath, err) } defer cf.Close() @@ -50,70 +48,75 @@ func fromCgroup(versionDetector func(mis []mountInfo) (bool, bool)) (uint64, err return 0, fmt.Errorf("failed to parse cgroup file: %w", err) } - if v2 { - limit, err := getMemoryLimitV2(chs, mis) - if err == nil { - return limit, nil - } else if !v1 { - return 0, err - } + controller, err := selectMemoryController(chs, mis) + if err != nil { + return 0, err + } + if controller.isV2 { + return getMemoryLimitV2FromControllerPath(controller.path, mis) } - return getMemoryLimitV1(chs, mis) + return getMemoryLimitV1FromControllerPath(controller.path, mis) } -// detectCgroupVersion detects the cgroup version from the mountinfo. -func detectCgroupVersion(mis []mountInfo) (bool, bool) { - var v1, v2 bool - for _, mi := range mis { - switch mi.FilesystemType { - case "cgroup": - v1 = true - case "cgroup2": - v2 = true +type memoryController struct { + path string + isV2 bool +} + +// selectMemoryController selects v1 when both v1 and v2 are available. +func selectMemoryController(chs []cgroupHierarchy, mis []mountInfo) (memoryController, error) { + var ( + v1Path string + v2Path string + hasV1Entry bool + hasV2Entry bool + hasV1Mount bool + hasV2Mount bool + ) + + for _, ch := range chs { + if !hasV1Entry && ch.HierarchyID != "0" && slices.Contains(strings.Split(ch.ControllerList, ","), "memory") { + v1Path = ch.CgroupPath + hasV1Entry = true + } else if !hasV2Entry && ch.HierarchyID == "0" && ch.ControllerList == "" { + v2Path = ch.CgroupPath + hasV2Entry = true } } - return v1, v2 -} -// getMemoryLimitV2 retrieves the memory limit from the cgroup v2 controller. -func getMemoryLimitV2(chs []cgroupHierarchy, mis []mountInfo) (uint64, error) { - // find the cgroup v2 path for the memory controller. - // in cgroup v2, the paths are unified and the controller list is empty. - idx := slices.IndexFunc(chs, func(ch cgroupHierarchy) bool { - return ch.HierarchyID == "0" && ch.ControllerList == "" - }) - if idx == -1 { - return 0, errors.New("cgroup v2 path not found") - } - relPath := chs[idx].CgroupPath - - // find the mountpoint for the cgroup v2 controller. - idx = slices.IndexFunc(mis, func(mi mountInfo) bool { - return mi.FilesystemType == "cgroup2" - }) - if idx == -1 { - return 0, errors.New("cgroup v2 mountpoint not found") - } - root, mountPoint := mis[idx].Root, mis[idx].MountPoint - - // resolve the actual cgroup path - cgroupPath, err := resolveCgroupPath(mountPoint, root, relPath) - if err != nil { - return 0, err + for _, mi := range mis { + if !hasV1Mount && mi.FilesystemType == "cgroup" && slices.Contains(strings.Split(mi.SuperOptions, ","), "memory") { + hasV1Mount = true + } else if !hasV2Mount && mi.FilesystemType == "cgroup2" { + // cgroup v2 uses a unified hierarchy, so the filesystem type is sufficient + hasV2Mount = true + } } - // retrieve the memory limit from the memory.max recursively. - return walkCgroupV2Hierarchy(cgroupPath, mountPoint) + switch { + case hasV1Entry: + if !hasV1Mount { + return memoryController{}, errors.New("memory controller found in /proc/self/cgroup but no cgroup v1 memory mount found") + } + return memoryController{path: v1Path}, nil + case hasV2Entry: + if !hasV2Mount { + return memoryController{}, errors.New("cgroup v2 hierarchy found in /proc/self/cgroup but no cgroup2 mount found") + } + return memoryController{path: v2Path, isV2: true}, nil + default: + return memoryController{}, ErrNoCgroup + } } -// readMemoryLimitV2FromPath reads the memory limit for cgroup v2 from the given path. -// this function expects the path to be memory.max file. +// readMemoryLimitV2FromPath reads a memory limit from a cgroup v2 memory.max file. +// It returns [ErrNoLimit] for "max" and preserves [os.ErrNotExist] for a missing file. func readMemoryLimitV2FromPath(path string) (uint64, error) { b, err := os.ReadFile(path) if err != nil { if errors.Is(err, os.ErrNotExist) { - return 0, ErrNoLimit + return 0, err } return 0, fmt.Errorf("failed to read memory.max: %w", err) } @@ -131,22 +134,21 @@ func readMemoryLimitV2FromPath(path string) (uint64, error) { return limit, nil } -// walkCgroupV2Hierarchy walks up the cgroup v2 hierarchy to find the most restrictive memory limit. +// walkCgroupV2Hierarchy returns the smallest limit between cgroupPath and mountPoint. +// It skips missing and unlimited values, returning [ErrNoLimit] if no concrete limit remains. func walkCgroupV2Hierarchy(cgroupPath, mountPoint string) (uint64, error) { var ( - found = false - minLimit uint64 = math.MaxUint64 - currentPath = cgroupPath + found = false + minLimit uint64 = math.MaxUint64 ) - for { + for currentPath := cgroupPath; ; { limit, err := readMemoryLimitV2FromPath(filepath.Join(currentPath, "memory.max")) - if err != nil && !errors.Is(err, ErrNoLimit) { - return 0, err - } else if err == nil { + if err == nil { found = true minLimit = min(minLimit, limit) + } else if !errors.Is(err, os.ErrNotExist) && !errors.Is(err, ErrNoLimit) { + return 0, err } - if currentPath == mountPoint { break } @@ -164,83 +166,236 @@ func walkCgroupV2Hierarchy(cgroupPath, mountPoint string) (uint64, error) { return minLimit, nil } -// getMemoryLimitV1 retrieves the memory limit from the cgroup v1 controller. -func getMemoryLimitV1(chs []cgroupHierarchy, mis []mountInfo) (uint64, error) { - // find the cgroup v1 path for the memory controller. - idx := slices.IndexFunc(chs, func(ch cgroupHierarchy) bool { - return slices.Contains(strings.Split(ch.ControllerList, ","), "memory") - }) - if idx == -1 { - return 0, errors.New("cgroup v1 path for memory controller not found") +// getMemoryLimitV2FromControllerPath prefers mounts rooted closest to the cgroup hierarchy root. +func getMemoryLimitV2FromControllerPath(relPath string, mis []mountInfo) (uint64, error) { + rootClosestMounts, err := getRootClosestMountCandidates( + relPath, + mis, + func(mi mountInfo) bool { + return mi.FilesystemType == "cgroup2" + }, + ) + if err != nil { + return 0, err + } + + limit, found, err := getMemoryLimitFromMountCandidates( + rootClosestMounts, + func(mi mountInfo) (uint64, bool, error) { + cgroupPath, err := resolveCgroupPath(mi.MountPoint, mi.Root, relPath) + if err != nil { + return 0, false, err + } + + stat, err := os.Stat(cgroupPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return 0, false, nil + } + return 0, false, err + } + if !stat.IsDir() { + return 0, false, fmt.Errorf("cgroup v2 path %s is not a directory", cgroupPath) + } + + limit, err := walkCgroupV2Hierarchy(cgroupPath, mi.MountPoint) + if err != nil { + return 0, false, err + } + + return limit, true, nil + }, + ) + if err != nil { + return 0, err + } else if found { + return limit, nil } - relPath := chs[idx].CgroupPath - // find the mountpoint for the cgroup v1 controller. - idx = slices.IndexFunc(mis, func(mi mountInfo) bool { - return mi.FilesystemType == "cgroup" && slices.Contains(strings.Split(mi.SuperOptions, ","), "memory") - }) - if idx == -1 { - return 0, errors.New("cgroup v1 mountpoint for memory controller not found") + return 0, errors.New("no usable cgroup v2 memory mount found") +} + +// getMemoryLimitV1FromControllerPath prefers mounts rooted closest to the cgroup hierarchy root. +func getMemoryLimitV1FromControllerPath(relPath string, mis []mountInfo) (uint64, error) { + rootClosestMounts, err := getRootClosestMountCandidates( + relPath, + mis, + func(mi mountInfo) bool { + return mi.FilesystemType == "cgroup" && slices.Contains(strings.Split(mi.SuperOptions, ","), "memory") + }, + ) + if err != nil { + return 0, err } - root, mountPoint := mis[idx].Root, mis[idx].MountPoint - // resolve the actual cgroup path - cgroupPath, err := resolveCgroupPath(mountPoint, root, relPath) + limit, found, err := getMemoryLimitFromMountCandidates( + rootClosestMounts, + func(mi mountInfo) (uint64, bool, error) { + cgroupPath, err := resolveCgroupPath(mi.MountPoint, mi.Root, relPath) + if err != nil { + return 0, false, err + } + + return readMemoryLimitV1FromPath(cgroupPath) + }, + ) if err != nil { return 0, err + } else if found { + return limit, nil } - // retrieve the memory limit from the memory.stat and memory.limit_in_bytes files. - return readMemoryLimitV1FromPath(cgroupPath) + return 0, errors.New("no usable cgroup v1 memory mount found") } -// getCgroupV1NoLimit returns the maximum value that is used to represent no limit in cgroup v1. -// the max memory limit is max int64, but it should be multiple of the page size. -func getCgroupV1NoLimit() uint64 { - ps := uint64(os.Getpagesize()) - return math.MaxInt64 / ps * ps +func getRootClosestMountCandidates( + relPath string, + mis []mountInfo, + isCandidate func(mi mountInfo) bool, +) ([]mountInfo, error) { + var ( + rootClosestMounts []mountInfo + maxDepth = -1 + ) + for _, mi := range mis { + if !isCandidate(mi) { + continue + } + + cgroupPath, err := resolveCgroupPath(mi.MountPoint, mi.Root, relPath) + if err != nil { + return nil, err + } else if cgroupPath == "" { + continue + } + if _, err := os.Stat(cgroupPath); errors.Is(err, os.ErrNotExist) { + continue + } + + rel, err := filepath.Rel(mi.MountPoint, cgroupPath) + if err != nil { + return nil, err + } + + depth := 0 + if rel != "." { + depth = strings.Count(rel, string(filepath.Separator)) + 1 + } + + switch { + case depth > maxDepth: + rootClosestMounts = []mountInfo{mi} + maxDepth = depth + case depth == maxDepth: + rootClosestMounts = append(rootClosestMounts, mi) + } + } + + return rootClosestMounts, nil +} + +// getLimit may return found=false to skip a candidate or [ErrNoLimit] for no usable limit. +// Other errors and conflicting concrete limits are returned. +func getMemoryLimitFromMountCandidates( + mis []mountInfo, + getLimit func(mi mountInfo) (uint64, bool, error), +) (uint64, bool, error) { + var ( + firstErr, conflictErr error + sawNoLimit, limitFound bool + firstLimit uint64 + ) + for _, mi := range mis { + limit, found, err := getLimit(mi) + if err != nil { + if errors.Is(err, ErrNoLimit) { + sawNoLimit = true + } else if firstErr == nil { + firstErr = err + } + continue + } else if !found { + continue + } + + if !limitFound { + limitFound = true + firstLimit = limit + } else if limit != firstLimit && conflictErr == nil { + conflictErr = fmt.Errorf("conflicting memory limits from cgroup mount candidates: %d and %d", firstLimit, limit) + } + } + if firstErr != nil { + return 0, false, firstErr + } + if conflictErr != nil { + return 0, false, conflictErr + } + + if limitFound { + return firstLimit, true, nil + } + if sawNoLimit { + return 0, false, ErrNoLimit + } + + return 0, false, nil +} + +// cgroup v1 uses the kernel's maximum page counter value to represent no limit +func isCgroupV1NoLimit(limit uint64) bool { + pageSize := uint64(os.Getpagesize()) + if limit >= math.MaxInt64/pageSize*pageSize { + return true + } + + // strconv.IntSize reflects the Go binary width, not the kernel bitness + if strconv.IntSize == 32 { + return limit == math.MaxInt32*pageSize + } + + return false } -// readMemoryLimitV1FromPath reads the memory limit for cgroup v1 from the given path. -// this function expects the path to be the cgroup directory. -func readMemoryLimitV1FromPath(cgroupPath string) (uint64, error) { - // read hierarchical_memory_limit and memory.limit_in_bytes files. - // but if hierarchical_memory_limit is not available, then use the max value as a fallback. - hml, err := readHierarchicalMemoryLimit(filepath.Join(cgroupPath, "memory.stat")) +// readMemoryLimitV1FromPath reads the effective memory limit from a cgroup v1 directory. +// It returns [ErrNoLimit] for a no-limit sentinel. +// The bool reports whether a limit value was found. +func readMemoryLimitV1FromPath(cgroupPath string) (uint64, bool, error) { + // use math.MaxUint64 as a neutral fallback so memory.limit_in_bytes determines the result + hml, hmlFound, err := readHierarchicalMemoryLimit(filepath.Join(cgroupPath, "memory.stat")) if err != nil && !errors.Is(err, os.ErrNotExist) { - return 0, fmt.Errorf("failed to read hierarchical_memory_limit: %w", err) - } else if hml == 0 { + return 0, false, fmt.Errorf("failed to read hierarchical_memory_limit: %w", err) + } else if !hmlFound { hml = math.MaxUint64 } - // read memory.limit_in_bytes file. - b, err := os.ReadFile(filepath.Join(cgroupPath, "memory.limit_in_bytes")) + var libFound bool + lib, err := readMemoryLimitInBytes(filepath.Join(cgroupPath, "memory.limit_in_bytes")) if err != nil && !errors.Is(err, os.ErrNotExist) { - return 0, fmt.Errorf("failed to read memory.limit_in_bytes: %w", err) + return 0, false, err + } else if errors.Is(err, os.ErrNotExist) { + lib = math.MaxUint64 + } else { + libFound = true } - lib, err := strconv.ParseUint(strings.TrimSpace(string(b)), 10, 64) - if err != nil { - return 0, fmt.Errorf("failed to parse memory.limit_in_bytes value: %w", err) - } else if lib == 0 { - hml = math.MaxUint64 + + if !hmlFound && !libFound { + return 0, false, nil } - // use the minimum value between hierarchical_memory_limit and memory.limit_in_bytes. - // if the limit is the maximum value, then it is considered as no limit. limit := min(hml, lib) - if limit >= getCgroupV1NoLimit() { - return 0, ErrNoLimit + if isCgroupV1NoLimit(limit) { + return 0, true, ErrNoLimit } - return limit, nil + return limit, true, nil } -// readHierarchicalMemoryLimit extracts hierarchical_memory_limit from memory.stat. -// this function expects the path to be memory.stat file. -func readHierarchicalMemoryLimit(path string) (uint64, error) { +// readHierarchicalMemoryLimit returns false if memory.stat has no hierarchical_memory_limit field. +func readHierarchicalMemoryLimit(path string) (uint64, bool, error) { file, err := os.Open(path) if err != nil { - return 0, err + return 0, false, err } defer file.Close() @@ -248,23 +403,42 @@ func readHierarchicalMemoryLimit(path string) (uint64, error) { for scanner.Scan() { line := scanner.Text() - fields := strings.Split(line, " ") + fields := strings.Fields(line) + if len(fields) == 0 || fields[0] != "hierarchical_memory_limit" { + continue + } if len(fields) < 2 { - return 0, fmt.Errorf("failed to parse memory.stat %q: not enough fields", line) + return 0, false, fmt.Errorf("failed to parse memory.stat %q: not enough fields", line) + } else if len(fields) > 2 { + return 0, false, fmt.Errorf("failed to parse memory.stat %q: too many fields for hierarchical_memory_limit", line) } - if fields[0] == "hierarchical_memory_limit" { - if len(fields) > 2 { - return 0, fmt.Errorf("failed to parse memory.stat %q: too many fields for hierarchical_memory_limit", line) - } - return strconv.ParseUint(fields[1], 10, 64) + limit, err := strconv.ParseUint(fields[1], 10, 64) + if err != nil { + return 0, false, fmt.Errorf("failed to parse hierarchical_memory_limit value: %w", err) } + + return limit, true, nil } if err := scanner.Err(); err != nil { - return 0, err + return 0, false, err } - return 0, nil + return 0, false, nil +} + +func readMemoryLimitInBytes(path string) (uint64, error) { + b, err := os.ReadFile(path) + if err != nil { + return 0, fmt.Errorf("failed to read memory.limit_in_bytes: %w", err) + } + + limit, err := strconv.ParseUint(strings.TrimSpace(string(b)), 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse memory.limit_in_bytes value: %w", err) + } + + return limit, nil } // https://www.man7.org/linux/man-pages/man5/proc_pid_mountinfo.5.html @@ -291,7 +465,6 @@ type mountInfo struct { SuperOptions string } -// parseMountInfoLine parses a line from the mountinfo file. func parseMountInfoLine(line string) (mountInfo, error) { if line == "" { return mountInfo{}, errors.New("empty line") @@ -315,14 +488,50 @@ func parseMountInfoLine(line string) (mountInfo, error) { } return mountInfo{ - Root: fields1[3], - MountPoint: fields1[4], + Root: unescapeMountInfoPath(fields1[3]), + MountPoint: unescapeMountInfoPath(fields1[4]), FilesystemType: fields2[0], SuperOptions: fields2[2], }, nil } -// parseMountInfo parses the mountinfo file. +// unescapeMountInfoPath decodes path escapes written to /proc//mountinfo. +// https://github.com/torvalds/linux/blob/master/fs/proc_namespace.c +func unescapeMountInfoPath(path string) string { + if strings.IndexByte(path, '\\') == -1 { + return path + } + + var b strings.Builder + b.Grow(len(path)) + for i := 0; i < len(path); i++ { + if path[i] == '\\' && i+3 < len(path) { + switch path[i : i+4] { + case `\040`: + b.WriteByte(' ') + i += 3 + continue + case `\011`: + b.WriteByte('\t') + i += 3 + continue + case `\012`: + b.WriteByte('\n') + i += 3 + continue + case `\134`: + b.WriteByte('\\') + i += 3 + continue + } + } + + b.WriteByte(path[i]) + } + + return b.String() +} + func parseMountInfo(r io.Reader) ([]mountInfo, error) { var ( s = bufio.NewScanner(r) @@ -377,17 +586,14 @@ type cgroupHierarchy struct { CgroupPath string } -// parseCgroupHierarchyLine parses a line from the cgroup file. func parseCgroupHierarchyLine(line string) (cgroupHierarchy, error) { if line == "" { return cgroupHierarchy{}, errors.New("empty line") } - fields := strings.Split(line, ":") + fields := strings.SplitN(line, ":", 3) if len(fields) < 3 { return cgroupHierarchy{}, fmt.Errorf("not enough fields: %v", fields) - } else if len(fields) > 3 { - return cgroupHierarchy{}, fmt.Errorf("too many fields: %v", fields) } return cgroupHierarchy{ @@ -397,7 +603,6 @@ func parseCgroupHierarchyLine(line string) (cgroupHierarchy, error) { }, nil } -// parseCgroupFile parses the cgroup file. func parseCgroupFile(r io.Reader) ([]cgroupHierarchy, error) { var ( s = bufio.NewScanner(r) @@ -420,22 +625,26 @@ func parseCgroupFile(r io.Reader) ([]cgroupHierarchy, error) { return chs, nil } -// resolveCgroupPath resolves the actual cgroup path from the mountpoint, root, and cgroupRelPath. -func resolveCgroupPath(mountpoint, root, cgroupRelPath string) (string, error) { - rel, err := filepath.Rel(root, cgroupRelPath) - if err != nil { - return "", err +// resolveCgroupPath maps cgroupRelPath from root into mountPoint. +// It returns an empty path when cgroupRelPath lies outside root. +func resolveCgroupPath(mountPoint, root, cgroupRelPath string) (string, error) { + if !strings.HasPrefix(root, "/") || !strings.HasPrefix(cgroupRelPath, "/") { + return "", errors.New("cgroup root and path must be absolute") + } + + if root == cgroupRelPath { + return mountPoint, nil } - // if the relative path is ".", then the cgroupRelPath is the root itself. - if rel == "." { - return mountpoint, nil + prefix := root + if root != "/" { + prefix += "/" } - // if the relative path starts with "..", then it is outside the root. - if strings.HasPrefix(rel, "..") { - return "", fmt.Errorf("invalid cgroup path: %s is not under root %s", cgroupRelPath, root) + rel, ok := strings.CutPrefix(cgroupRelPath, prefix) + if !ok || !filepath.IsLocal(rel) { + return "", nil } - return filepath.Join(mountpoint, rel), nil + return filepath.Join(mountPoint, rel), nil } diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_linux.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_linux.go index fd2c7e49..575f0a9e 100644 --- a/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_linux.go +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_linux.go @@ -1,32 +1,8 @@ //go:build linux -// +build linux package memlimit -// FromCgroup retrieves the memory limit from the cgroup. +// FromCgroup returns the memory limit for the current process's cgroup. func FromCgroup() (uint64, error) { - return fromCgroup(detectCgroupVersion) -} - -// FromCgroupV1 retrieves the memory limit from the cgroup v1 controller. -// After v1.0.0, this function could be removed and FromCgroup should be used instead. -func FromCgroupV1() (uint64, error) { - return fromCgroup(func(_ []mountInfo) (bool, bool) { - return true, false - }) -} - -// FromCgroupHybrid retrieves the memory limit from the cgroup v2 and v1 controller sequentially, -// basically, it is equivalent to FromCgroup. -// After v1.0.0, this function could be removed and FromCgroup should be used instead. -func FromCgroupHybrid() (uint64, error) { - return FromCgroup() -} - -// FromCgroupV2 retrieves the memory limit from the cgroup v2 controller. -// After v1.0.0, this function could be removed and FromCgroup should be used instead. -func FromCgroupV2() (uint64, error) { - return fromCgroup(func(_ []mountInfo) (bool, bool) { - return false, true - }) + return fromCgroup(procSelfMountInfoPath, procSelfCgroupPath) } diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_unsupported.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_unsupported.go index 9feca81a..f96f7dad 100644 --- a/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_unsupported.go +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_unsupported.go @@ -1,20 +1,9 @@ //go:build !linux -// +build !linux package memlimit +// FromCgroup returns the memory limit for the current process's cgroup. +// On non-Linux platforms, it always returns [ErrCgroupsNotSupported]. func FromCgroup() (uint64, error) { return 0, ErrCgroupsNotSupported } - -func FromCgroupV1() (uint64, error) { - return 0, ErrCgroupsNotSupported -} - -func FromCgroupHybrid() (uint64, error) { - return 0, ErrCgroupsNotSupported -} - -func FromCgroupV2() (uint64, error) { - return 0, ErrCgroupsNotSupported -} diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/experiment.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/experiment.go deleted file mode 100644 index 2a7c320e..00000000 --- a/vendor/github.com/KimMachineGun/automemlimit/memlimit/experiment.go +++ /dev/null @@ -1,59 +0,0 @@ -package memlimit - -import ( - "fmt" - "os" - "reflect" - "strings" -) - -const ( - envAUTOMEMLIMIT_EXPERIMENT = "AUTOMEMLIMIT_EXPERIMENT" -) - -// Experiments is a set of experiment flags. -// It is used to enable experimental features. -// -// You can set the flags by setting the environment variable AUTOMEMLIMIT_EXPERIMENT. -// The value of the environment variable is a comma-separated list of experiment names. -// -// The following experiment names are known: -// -// - none: disable all experiments -// - system: enable fallback to system memory limit -type Experiments struct { - // System enables fallback to system memory limit. - System bool -} - -func parseExperiments() (Experiments, error) { - var exp Experiments - - // Create a map of known experiment names. - names := make(map[string]func(bool)) - rv := reflect.ValueOf(&exp).Elem() - rt := rv.Type() - for i := 0; i < rt.NumField(); i++ { - field := rv.Field(i) - names[strings.ToLower(rt.Field(i).Name)] = field.SetBool - } - - // Parse names. - for _, f := range strings.Split(os.Getenv(envAUTOMEMLIMIT_EXPERIMENT), ",") { - if f == "" { - continue - } - if f == "none" { - exp = Experiments{} - continue - } - val := true - set, ok := names[f] - if !ok { - return Experiments{}, fmt.Errorf("unknown AUTOMEMLIMIT_EXPERIMENT %s", f) - } - set(val) - } - - return exp, nil -} diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/logger.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/logger.go index 4cf0b589..8f683043 100644 --- a/vendor/github.com/KimMachineGun/automemlimit/memlimit/logger.go +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/logger.go @@ -5,9 +5,18 @@ import ( "log/slog" ) -type noopLogger struct{} +var _ slog.Handler = discardHandler{} -func (noopLogger) Enabled(context.Context, slog.Level) bool { return false } -func (noopLogger) Handle(context.Context, slog.Record) error { return nil } -func (d noopLogger) WithAttrs([]slog.Attr) slog.Handler { return d } -func (d noopLogger) WithGroup(string) slog.Handler { return d } +type discardHandler struct{} + +func (discardHandler) Enabled(context.Context, slog.Level) bool { return false } +func (discardHandler) Handle(context.Context, slog.Record) error { return nil } +func (dh discardHandler) WithAttrs([]slog.Attr) slog.Handler { return dh } +func (dh discardHandler) WithGroup(string) slog.Handler { return dh } + +func memlimitLogger(logger *slog.Logger) *slog.Logger { + if logger == nil { + return slog.New(discardHandler{}) + } + return logger.With(slog.String("package", "github.com/KimMachineGun/automemlimit/memlimit")) +} diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/memlimit.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/memlimit.go index b23980a5..d594c349 100644 --- a/vendor/github.com/KimMachineGun/automemlimit/memlimit/memlimit.go +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/memlimit.go @@ -1,6 +1,8 @@ +// Package memlimit configures GOMEMLIMIT from a memory limit provider. package memlimit import ( + "context" "errors" "fmt" "log/slog" @@ -14,26 +16,28 @@ import ( const ( envGOMEMLIMIT = "GOMEMLIMIT" envAUTOMEMLIMIT = "AUTOMEMLIMIT" - // Deprecated: use memlimit.WithLogger instead - envAUTOMEMLIMIT_DEBUG = "AUTOMEMLIMIT_DEBUG" defaultAUTOMEMLIMIT = 0.9 ) -// ErrNoLimit is returned when the memory limit is not set. +// ErrNoLimit indicates that a [Provider] reports no memory limit. +// [Set] handles it as success by setting GOMEMLIMIT to [math.MaxInt64]. var ErrNoLimit = errors.New("memory is not limited") type config struct { - logger *slog.Logger - ratio float64 - provider Provider - refresh time.Duration + logger *slog.Logger + ratio float64 + minLimit int64 + provider Provider + refresh time.Duration + refreshCtx context.Context } -// Option is a function that configures the behavior of SetGoMemLimitWithOptions. +// Option configures the behavior of [Set]. type Option func(cfg *config) -// WithRatio configures the ratio of the memory limit to set as GOMEMLIMIT. +// WithRatio configures the fraction of the memory limit used for GOMEMLIMIT. +// The ratio must be in (0.0, 1.0]. // // Default: 0.9 func WithRatio(ratio float64) Option { @@ -42,9 +46,19 @@ func WithRatio(ratio float64) Option { } } -// WithProvider configures the provider. +// WithMin configures the minimum GOMEMLIMIT after applying the ratio. +// A non-positive value disables the minimum. // -// Default: FromCgroup +// Default: 0 (no minimum) +func WithMin(minLimit int64) Option { + return func(cfg *config) { + cfg.minLimit = minLimit + } +} + +// WithProvider configures the provider used by [Set]. +// +// Default: [FromCgroup] func WithProvider(provider Provider) Option { return func(cfg *config) { cfg.provider = provider @@ -52,233 +66,149 @@ func WithProvider(provider Provider) Option { } // WithLogger configures the logger. -// It automatically attaches the "package" attribute to the logs. +// It adds the "package" attribute to every log record. // -// Default: slog.New(noopLogger{}) +// Default: logging disabled func WithLogger(logger *slog.Logger) Option { return func(cfg *config) { cfg.logger = memlimitLogger(logger) } } -// WithRefreshInterval configures the refresh interval for automemlimit. -// If a refresh interval is greater than 0, automemlimit periodically fetches -// the memory limit from the provider and reapplies it if it has changed. -// If the provider returns an error, it logs the error and continues. -// ErrNoLimit is treated as math.MaxInt64. +// WithRefreshInterval configures [Set] to periodically refresh GOMEMLIMIT. +// +// Set starts a refresh goroutine after the initial provider call +// when refresh is positive and ctx is non-nil. +// The goroutine starts even if the initial call returns an error. +// Canceling ctx stops the refresh loop but does not interrupt an active provider call. +// Provider errors other than [ErrNoLimit] are reported to the configured logger +// and do not stop later refreshes. // // Default: 0 (no refresh) -func WithRefreshInterval(refresh time.Duration) Option { +func WithRefreshInterval(ctx context.Context, refresh time.Duration) Option { return func(cfg *config) { cfg.refresh = refresh + cfg.refreshCtx = ctx } } -// WithEnv configures whether to use environment variables. -// -// Default: false -// -// Deprecated: currently this does nothing. -func WithEnv() Option { - return func(cfg *config) {} -} - -func memlimitLogger(logger *slog.Logger) *slog.Logger { - if logger == nil { - return slog.New(noopLogger{}) - } - return logger.With(slog.String("package", "github.com/KimMachineGun/automemlimit/memlimit")) -} - -// SetGoMemLimitWithOpts sets GOMEMLIMIT with options and environment variables. +// Set sets GOMEMLIMIT using the configured provider. // -// You can configure how much memory of the cgroup's memory limit to set as GOMEMLIMIT -// through AUTOMEMLIMIT environment variable in the half-open range (0.0,1.0]. +// By default, Set uses 90% of the limit reported by [FromCgroup]. +// AUTOMEMLIMIT overrides the configured ratio with a value in (0.0, 1.0]. +// Setting it to "off" disables Set. // -// If AUTOMEMLIMIT is not set, it defaults to 0.9. (10% is the headroom for memory sources the Go runtime is unaware of.) -// If GOMEMLIMIT is already set or AUTOMEMLIMIT=off, this function does nothing. +// If the GOMEMLIMIT environment variable is present or AUTOMEMLIMIT is set to "off", +// Set returns the current GOMEMLIMIT without calling the provider or starting a refresh goroutine. // -// If AUTOMEMLIMIT_EXPERIMENT is set, it enables experimental features. -// Please see the documentation of Experiments for more details. -// -// Options: -// - WithRatio -// - WithProvider -// - WithLogger -func SetGoMemLimitWithOpts(opts ...Option) (_ int64, _err error) { - // init config +// Set returns the resulting GOMEMLIMIT on success. +// On error, it returns the previous GOMEMLIMIT and the error. +// It handles [ErrNoLimit] as success by setting GOMEMLIMIT to [math.MaxInt64]. +func Set(opts ...Option) (_limit int64, _err error) { cfg := &config{ - logger: slog.New(noopLogger{}), + logger: slog.New(discardHandler{}), ratio: defaultAUTOMEMLIMIT, provider: FromCgroup, } - // TODO: remove this - if debug, ok := os.LookupEnv(envAUTOMEMLIMIT_DEBUG); ok { - defaultLogger := memlimitLogger(slog.Default()) - defaultLogger.Warn("AUTOMEMLIMIT_DEBUG is deprecated, use memlimit.WithLogger instead") - if debug == "true" { - cfg.logger = defaultLogger - } - } for _, opt := range opts { opt(cfg) } - // log error if any on return defer func() { if _err != nil { cfg.logger.Error("failed to set GOMEMLIMIT", slog.Any("error", _err)) } }() - // parse experiments - exps, err := parseExperiments() - if err != nil { - return 0, fmt.Errorf("failed to parse experiments: %w", err) - } - if exps.System { - cfg.logger.Info("system experiment is enabled: using system memory limit as a fallback") - cfg.provider = ApplyFallback(cfg.provider, FromSystem) - } - - // rollback to previous memory limit on panic snapshot := debug.SetMemoryLimit(-1) - defer rollbackOnPanic(cfg.logger, snapshot, &_err) - // check if GOMEMLIMIT is already set if val, ok := os.LookupEnv(envGOMEMLIMIT); ok { cfg.logger.Info("GOMEMLIMIT is already set, skipping", slog.String(envGOMEMLIMIT, val)) - return 0, nil + return snapshot, nil } - // parse AUTOMEMLIMIT ratio := cfg.ratio if val, ok := os.LookupEnv(envAUTOMEMLIMIT); ok { if val == "off" { - cfg.logger.Info("AUTOMEMLIMIT is set to off, skipping") - return 0, nil + cfg.logger.Info("AUTOMEMLIMIT is off, skipping") + return snapshot, nil } - ratio, err = strconv.ParseFloat(val, 64) + + r, err := strconv.ParseFloat(val, 64) if err != nil { - return 0, fmt.Errorf("cannot parse AUTOMEMLIMIT: %s", val) + return snapshot, fmt.Errorf("cannot parse AUTOMEMLIMIT: %s", val) } + ratio = r + } + if math.IsNaN(ratio) || ratio <= 0 || ratio > 1 { + return snapshot, fmt.Errorf( + "failed to set GOMEMLIMIT: invalid ratio: %f, ratio should be in the range (0.0,1.0]", + ratio, + ) } - // apply ratio to the provider - provider := capProvider(ApplyRatio(cfg.provider, ratio)) - - // set the memory limit and start refresh - limit, err := updateGoMemLimit(uint64(snapshot), provider, cfg.logger) - refresh(provider, cfg.logger, cfg.refresh) + provider := boundedProvider(ApplyRatio(cfg.provider, ratio), cfg.minLimit) + limit, err := updateGoMemLimit(provider, cfg.logger) + if cfg.refresh > 0 && cfg.refreshCtx != nil { + go refresh(cfg.refreshCtx, provider, cfg.logger, cfg.refresh) + } if err != nil { - if errors.Is(err, ErrNoLimit) { - cfg.logger.Info("memory is not limited, skipping") - // TODO: consider returning the snapshot - return 0, nil - } - return 0, fmt.Errorf("failed to set GOMEMLIMIT: %w", err) + return snapshot, fmt.Errorf("failed to set GOMEMLIMIT: %w", err) } return int64(limit), nil } -// updateGoMemLimit updates the Go's memory limit, if it has changed. -func updateGoMemLimit(currLimit uint64, provider Provider, logger *slog.Logger) (uint64, error) { +func updateGoMemLimit(provider Provider, logger *slog.Logger) (uint64, error) { newLimit, err := provider() if err != nil { + if errors.Is(err, ErrNoLimit) { + return updateGoMemLimit(Limit(math.MaxInt64), logger) + } return 0, err } - if newLimit == currLimit { - logger.Debug("GOMEMLIMIT is not changed, skipping", slog.Uint64(envGOMEMLIMIT, newLimit)) + previous := debug.SetMemoryLimit(int64(newLimit)) + if newLimit == uint64(previous) { + logger.Debug("GOMEMLIMIT is unchanged", slog.Uint64(envGOMEMLIMIT, newLimit)) return newLimit, nil } - debug.SetMemoryLimit(int64(newLimit)) - logger.Info("GOMEMLIMIT is updated", slog.Uint64(envGOMEMLIMIT, newLimit), slog.Uint64("previous", currLimit)) + logger.Info("GOMEMLIMIT is updated", slog.Uint64(envGOMEMLIMIT, newLimit), slog.Uint64("previous", uint64(previous))) return newLimit, nil } -// refresh spawns a goroutine that runs every refresh duration and updates the GOMEMLIMIT if it has changed. -// See more details in the documentation of WithRefreshInterval. -func refresh(provider Provider, logger *slog.Logger, refresh time.Duration) { +func refresh(ctx context.Context, provider Provider, logger *slog.Logger, refresh time.Duration) { if refresh == 0 { return } - provider = noErrNoLimitProvider(provider) + ticker := time.NewTicker(refresh) + defer ticker.Stop() - t := time.NewTicker(refresh) - go func() { - for range t.C { - err := func() (_err error) { - snapshot := debug.SetMemoryLimit(-1) - defer rollbackOnPanic(logger, snapshot, &_err) - - _, err := updateGoMemLimit(uint64(snapshot), provider, logger) - if err != nil { - return err - } - - return nil - }() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + _, err := updateGoMemLimit(provider, logger) if err != nil { logger.Error("failed to refresh GOMEMLIMIT", slog.Any("error", err)) } } - }() -} - -// rollbackOnPanic rollbacks to the snapshot on panic. -// Since it uses recover, it should be called in a deferred function. -func rollbackOnPanic(logger *slog.Logger, snapshot int64, err *error) { - panicErr := recover() - if panicErr != nil { - if *err != nil { - logger.Error("failed to set GOMEMLIMIT", slog.Any("error", *err)) - } - *err = fmt.Errorf("panic during setting the Go's memory limit, rolling back to previous limit %d: %v", - snapshot, panicErr, - ) - debug.SetMemoryLimit(snapshot) - } -} - -// SetGoMemLimitWithEnv sets GOMEMLIMIT with the value from the environment variables. -// Since WithEnv is deprecated, this function is equivalent to SetGoMemLimitWithOpts(). -// Deprecated: use SetGoMemLimitWithOpts instead. -func SetGoMemLimitWithEnv() { - _, _ = SetGoMemLimitWithOpts() -} - -// SetGoMemLimit sets GOMEMLIMIT with the value from the cgroup's memory limit and given ratio. -func SetGoMemLimit(ratio float64) (int64, error) { - return SetGoMemLimitWithOpts(WithRatio(ratio)) -} - -// SetGoMemLimitWithProvider sets GOMEMLIMIT with the value from the given provider and ratio. -func SetGoMemLimitWithProvider(provider Provider, ratio float64) (int64, error) { - return SetGoMemLimitWithOpts(WithProvider(provider), WithRatio(ratio)) -} - -func noErrNoLimitProvider(provider Provider) Provider { - return func() (uint64, error) { - limit, err := provider() - if errors.Is(err, ErrNoLimit) { - return math.MaxInt64, nil - } - return limit, err } } -func capProvider(provider Provider) Provider { +func boundedProvider(provider Provider, minLimit int64) Provider { return func() (uint64, error) { limit, err := provider() if err != nil { return 0, err } else if limit > math.MaxInt64 { return math.MaxInt64, nil + } else if minLimit > 0 && limit < uint64(minLimit) { + return uint64(minLimit), nil } return limit, nil } diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/provider.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/provider.go index 4f83770d..70c730d1 100644 --- a/vendor/github.com/KimMachineGun/automemlimit/memlimit/provider.go +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/provider.go @@ -2,25 +2,28 @@ package memlimit import ( "fmt" + "math" ) -// Provider is a function that returns the memory limit. +// Provider returns a memory limit in bytes. +// It should return [ErrNoLimit] when the source reports no limit. type Provider func() (uint64, error) -// Limit is a helper Provider function that returns the given limit. -func Limit(limit uint64) func() (uint64, error) { +// Limit returns a [Provider] that always returns limit. +func Limit(limit uint64) Provider { return func() (uint64, error) { return limit, nil } } -// ApplyRationA is a helper Provider function that applies the given ratio to the given provider. +// ApplyRatio wraps provider and applies ratio to its result. +// The ratio must be in (0.0, 1.0]. func ApplyRatio(provider Provider, ratio float64) Provider { if ratio == 1 { return provider } return func() (uint64, error) { - if ratio <= 0 || ratio > 1 { + if math.IsNaN(ratio) || ratio <= 0 || ratio > 1 { return 0, fmt.Errorf("invalid ratio: %f, ratio should be in the range (0.0,1.0]", ratio) } limit, err := provider() @@ -31,7 +34,8 @@ func ApplyRatio(provider Provider, ratio float64) Provider { } } -// ApplyFallback is a helper Provider function that sets the fallback provider. +// ApplyFallback returns a [Provider] that calls fallback when provider returns an error, +// including [ErrNoLimit]. func ApplyFallback(provider Provider, fallback Provider) Provider { return func() (uint64, error) { limit, err := provider() diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/exp_system.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/system.go similarity index 77% rename from vendor/github.com/KimMachineGun/automemlimit/memlimit/exp_system.go rename to vendor/github.com/KimMachineGun/automemlimit/memlimit/system.go index dee95f52..9901fe81 100644 --- a/vendor/github.com/KimMachineGun/automemlimit/memlimit/exp_system.go +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/system.go @@ -4,7 +4,7 @@ import ( "github.com/pbnjay/memory" ) -// FromSystem returns the total memory of the system. +// FromSystem returns the total system memory. func FromSystem() (uint64, error) { limit := memory.TotalMemory() if limit == 0 { diff --git a/vendor/modules.txt b/vendor/modules.txt index b3251f0d..e0265f73 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -2,8 +2,8 @@ ## explicit; go 1.18 github.com/BurntSushi/toml github.com/BurntSushi/toml/internal -# github.com/KimMachineGun/automemlimit v0.7.5 -## explicit; go 1.22.0 +# github.com/KimMachineGun/automemlimit v1.0.0 +## explicit; go 1.21.0 github.com/KimMachineGun/automemlimit/memlimit # github.com/KyleBanks/depth v1.2.1 ## explicit