diff --git a/.gitignore b/.gitignore index c2a31bc9..e972cac3 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ /gha-fleet /gha-fleet-gateway /gha-fleet-observer +/gha-pressure-observer /coverage.out /*.prof *.test diff --git a/CHANGELOG.md b/CHANGELOG.md index bb770b1e..999891e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ ## Unreleased +- Count only host-global kernel OOM kills (`constraint=CONSTRAINT_NONE`) into + `gha_fleet_host_oom_kills_total`, and page `host_oom_detected` on the + in-window span of that counter instead of `increase()`. `/proc/vmstat + oom_kill` includes memory-cgroup kills, which closed the whole member when a + JVM hit its envelope; `increase()` treated an observer restart that + republished the boot total as a burst of new kills (observed 4 and 3 with no + dmesg). Kernel logs that cannot be read still fall back to vmstat so a + sandboxed observer does not fail the host snapshot. The provider compiles + the same hostprobe package, so this is `v0.1.5-nddev.122`. + +- Stamp `gha-pressure-observer` from `make build-controller` with the same + version and commit as the other host binaries. The live members were still + running an older unstamped identity because that target never produced it. + - Advance every image identity changed by the SQLite development headers to a new immutable alias. Pull-request CI now rejects a manifest or provisioning change that keeps the old alias, before a live image reconcile has to detect diff --git a/Makefile b/Makefile index 4fd32f03..2ec404ee 100644 --- a/Makefile +++ b/Makefile @@ -2,10 +2,10 @@ .PHONY: build build-controller build-garm-derivative fmt-check garm-derivative-script test test-race vet verify -# The controller and observer that ship to the hosts. Stamped, because an -# unstamped binary reports `version: dev, commit: unknown` -- which is what all -# five hosts reported, since `build` compiles without ldflags and no other -# target ever produced these. runProviderRelease's own comment already claimed +# The controller, observer, cache broker and pressure observer that ship to +# the hosts. Stamped, because an unstamped binary reports `version: dev, +# commit: unknown` -- which is what the hosts reported whenever a target +# compiled without ldflags. runProviderRelease's own comment already claimed # the Makefile stamped them; this is that claim made true. # # The version is the provider derivative version, read from the manifest rather @@ -30,8 +30,10 @@ build-controller: CGO_ENABLED=0 go build -trimpath -buildvcs=false -ldflags "$(CONTROLLER_LDFLAGS)" -o dist/gha-fleet ./cmd/gha-fleet CGO_ENABLED=0 go build -trimpath -buildvcs=false -ldflags "$(CONTROLLER_LDFLAGS)" -o dist/gha-fleet-observer ./cmd/gha-fleet-observer CGO_ENABLED=0 go build -trimpath -buildvcs=false -ldflags "$(CONTROLLER_LDFLAGS)" -o dist/gha-cache-broker ./cmd/gha-cache-broker + CGO_ENABLED=0 go build -trimpath -buildvcs=false -ldflags "$(CONTROLLER_LDFLAGS)" -o dist/gha-pressure-observer ./cmd/gha-pressure-observer @./dist/gha-fleet version @./dist/gha-cache-broker -version + @./dist/gha-pressure-observer --version garm-derivative-script: go run ./cmd/gha-fleet render-garm-build diff --git a/config/example-runner-1.yaml b/config/example-runner-1.yaml index f466fed8..6cec1c41 100644 --- a/config/example-runner-1.yaml +++ b/config/example-runner-1.yaml @@ -9,7 +9,7 @@ control_plane: manager_version: v0.2.1-nddev.88 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.121 + provider_version: v0.1.5-nddev.122 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/example-runner-2.yaml b/config/example-runner-2.yaml index b49dc12f..2266fe0c 100644 --- a/config/example-runner-2.yaml +++ b/config/example-runner-2.yaml @@ -9,7 +9,7 @@ control_plane: manager_version: v0.2.1-nddev.88 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.121 + provider_version: v0.1.5-nddev.122 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/example-runner-3.yaml b/config/example-runner-3.yaml index 6cd0e37d..5c18ed1e 100644 --- a/config/example-runner-3.yaml +++ b/config/example-runner-3.yaml @@ -9,7 +9,7 @@ control_plane: manager_version: v0.2.1-nddev.88 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.121 + provider_version: v0.1.5-nddev.122 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/example-runner-4.yaml b/config/example-runner-4.yaml index b1de833a..6fc47859 100644 --- a/config/example-runner-4.yaml +++ b/config/example-runner-4.yaml @@ -9,7 +9,7 @@ control_plane: manager_version: v0.2.1-nddev.88 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.121 + provider_version: v0.1.5-nddev.122 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/example-services.yaml b/config/example-services.yaml index 584a7966..f9be58d6 100644 --- a/config/example-services.yaml +++ b/config/example-services.yaml @@ -27,7 +27,7 @@ control_plane: manager_version: v0.2.1-nddev.88 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.121 + provider_version: v0.1.5-nddev.122 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/observability-rules.yaml b/config/observability-rules.yaml index 3373071a..9e83d4d3 100644 --- a/config/observability-rules.yaml +++ b/config/observability-rules.yaml @@ -307,13 +307,14 @@ rules: severity: page query_language: promql stream_name: gha_fleet_host_oom_kills_total - # PromQL increase extrapolates a counter to both range boundaries. A single - # exact kernel event therefore appeared in the notification as - # 1.0526315789473684. Preserve increase's reset handling, but report the - # physical event count rather than its sampling extrapolation. `round` is - # supported by the deployed OpenObserve engine and leaves the > 0 detector - # unchanged for integer counter increments. - expression: round(max by (host_name) (increase(gha_fleet_host_oom_kills_total[5m]))) + # Do not use increase() on this boot-scoped counter. The pressure observer + # republishes the same total after a restart, OpenObserve starts a new + # series at that value, and increase() treats 0→N as N new kills. Measured + # 2026-09-04 01:36Z: observed 4 and 3 with no dmesg after the image-cutover + # observer restart. The in-window span is zero across a republish and one + # when the boot total actually moves. `round` still strips PromQL's + # boundary extrapolation so a single integer event does not page as 1.05. + expression: round(max by (host_name) (max_over_time(gha_fleet_host_oom_kills_total[5m]) - min_over_time(gha_fleet_host_oom_kills_total[5m]))) operator: ">" threshold: 0 evaluation_seconds: 30 diff --git a/config/provider-derivative.yaml b/config/provider-derivative.yaml index 8c4035a1..99fc81f4 100644 --- a/config/provider-derivative.yaml +++ b/config/provider-derivative.yaml @@ -16,7 +16,7 @@ artifact: garm-provider-incus # state all move together, because all three derive from here. A provider change # that does not bump it ships under the previous version, which is exactly how # runner-1 and runner-2 diverged. -derivative_version: v0.1.5-nddev.121 +derivative_version: v0.1.5-nddev.122 # The external-provider protocol GARM speaks to this binary. It moves on its own # schedule -- a provider release does not imply an interface release -- so it is @@ -37,8 +37,8 @@ runtime: queue_intent_schema_version: 6 build: - source_commit: 53ec735cd97727e5ba6175551b07e7c5621a533c - binary_sha256: b0d1a6980bb7efdd112acaed141d01708eeed9bb211770edc965cc033ff322a7 + source_commit: 40a8402d517fd8cdc1edb04ec27ec524a4546be1 + binary_sha256: 8997d4f18550bdb94805cfe981390fb4ab2104460bb513f6c95017fac8b1a38b go_version: go1.26.7 cgo_enabled: false target_os: linux diff --git a/docs/runbooks/fleet-alerts.md b/docs/runbooks/fleet-alerts.md index 3815b339..968feb61 100644 --- a/docs/runbooks/fleet-alerts.md +++ b/docs/runbooks/fleet-alerts.md @@ -69,7 +69,11 @@ gha-fleet reconcile-openobserve-alerts \ - Collector queue pages: preserve the queue directory, restore the private OpenObserve route/backend, and verify queue drain plus exact record recovery. - OOM or pressure pages: close admission; never stop an already running worker - merely to make utilization look healthy. + merely to make utilization look healthy. `host_oom_detected` is host-global + `CONSTRAINT_NONE` only. Memory-cgroup kills are the envelope working; they + must not close the member. The detector is the in-window span of + `gha_fleet_host_oom_kills_total`, not `increase()`, because an observer + restart republishes the boot total and `increase()` treats that as a burst. - Host-signal tickets: use `gha_fleet_host_signal_events` cumulative deltas. LVM activation and overlay `xino=off` are workload-volume context; audit suppression and workqueue-hog alerts act only on their bounded burst budget. diff --git a/internal/fleetobserve/metrics.go b/internal/fleetobserve/metrics.go index c9c2dc8c..1de4c9de 100644 --- a/internal/fleetobserve/metrics.go +++ b/internal/fleetobserve/metrics.go @@ -34,7 +34,7 @@ func RenderPrometheus(snapshot Snapshot, now time.Time, maxStaleness time.Durati gauge(&output, "gha_fleet_host_load1", "One-minute host load average.", snapshot.Host.CPU.Load1) gauge(&output, "gha_fleet_host_memory_total_bytes", "Total host memory in bytes.", float64(snapshot.Host.Memory.TotalMiB)*1024*1024) gauge(&output, "gha_fleet_host_memory_available_bytes", "Available host memory in bytes.", float64(snapshot.Host.Memory.AvailableMiB)*1024*1024) - counter(&output, "gha_fleet_host_oom_kills_total", "Kernel OOM kills observed since host boot.", float64(snapshot.Host.Memory.OOMKillsTotal)) + counter(&output, "gha_fleet_host_oom_kills_total", "Host-global kernel OOM kills since boot (constraint=CONSTRAINT_NONE), excluding memory-cgroup kills.", float64(snapshot.Host.Memory.OOMKillsTotal)) output.WriteString(psimetrics.Render(snapshot.Host.Pressure)) gauge(&output, "gha_fleet_host_root_available_bytes", "Available bytes on the root filesystem.", float64(snapshot.Host.RootFilesystem.AvailableMiB)*1024*1024) gauge(&output, "gha_fleet_host_root_free_percent", "Free block percentage of host-usable root space, excluding the Incus loop-backed pool file.", float64(snapshot.Host.RootFilesystem.FreePercent)) diff --git a/internal/hostprobe/collect_linux.go b/internal/hostprobe/collect_linux.go index 457a14ca..2b58814a 100644 --- a/internal/hostprobe/collect_linux.go +++ b/internal/hostprobe/collect_linux.go @@ -2,6 +2,7 @@ package hostprobe import ( "bufio" + "bytes" "context" "errors" "fmt" @@ -58,9 +59,9 @@ func collect( if err != nil { return Snapshot{}, fmt.Errorf("read meminfo: %w", err) } - memory.OOMKillsTotal, err = parseOOMKills(filepath.Join(root, "proc", "vmstat")) + memory.OOMKillsTotal, err = observeOOMKills(ctx, root, runner) if err != nil { - return Snapshot{}, fmt.Errorf("read vmstat OOM counter: %w", err) + return Snapshot{}, fmt.Errorf("read host-global OOM counter: %w", err) } pressure, err := parsePressure(filepath.Join(root, "proc", "pressure")) if err != nil { @@ -164,6 +165,68 @@ func parseOOMKills(path string) (uint64, error) { return 0, nil } +const hostGlobalOOMMarker = "constraint=CONSTRAINT_NONE" + +func observeOOMKills(ctx context.Context, root string, runner CommandRunner) (uint64, error) { + kmsgPath := filepath.Join(root, "dev", "kmsg") + vmstatPath := filepath.Join(root, "proc", "vmstat") + if root != "/" { + if _, err := os.Stat(kmsgPath); err == nil { + return parseHostGlobalOOMKills(kmsgPath) + } + return parseOOMKills(vmstatPath) + } + count, err := countJournalHostGlobalOOM(ctx, runner) + if err == nil { + return count, nil + } + if n, kmsgErr := parseHostGlobalOOMKills(kmsgPath); kmsgErr == nil { + return n, nil + } + return parseOOMKills(vmstatPath) +} + +func countJournalHostGlobalOOM(ctx context.Context, runner CommandRunner) (uint64, error) { + out, err := runner.Run(ctx, "journalctl", "-k", "-b", "--grep="+hostGlobalOOMMarker, "--no-pager", "-o", "cat", "-q") + if err != nil { + var ee *exec.ExitError + if errors.As(err, &ee) && ee.ExitCode() == 1 && len(bytes.TrimSpace(out)) == 0 { + return 0, nil + } + return 0, err + } + return countHostGlobalOOMLines(out), nil +} + +func parseHostGlobalOOMKills(path string) (uint64, error) { + file, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NONBLOCK, 0) + if err != nil { + return 0, err + } + defer file.Close() + var count uint64 + scanner := bufio.NewScanner(file) + for scanner.Scan() { + if bytes.Contains(scanner.Bytes(), []byte(hostGlobalOOMMarker)) { + count++ + } + } + if scanErr := scanner.Err(); scanErr != nil && !errors.Is(scanErr, syscall.EAGAIN) && !errors.Is(scanErr, syscall.EWOULDBLOCK) { + return 0, scanErr + } + return count, nil +} + +func countHostGlobalOOMLines(body []byte) uint64 { + var count uint64 + for _, line := range bytes.Split(body, []byte("\n")) { + if bytes.Contains(line, []byte(hostGlobalOOMMarker)) { + count++ + } + } + return count +} + // ReadPressure observes Linux pressure-stall information under root without // running the rest of the host preflight. The compute-member observer needs // exactly this and nothing else: it must not shell out, inspect services or diff --git a/internal/hostprobe/hostprobe_test.go b/internal/hostprobe/hostprobe_test.go index 5956b034..db63fab2 100644 --- a/internal/hostprobe/hostprobe_test.go +++ b/internal/hostprobe/hostprobe_test.go @@ -1,8 +1,10 @@ package hostprobe import ( + "context" "errors" "os" + "os/exec" "path/filepath" "testing" @@ -81,6 +83,86 @@ func TestParsePressureAndOOMCounters(t *testing.T) { } } +func TestCountHostGlobalOOMLinesIgnoresMemoryCgroupKills(t *testing.T) { + t.Parallel() + body := []byte("" + + "6,1,0,-;Out of memory: Killed process 100 (java)\n" + + "6,2,0,-;oom-kill:constraint=CONSTRAINT_MEMCG,nodemask=(null),cpuset=/,mems_allowed=0,task_memcg=/incus.gha-fleet/1\n" + + "6,3,0,-;Memory cgroup out of memory: Killed process 100 (java)\n" + + "6,4,0,-;oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0,global_oom,task_memcg=/\n" + + "6,5,0,-;Out of memory: Killed process 200 (java)\n" + + "Sep 4 01:16:00 gha-runner-1 kernel: oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),global_oom\n") + if got := countHostGlobalOOMLines(body); got != 2 { + t.Fatalf("host-global oom lines=%d, want 2", got) + } +} + +func TestObserveOOMKillsPrefersKmsgHostGlobalOverVmstat(t *testing.T) { + t.Parallel() + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "dev"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "proc"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "proc", "vmstat"), []byte("oom_kill 7\n"), 0o600); err != nil { + t.Fatal(err) + } + kmsg := []byte("oom-kill:constraint=CONSTRAINT_MEMCG,task_memcg=/incus\n" + + "oom-kill:constraint=CONSTRAINT_NONE,global_oom\n") + if err := os.WriteFile(filepath.Join(root, "dev", "kmsg"), kmsg, 0o600); err != nil { + t.Fatal(err) + } + got, err := observeOOMKills(t.Context(), root, stubCommandRunner{}) + if err != nil || got != 1 { + t.Fatalf("observeOOMKills=%d err=%v, want 1 host-global kill", got, err) + } +} + +func TestObserveOOMKillsFallsBackToVmstatWithoutKmsg(t *testing.T) { + t.Parallel() + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "proc"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "proc", "vmstat"), []byte("oom_kill 7\n"), 0o600); err != nil { + t.Fatal(err) + } + got, err := observeOOMKills(t.Context(), root, stubCommandRunner{}) + if err != nil || got != 7 { + t.Fatalf("observeOOMKills=%d err=%v, want vmstat 7", got, err) + } +} + +func TestCountJournalHostGlobalOOMTreatsEmptyExitOneAsZero(t *testing.T) { + t.Parallel() + got, err := countJournalHostGlobalOOM(t.Context(), stubCommandRunner{err: exec.Command("sh", "-c", "exit 1").Run()}) + if err != nil || got != 0 { + t.Fatalf("empty journal grep got=%d err=%v, want 0 nil", got, err) + } +} + +func TestCountJournalHostGlobalOOMDoesNotTreatPermissionNoiseAsZero(t *testing.T) { + t.Parallel() + _, err := countJournalHostGlobalOOM(t.Context(), stubCommandRunner{ + out: []byte("No journal files were opened due to insufficient permissions.\n"), + err: exec.Command("sh", "-c", "exit 1").Run(), + }) + if err == nil { + t.Fatal("permission failure must fall back, not report zero host-global kills") + } +} + +type stubCommandRunner struct { + out []byte + err error +} + +func (s stubCommandRunner) Run(context.Context, string, ...string) ([]byte, error) { + return s.out, s.err +} + func TestParsePressureRejectsMalformedAvailableData(t *testing.T) { t.Parallel() directory := t.TempDir() diff --git a/internal/hostprobe/types.go b/internal/hostprobe/types.go index 4f3ec521..f6ebcd3a 100644 --- a/internal/hostprobe/types.go +++ b/internal/hostprobe/types.go @@ -39,10 +39,13 @@ type CPU struct { } type Memory struct { - TotalMiB int `json:"total_mib"` - AvailableMiB int `json:"available_mib"` - SwapTotalMiB int `json:"swap_total_mib"` - SwapFreeMiB int `json:"swap_free_mib"` + TotalMiB int `json:"total_mib"` + AvailableMiB int `json:"available_mib"` + SwapTotalMiB int `json:"swap_total_mib"` + SwapFreeMiB int `json:"swap_free_mib"` + // Host-global kernel OOM kills since boot (`constraint=CONSTRAINT_NONE`). + // Memory-cgroup kills are excluded: they are the admission envelope working. + // When kernel logs are unreadable the value falls back to vmstat oom_kill. OOMKillsTotal uint64 `json:"oom_kills_total"` } diff --git a/internal/observabilityrules/rules_test.go b/internal/observabilityrules/rules_test.go index 54f33772..b138b7ce 100644 --- a/internal/observabilityrules/rules_test.go +++ b/internal/observabilityrules/rules_test.go @@ -34,7 +34,7 @@ func TestRepositoryRulesUseCurrentMetricSemantics(t *testing.T) { "audit_suppression_burst": `signal_class="audit_suppressed"`, "kernel_workqueue_hog": `signal_class="kernel_workqueue_hog"`, "host_compliance_observer_missing": "gha_fleet_host_compliance_observer_up", - "host_oom_detected": "round(max by (host_name) (increase(gha_fleet_host_oom_kills_total[5m])))", + "host_oom_detected": "round(max by (host_name) (max_over_time(gha_fleet_host_oom_kills_total[5m]) - min_over_time(gha_fleet_host_oom_kills_total[5m])))", "host_package_inventory_stale": "gha_fleet_host_package_inventory_age_seconds", "host_reboot_required": "gha_fleet_host_reboot_required", "host_standard_updates_available": "gha_fleet_host_standard_updates_available", diff --git a/internal/pressureobserve/observe.go b/internal/pressureobserve/observe.go index 5fa399ca..f7236571 100644 --- a/internal/pressureobserve/observe.go +++ b/internal/pressureobserve/observe.go @@ -116,7 +116,7 @@ func Render(state pressuregate.State, now time.Time, maxStaleness time.Duration, } gauge("gha_fleet_pressure_observer_up", "Whether the local pressure state is valid and fresh.", boolFloat(fresh)) gauge("gha_fleet_pressure_sample_age_seconds", "Age of the local pressure publication, or -1 when unavailable.", age) - counter("gha_fleet_host_oom_kills_total", "Kernel OOM kills observed since host boot.", state.OOMKillsTotal) + counter("gha_fleet_host_oom_kills_total", "Host-global kernel OOM kills since boot (constraint=CONSTRAINT_NONE), excluding memory-cgroup kills.", state.OOMKillsTotal) gauge("gha_fleet_host_pressure_open", "Whether local pressure admission is open and fresh.", boolFloat(fresh && state.State == pressuregate.StateOpen)) // The gate records why it is in its current state and that reason was // readable only by opening a JSON file on the host. When admission closes