fix(fleet): page only host-global OOM and survive observer restart

.gitignore

@@ -10,6 +10,7 @@
 /gha-fleet
 /gha-fleet-gateway
 /gha-fleet-observer
+/gha-pressure-observer
 /coverage.out
 /*.prof
 *.test

CHANGELOG.md

@@ -2,6 +2,20 @@
 
 ## Unreleased
 
+- Count only host-global kernel OOM kills (`constraint=CONSTRAINT_NONE`) into
+  `gha_fleet_host_oom_kills_total`, and page `host_oom_detected` on the
+  in-window span of that counter instead of `increase()`. `/proc/vmstat
+  oom_kill` includes memory-cgroup kills, which closed the whole member when a
+  JVM hit its envelope; `increase()` treated an observer restart that
+  republished the boot total as a burst of new kills (observed 4 and 3 with no
+  dmesg). Kernel logs that cannot be read still fall back to vmstat so a
+  sandboxed observer does not fail the host snapshot. The provider compiles
+  the same hostprobe package, so this is `v0.1.5-nddev.122`.
+
+- Stamp `gha-pressure-observer` from `make build-controller` with the same
+  version and commit as the other host binaries. The live members were still
+  running an older unstamped identity because that target never produced it.
+
 - Advance every image identity changed by the SQLite development headers to a
   new immutable alias. Pull-request CI now rejects a manifest or provisioning
   change that keeps the old alias, before a live image reconcile has to detect

Makefile

@@ -2,10 +2,10 @@
 
 .PHONY: build build-controller build-garm-derivative fmt-check garm-derivative-script test test-race vet verify
 
-# The controller and observer that ship to the hosts. Stamped, because an
-# unstamped binary reports `version: dev, commit: unknown` -- which is what all
-# five hosts reported, since `build` compiles without ldflags and no other
-# target ever produced these. runProviderRelease's own comment already claimed
+# The controller, observer, cache broker and pressure observer that ship to
+# the hosts. Stamped, because an unstamped binary reports `version: dev,
+# commit: unknown` -- which is what the hosts reported whenever a target
+# compiled without ldflags. runProviderRelease's own comment already claimed
 # the Makefile stamped them; this is that claim made true.
 #
 # The version is the provider derivative version, read from the manifest rather
@@ -30,8 +30,10 @@ build-controller:
 	CGO_ENABLED=0 go build -trimpath -buildvcs=false -ldflags "$(CONTROLLER_LDFLAGS)" -o dist/gha-fleet ./cmd/gha-fleet
 	CGO_ENABLED=0 go build -trimpath -buildvcs=false -ldflags "$(CONTROLLER_LDFLAGS)" -o dist/gha-fleet-observer ./cmd/gha-fleet-observer
 	CGO_ENABLED=0 go build -trimpath -buildvcs=false -ldflags "$(CONTROLLER_LDFLAGS)" -o dist/gha-cache-broker ./cmd/gha-cache-broker
+	CGO_ENABLED=0 go build -trimpath -buildvcs=false -ldflags "$(CONTROLLER_LDFLAGS)" -o dist/gha-pressure-observer ./cmd/gha-pressure-observer
 	@./dist/gha-fleet version
 	@./dist/gha-cache-broker -version
+	@./dist/gha-pressure-observer --version
 
 garm-derivative-script:
 	go run ./cmd/gha-fleet render-garm-build

config/example-runner-1.yaml

@@ -9,7 +9,7 @@ control_plane:
   manager_version: v0.2.1-nddev.88
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.121
+  provider_version: v0.1.5-nddev.122
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-runner-2.yaml

@@ -9,7 +9,7 @@ control_plane:
   manager_version: v0.2.1-nddev.88
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.121
+  provider_version: v0.1.5-nddev.122
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-runner-3.yaml

@@ -9,7 +9,7 @@ control_plane:
   manager_version: v0.2.1-nddev.88
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.121
+  provider_version: v0.1.5-nddev.122
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-runner-4.yaml

@@ -9,7 +9,7 @@ control_plane:
   manager_version: v0.2.1-nddev.88
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.121
+  provider_version: v0.1.5-nddev.122
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-services.yaml

@@ -27,7 +27,7 @@ control_plane:
   manager_version: v0.2.1-nddev.88
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.121
+  provider_version: v0.1.5-nddev.122
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/observability-rules.yaml

@@ -307,13 +307,14 @@ rules:
     severity: page
     query_language: promql
     stream_name: gha_fleet_host_oom_kills_total
-    # PromQL increase extrapolates a counter to both range boundaries. A single
-    # exact kernel event therefore appeared in the notification as
-    # 1.0526315789473684. Preserve increase's reset handling, but report the
-    # physical event count rather than its sampling extrapolation. `round` is
-    # supported by the deployed OpenObserve engine and leaves the > 0 detector
-    # unchanged for integer counter increments.
-    expression: round(max by (host_name) (increase(gha_fleet_host_oom_kills_total[5m])))
+    # Do not use increase() on this boot-scoped counter. The pressure observer
+    # republishes the same total after a restart, OpenObserve starts a new
+    # series at that value, and increase() treats 0→N as N new kills. Measured
+    # 2026-09-04 01:36Z: observed 4 and 3 with no dmesg after the image-cutover
+    # observer restart. The in-window span is zero across a republish and one
+    # when the boot total actually moves. `round` still strips PromQL's
+    # boundary extrapolation so a single integer event does not page as 1.05.
+    expression: round(max by (host_name) (max_over_time(gha_fleet_host_oom_kills_total[5m]) - min_over_time(gha_fleet_host_oom_kills_total[5m])))
     operator: ">"
     threshold: 0
     evaluation_seconds: 30

config/provider-derivative.yaml

@@ -16,7 +16,7 @@ artifact: garm-provider-incus
 # state all move together, because all three derive from here. A provider change
 # that does not bump it ships under the previous version, which is exactly how
 # runner-1 and runner-2 diverged.
-derivative_version: v0.1.5-nddev.121
+derivative_version: v0.1.5-nddev.122
 
 # The external-provider protocol GARM speaks to this binary. It moves on its own
 # schedule -- a provider release does not imply an interface release -- so it is
@@ -37,8 +37,8 @@ runtime:
   queue_intent_schema_version: 6
 
 build:
-  source_commit: 53ec735cd97727e5ba6175551b07e7c5621a533c
-  binary_sha256: b0d1a6980bb7efdd112acaed141d01708eeed9bb211770edc965cc033ff322a7
+  source_commit: 40a8402d517fd8cdc1edb04ec27ec524a4546be1
+  binary_sha256: 8997d4f18550bdb94805cfe981390fb4ab2104460bb513f6c95017fac8b1a38b
   go_version: go1.26.7
   cgo_enabled: false
   target_os: linux

docs/runbooks/fleet-alerts.md

@@ -69,7 +69,11 @@ gha-fleet reconcile-openobserve-alerts \
 - Collector queue pages: preserve the queue directory, restore the private
   OpenObserve route/backend, and verify queue drain plus exact record recovery.
 - OOM or pressure pages: close admission; never stop an already running worker
-  merely to make utilization look healthy.
+  merely to make utilization look healthy. `host_oom_detected` is host-global
+  `CONSTRAINT_NONE` only. Memory-cgroup kills are the envelope working; they
+  must not close the member. The detector is the in-window span of
+  `gha_fleet_host_oom_kills_total`, not `increase()`, because an observer
+  restart republishes the boot total and `increase()` treats that as a burst.
 - Host-signal tickets: use `gha_fleet_host_signal_events` cumulative deltas.
   LVM activation and overlay `xino=off` are workload-volume context; audit
   suppression and workqueue-hog alerts act only on their bounded burst budget.

internal/fleetobserve/metrics.go

@@ -34,7 +34,7 @@ func RenderPrometheus(snapshot Snapshot, now time.Time, maxStaleness time.Durati
 	gauge(&output, "gha_fleet_host_load1", "One-minute host load average.", snapshot.Host.CPU.Load1)
 	gauge(&output, "gha_fleet_host_memory_total_bytes", "Total host memory in bytes.", float64(snapshot.Host.Memory.TotalMiB)*1024*1024)
 	gauge(&output, "gha_fleet_host_memory_available_bytes", "Available host memory in bytes.", float64(snapshot.Host.Memory.AvailableMiB)*1024*1024)
-	counter(&output, "gha_fleet_host_oom_kills_total", "Kernel OOM kills observed since host boot.", float64(snapshot.Host.Memory.OOMKillsTotal))
+	counter(&output, "gha_fleet_host_oom_kills_total", "Host-global kernel OOM kills since boot (constraint=CONSTRAINT_NONE), excluding memory-cgroup kills.", float64(snapshot.Host.Memory.OOMKillsTotal))
 	output.WriteString(psimetrics.Render(snapshot.Host.Pressure))
 	gauge(&output, "gha_fleet_host_root_available_bytes", "Available bytes on the root filesystem.", float64(snapshot.Host.RootFilesystem.AvailableMiB)*1024*1024)
 	gauge(&output, "gha_fleet_host_root_free_percent", "Free block percentage of host-usable root space, excluding the Incus loop-backed pool file.", float64(snapshot.Host.RootFilesystem.FreePercent))

internal/hostprobe/collect_linux.go

@@ -2,6 +2,7 @@ package hostprobe
 
 import (
 	"bufio"
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -58,9 +59,9 @@ func collect(
 	if err != nil {
 		return Snapshot{}, fmt.Errorf("read meminfo: %w", err)
 	}
-	memory.OOMKillsTotal, err = parseOOMKills(filepath.Join(root, "proc", "vmstat"))
+	memory.OOMKillsTotal, err = observeOOMKills(ctx, root, runner)
 	if err != nil {
-		return Snapshot{}, fmt.Errorf("read vmstat OOM counter: %w", err)
+		return Snapshot{}, fmt.Errorf("read host-global OOM counter: %w", err)
 	}
 	pressure, err := parsePressure(filepath.Join(root, "proc", "pressure"))
 	if err != nil {
@@ -164,6 +165,68 @@ func parseOOMKills(path string) (uint64, error) {
 	return 0, nil
 }
 
+const hostGlobalOOMMarker = "constraint=CONSTRAINT_NONE"
+
+func observeOOMKills(ctx context.Context, root string, runner CommandRunner) (uint64, error) {
+	kmsgPath := filepath.Join(root, "dev", "kmsg")
+	vmstatPath := filepath.Join(root, "proc", "vmstat")
+	if root != "/" {
+		if _, err := os.Stat(kmsgPath); err == nil {
+			return parseHostGlobalOOMKills(kmsgPath)
+		}
+		return parseOOMKills(vmstatPath)
+	}
+	count, err := countJournalHostGlobalOOM(ctx, runner)
+	if err == nil {
+		return count, nil
+	}
+	if n, kmsgErr := parseHostGlobalOOMKills(kmsgPath); kmsgErr == nil {
+		return n, nil
+	}
+	return parseOOMKills(vmstatPath)
+}
+
+func countJournalHostGlobalOOM(ctx context.Context, runner CommandRunner) (uint64, error) {
+	out, err := runner.Run(ctx, "journalctl", "-k", "-b", "--grep="+hostGlobalOOMMarker, "--no-pager", "-o", "cat", "-q")
+	if err != nil {
+		var ee *exec.ExitError
+		if errors.As(err, &ee) && ee.ExitCode() == 1 && len(bytes.TrimSpace(out)) == 0 {
+			return 0, nil
+		}
+		return 0, err
+	}
+	return countHostGlobalOOMLines(out), nil
+}
+
+func parseHostGlobalOOMKills(path string) (uint64, error) {
+	file, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NONBLOCK, 0)
+	if err != nil {
+		return 0, err
+	}
+	defer file.Close()
+	var count uint64
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		if bytes.Contains(scanner.Bytes(), []byte(hostGlobalOOMMarker)) {
+			count++
+		}
+	}
+	if scanErr := scanner.Err(); scanErr != nil && !errors.Is(scanErr, syscall.EAGAIN) && !errors.Is(scanErr, syscall.EWOULDBLOCK) {
+		return 0, scanErr
+	}
+	return count, nil
+}
+
+func countHostGlobalOOMLines(body []byte) uint64 {
+	var count uint64
+	for _, line := range bytes.Split(body, []byte("\n")) {
+		if bytes.Contains(line, []byte(hostGlobalOOMMarker)) {
+			count++
+		}
+	}
+	return count
+}
+
 // ReadPressure observes Linux pressure-stall information under root without
 // running the rest of the host preflight. The compute-member observer needs
 // exactly this and nothing else: it must not shell out, inspect services or

internal/hostprobe/hostprobe_test.go

@@ -1,8 +1,10 @@
 package hostprobe
 
 import (
+	"context"
 	"errors"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"testing"
 
@@ -81,6 +83,86 @@ func TestParsePressureAndOOMCounters(t *testing.T) {
 	}
 }
 
+func TestCountHostGlobalOOMLinesIgnoresMemoryCgroupKills(t *testing.T) {
+	t.Parallel()
+	body := []byte("" +
+		"6,1,0,-;Out of memory: Killed process 100 (java)\n" +
+		"6,2,0,-;oom-kill:constraint=CONSTRAINT_MEMCG,nodemask=(null),cpuset=/,mems_allowed=0,task_memcg=/incus.gha-fleet/1\n" +
+		"6,3,0,-;Memory cgroup out of memory: Killed process 100 (java)\n" +
+		"6,4,0,-;oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0,global_oom,task_memcg=/\n" +
+		"6,5,0,-;Out of memory: Killed process 200 (java)\n" +
+		"Sep  4 01:16:00 gha-runner-1 kernel: oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),global_oom\n")
+	if got := countHostGlobalOOMLines(body); got != 2 {
+		t.Fatalf("host-global oom lines=%d, want 2", got)
+	}
+}
+
+func TestObserveOOMKillsPrefersKmsgHostGlobalOverVmstat(t *testing.T) {
+	t.Parallel()
+	root := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(root, "dev"), 0o700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.MkdirAll(filepath.Join(root, "proc"), 0o700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(root, "proc", "vmstat"), []byte("oom_kill 7\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	kmsg := []byte("oom-kill:constraint=CONSTRAINT_MEMCG,task_memcg=/incus\n" +
+		"oom-kill:constraint=CONSTRAINT_NONE,global_oom\n")
+	if err := os.WriteFile(filepath.Join(root, "dev", "kmsg"), kmsg, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	got, err := observeOOMKills(t.Context(), root, stubCommandRunner{})
+	if err != nil || got != 1 {
+		t.Fatalf("observeOOMKills=%d err=%v, want 1 host-global kill", got, err)
+	}
+}
+
+func TestObserveOOMKillsFallsBackToVmstatWithoutKmsg(t *testing.T) {
+	t.Parallel()
+	root := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(root, "proc"), 0o700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(root, "proc", "vmstat"), []byte("oom_kill 7\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	got, err := observeOOMKills(t.Context(), root, stubCommandRunner{})
+	if err != nil || got != 7 {
+		t.Fatalf("observeOOMKills=%d err=%v, want vmstat 7", got, err)
+	}
+}
+
+func TestCountJournalHostGlobalOOMTreatsEmptyExitOneAsZero(t *testing.T) {
+	t.Parallel()
+	got, err := countJournalHostGlobalOOM(t.Context(), stubCommandRunner{err: exec.Command("sh", "-c", "exit 1").Run()})
+	if err != nil || got != 0 {
+		t.Fatalf("empty journal grep got=%d err=%v, want 0 nil", got, err)
+	}
+}
+
+func TestCountJournalHostGlobalOOMDoesNotTreatPermissionNoiseAsZero(t *testing.T) {
+	t.Parallel()
+	_, err := countJournalHostGlobalOOM(t.Context(), stubCommandRunner{
+		out: []byte("No journal files were opened due to insufficient permissions.\n"),
+		err: exec.Command("sh", "-c", "exit 1").Run(),
+	})
+	if err == nil {
+		t.Fatal("permission failure must fall back, not report zero host-global kills")
+	}
+}
+
+type stubCommandRunner struct {
+	out []byte
+	err error
+}
+
+func (s stubCommandRunner) Run(context.Context, string, ...string) ([]byte, error) {
+	return s.out, s.err
+}
+
 func TestParsePressureRejectsMalformedAvailableData(t *testing.T) {
 	t.Parallel()
 	directory := t.TempDir()

internal/hostprobe/types.go

@@ -39,10 +39,13 @@ type CPU struct {
 }
 
 type Memory struct {
-	TotalMiB      int    `json:"total_mib"`
-	AvailableMiB  int    `json:"available_mib"`
-	SwapTotalMiB  int    `json:"swap_total_mib"`
-	SwapFreeMiB   int    `json:"swap_free_mib"`
+	TotalMiB     int `json:"total_mib"`
+	AvailableMiB int `json:"available_mib"`
+	SwapTotalMiB int `json:"swap_total_mib"`
+	SwapFreeMiB  int `json:"swap_free_mib"`
+	// Host-global kernel OOM kills since boot (`constraint=CONSTRAINT_NONE`).
+	// Memory-cgroup kills are excluded: they are the admission envelope working.
+	// When kernel logs are unreadable the value falls back to vmstat oom_kill.
 	OOMKillsTotal uint64 `json:"oom_kills_total"`
 }
 

internal/observabilityrules/rules_test.go

@@ -34,7 +34,7 @@ func TestRepositoryRulesUseCurrentMetricSemantics(t *testing.T) {
 		"audit_suppression_burst":           `signal_class="audit_suppressed"`,
 		"kernel_workqueue_hog":              `signal_class="kernel_workqueue_hog"`,
 		"host_compliance_observer_missing":  "gha_fleet_host_compliance_observer_up",
-		"host_oom_detected":                 "round(max by (host_name) (increase(gha_fleet_host_oom_kills_total[5m])))",
+		"host_oom_detected":                 "round(max by (host_name) (max_over_time(gha_fleet_host_oom_kills_total[5m]) - min_over_time(gha_fleet_host_oom_kills_total[5m])))",
 		"host_package_inventory_stale":      "gha_fleet_host_package_inventory_age_seconds",
 		"host_reboot_required":              "gha_fleet_host_reboot_required",
 		"host_standard_updates_available":   "gha_fleet_host_standard_updates_available",

internal/pressureobserve/observe.go

@@ -116,7 +116,7 @@ func Render(state pressuregate.State, now time.Time, maxStaleness time.Duration,
 	}
 	gauge("gha_fleet_pressure_observer_up", "Whether the local pressure state is valid and fresh.", boolFloat(fresh))
 	gauge("gha_fleet_pressure_sample_age_seconds", "Age of the local pressure publication, or -1 when unavailable.", age)
-	counter("gha_fleet_host_oom_kills_total", "Kernel OOM kills observed since host boot.", state.OOMKillsTotal)
+	counter("gha_fleet_host_oom_kills_total", "Host-global kernel OOM kills since boot (constraint=CONSTRAINT_NONE), excluding memory-cgroup kills.", state.OOMKillsTotal)
 	gauge("gha_fleet_host_pressure_open", "Whether local pressure admission is open and fresh.", boolFloat(fresh && state.State == pressuregate.StateOpen))
 	// The gate records why it is in its current state and that reason was
 	// readable only by opening a JSON file on the host. When admission closes