Publish usable root free percent from each pressure observer

CHANGELOG.md

@@ -2,13 +2,14 @@
 
 ## Unreleased
 
-- Compute `compute_root_disk_low` from usable root space, excluding the Incus
-  loop-backed thin-pool file. With `discards=nopassdown` that file stays fully
-  allocated (~200 GiB on a 309 GiB root) while pool `data_percent` is ~20%, so
-  the OTel filesystem ratio fired at 17–18% free of the whole disk. Placement
-  already reads the thin pool; the observer metric and ticket now match that
-  fact. Provider admission still uses raw statvfs until the next provider
-  derivative, because hostprobe is part of that artifact.
+- Publish `gha_fleet_host_root_free_percent` from each compute member's
+  pressure observer, excluding the Incus loop-backed thin-pool file. The
+  central fleet observer runs only on gha-services, so switching the ticket
+  to that family without this series would have watched the control-plane
+  host and gone blind to the four members whose 200 GiB loop file is the
+  whole problem. Clustered admission already reads Incus pool free percent;
+  the OTel filesystem ratio remains the live ticket until these series exist
+  on every member.
 
 - Treat the authenticated job-start claim as the only repository identity for
   cache delivery and teardown diagnostics. An organization scale-set runner can

config/observability-rules.yaml

@@ -151,7 +151,9 @@ rules:
     # with discards=nopassdown it stays fully allocated while pool data_percent
     # is ~20%. The ticket then fired at 17–18% free of the whole root, which
     # is the loop file, not tenant data. Placement already reads the thin pool.
-    # Admission and this rule use the observer percent that excludes the loop.
+    # Clustered admission uses Incus pool free percent. This rule reads
+    # gha_fleet_host_root_free_percent from each member's pressure observer
+    # (compute hosts do not run the central fleet observer).
     expression: min by (host_name) (gha_fleet_host_root_free_percent)
     operator: "<"
     threshold: 20

internal/hostdisk/hostdisk.go

@@ -4,6 +4,8 @@
 // Placement already reads the thin pool. The OTel host filesystem series and
 // raw statvfs count that file as ordinary used space, so compute_root_disk_low
 // fired at 17–18% free of a 309 GiB root while pool data_percent was ~20%.
+// Compute members publish the result from the pressure observer; the central
+// fleet observer only sees gha-services.
 package hostdisk
 
 import (
@@ -41,6 +43,22 @@ func UsableFreePercent(total, available, loopAllocated uint64) int {
 	return percent(available, usableTotal)
 }
 
+// Observe is the live usable free percent of root, excluding loop-backed
+// Incus pool files on the same device. A stat error is returned so the
+// publisher can fail closed instead of omitting the series.
+func Observe(root string) (int, error) {
+	if root == "" {
+		root = "/"
+	}
+	var stats syscall.Statfs_t
+	if err := syscall.Statfs(root, &stats); err != nil {
+		return 0, err
+	}
+	total := uint64(stats.Blocks) * uint64(stats.Bsize)
+	available := uint64(stats.Bavail) * uint64(stats.Bsize)
+	return UsableFreePercent(total, available, LoopBackingAllocatedOn(root)), nil
+}
+
 // LoopBackingAllocatedOn sums allocated bytes of Incus pool images on the
 // same device as root. Sparse holes are not counted (st_blocks).
 func LoopBackingAllocatedOn(root string) uint64 {

internal/hostdisk/hostdisk_test.go

@@ -1,6 +1,10 @@
 package hostdisk
 
-import "testing"
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
 
 func TestUsableFreePercentExcludesLoopBackingFile(t *testing.T) {
 	t.Parallel()
@@ -19,3 +23,36 @@ func TestUsableFreePercentExcludesLoopBackingFile(t *testing.T) {
 		t.Fatalf("available covering usable total = %d, want 100", got)
 	}
 }
+
+func TestLoopBackingAllocatedOnCountsWrittenBlocks(t *testing.T) {
+	root := t.TempDir()
+	dir := filepath.Join(root, "var", "lib", "incus", "disks")
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	path := filepath.Join(dir, "gha-lvm.img")
+	if err := os.WriteFile(path, make([]byte, 1024*1024), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	got := LoopBackingAllocatedOn(root)
+	if got < 1024*1024 {
+		t.Fatalf("allocated %d, want at least 1MiB of written blocks", got)
+	}
+}
+
+func TestObserveReturnsErrorForMissingRoot(t *testing.T) {
+	if _, err := Observe(filepath.Join(t.TempDir(), "missing")); err == nil {
+		t.Fatal("missing root must not look like a successful observation")
+	}
+}
+
+func TestObserveWithoutLoopMatchesRawStatfsRatio(t *testing.T) {
+	root := t.TempDir()
+	got, err := Observe(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got < 0 || got > 100 {
+		t.Fatalf("usable free percent %d is not a percentage", got)
+	}
+}

internal/pressureobserve/observe.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/NDDev-OpenNetwork/github-actions/internal/hostdisk"
 	"github.com/NDDev-OpenNetwork/github-actions/internal/hostprobe"
 	"github.com/NDDev-OpenNetwork/github-actions/internal/pressuregate"
 	"github.com/NDDev-OpenNetwork/github-actions/internal/psimetrics"
@@ -49,7 +50,8 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "text/plain; version=0.0.4")
 		metrics := Render(state, now, h.maxStaleness(), err) +
 			RenderPressure(hostprobe.ReadPressure(h.hostRoot())) +
-			RenderCompliance(CollectCompliance(h.HostRoot, now))
+			RenderCompliance(CollectCompliance(h.HostRoot, now)) +
+			RenderRootDisk(h.hostRoot())
 		_, _ = w.Write([]byte(metrics))
 	default:
 		w.WriteHeader(http.StatusNotFound)
@@ -148,3 +150,19 @@ func RenderPressure(pressure hostprobe.Pressure, err error) string {
 	}
 	return psimetrics.Render(pressure)
 }
+
+// RenderRootDisk publishes this host's usable root free percent. Compute
+// members do not run the central fleet observer, so compute_root_disk_low
+// would otherwise see only gha-services. The value excludes the Incus
+// loop-backed pool file; a stat error publishes 0 so the ticket fails closed
+// instead of going silent.
+func RenderRootDisk(root string) string {
+	percent, err := hostdisk.Observe(root)
+	if err != nil {
+		percent = 0
+	}
+	return fmt.Sprintf(
+		"# HELP gha_fleet_host_root_free_percent Free block percentage of host-usable root space, excluding the Incus loop-backed pool file.\n"+
+			"# TYPE gha_fleet_host_root_free_percent gauge\n"+
+			"gha_fleet_host_root_free_percent %d\n", percent)
+}

internal/pressureobserve/observe_test.go

@@ -23,13 +23,20 @@ func TestHandlerExportsOOMCounterAndFreshPressureState(t *testing.T) {
 	if response.Code != http.StatusOK {
 		t.Fatalf("status=%d", response.Code)
 	}
-	for _, wanted := range []string{"gha_fleet_pressure_observer_up 1\n", "gha_fleet_host_oom_kills_total 23\n", "gha_fleet_host_pressure_open 1\n"} {
+	for _, wanted := range []string{"gha_fleet_pressure_observer_up 1\n", "gha_fleet_host_oom_kills_total 23\n", "gha_fleet_host_pressure_open 1\n", "gha_fleet_host_root_free_percent "} {
 		if !strings.Contains(response.Body.String(), wanted) {
 			t.Fatalf("metrics missing %q\n%s", wanted, response.Body.String())
 		}
 	}
 }
 
+func TestRenderRootDiskFailsClosedWhenRootCannotBeStat(t *testing.T) {
+	got := RenderRootDisk(filepath.Join(t.TempDir(), "missing"))
+	if !strings.Contains(got, "gha_fleet_host_root_free_percent 0\n") {
+		t.Fatalf("missing root did not fail closed\n%s", got)
+	}
+}
+
 func TestHandlerFailsClosedForStaleOrInvalidState(t *testing.T) {
 	now := time.Date(2026, 8, 27, 0, 2, 0, 0, time.UTC)
 	path := filepath.Join(t.TempDir(), "pressure.json")