Exclude the Incus loop file from host disk free-percent

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## Unreleased
 
+- Compute `compute_root_disk_low` from usable root space, excluding the Incus
+  loop-backed thin-pool file. With `discards=nopassdown` that file stays fully
+  allocated (~200 GiB on a 309 GiB root) while pool `data_percent` is ~20%, so
+  the OTel filesystem ratio fired at 17–18% free of the whole disk. Placement
+  already reads the thin pool; the observer metric and ticket now match that
+  fact. Provider admission still uses raw statvfs until the next provider
+  derivative, because hostprobe is part of that artifact.
+
 - Treat the authenticated job-start claim as the only repository identity for
   cache delivery and teardown diagnostics. An organization scale-set runner can
   be created for one queued repository and receive another job from GitHub;

cmd/gha-fleet-observer/main.go

@@ -21,6 +21,7 @@ import (
 	"github.com/NDDev-OpenNetwork/github-actions/internal/fleettrace"
 	providerconfig "github.com/NDDev-OpenNetwork/github-actions/internal/garmproviderincus/config"
 	"github.com/NDDev-OpenNetwork/github-actions/internal/garmproviderincus/provider"
+	"github.com/NDDev-OpenNetwork/github-actions/internal/hostdisk"
 	"github.com/NDDev-OpenNetwork/github-actions/internal/hostprobe"
 	"github.com/NDDev-OpenNetwork/github-actions/internal/providerjournal"
 	"github.com/NDDev-OpenNetwork/github-actions/internal/providerretry"
@@ -189,7 +190,14 @@ func buildCollector(options options) (fleetobserve.Collector, error) {
 	}
 	collector := fleetobserve.Collector{
 		Config: platform,
-		Host:   hostprobe.Collect,
+		Host: func(ctx context.Context) (hostprobe.Snapshot, error) {
+			snapshot, err := hostprobe.Collect(ctx)
+			if err != nil {
+				return snapshot, err
+			}
+			hostdisk.ApplyUsableRootPercent(&snapshot)
+			return snapshot, nil
+		},
 		Journal: func(ctx context.Context) (providerjournal.Journal, error) {
 			return journalStore.ReadOnly(ctx)
 		},

config/observability-rules.yaml

@@ -145,10 +145,16 @@ rules:
   - id: compute_root_disk_low
     severity: ticket
     query_language: promql
-    stream_name: system_filesystem_usage
-    expression: sum by (host_name) (system_filesystem_usage{service_namespace="nddev-github-actions",mountpoint="/",state="free"}) / sum by (host_name) (system_filesystem_usage{service_namespace="nddev-github-actions",mountpoint="/",state=~"free|used"})
+    stream_name: gha_fleet_host_root_free_percent
+    # The OTel host filesystem series counts the Incus loop-backed thin pool
+    # as ordinary used space on /. That file is a fixed 200 GiB reservation;
+    # with discards=nopassdown it stays fully allocated while pool data_percent
+    # is ~20%. The ticket then fired at 17–18% free of the whole root, which
+    # is the loop file, not tenant data. Placement already reads the thin pool.
+    # Admission and this rule use the observer percent that excludes the loop.
+    expression: min by (host_name) (gha_fleet_host_root_free_percent)
     operator: "<"
-    threshold: 0.2
+    threshold: 20
     evaluation_seconds: 300
     hold_seconds: 900
     # Disk pressure changes slowly and the ticket already carries one exact

internal/fleetobserve/metrics.go

@@ -37,7 +37,7 @@ func RenderPrometheus(snapshot Snapshot, now time.Time, maxStaleness time.Durati
 	counter(&output, "gha_fleet_host_oom_kills_total", "Kernel OOM kills observed since host boot.", float64(snapshot.Host.Memory.OOMKillsTotal))
 	output.WriteString(psimetrics.Render(snapshot.Host.Pressure))
 	gauge(&output, "gha_fleet_host_root_available_bytes", "Available bytes on the root filesystem.", float64(snapshot.Host.RootFilesystem.AvailableMiB)*1024*1024)
-	gauge(&output, "gha_fleet_host_root_free_percent", "Free block percentage on the root filesystem.", float64(snapshot.Host.RootFilesystem.FreePercent))
+	gauge(&output, "gha_fleet_host_root_free_percent", "Free block percentage of host-usable root space, excluding the Incus loop-backed pool file.", float64(snapshot.Host.RootFilesystem.FreePercent))
 	gauge(&output, "gha_fleet_host_root_free_inodes_percent", "Free inode percentage on the root filesystem.", float64(snapshot.Host.RootFilesystem.FreeInodesPercent))
 	gauge(&output, "gha_fleet_host_kvm_present", "Whether /dev/kvm is present in the observer service.", boolFloat(snapshot.Host.KVM.Present))
 	gauge(&output, "gha_fleet_host_kvm_accessible", "Whether /dev/kvm is accessible to the observer service.", boolFloat(snapshot.Host.KVM.Accessible))

internal/hostdisk/hostdisk.go

@@ -0,0 +1,79 @@
+// Package hostdisk computes the host-usable root free percent, excluding the
+// Incus loop-backed thin-pool file that lives on the same filesystem.
+//
+// Placement already reads the thin pool. The OTel host filesystem series and
+// raw statvfs count that file as ordinary used space, so compute_root_disk_low
+// fired at 17–18% free of a 309 GiB root while pool data_percent was ~20%.
+package hostdisk
+
+import (
+	"math/bits"
+	"path/filepath"
+	"syscall"
+
+	"github.com/NDDev-OpenNetwork/github-actions/internal/hostprobe"
+)
+
+const mebibyte = 1024 * 1024
+
+// ApplyUsableRootPercent rewrites snapshot.RootFilesystem.FreePercent so
+// metrics and alerts describe remaining host space, not the loop reservation.
+func ApplyUsableRootPercent(snapshot *hostprobe.Snapshot) {
+	if snapshot == nil {
+		return
+	}
+	total := snapshot.RootFilesystem.TotalMiB * mebibyte
+	available := snapshot.RootFilesystem.AvailableMiB * mebibyte
+	snapshot.RootFilesystem.FreePercent = UsableFreePercent(total, available, LoopBackingAllocatedOn("/"))
+}
+
+// UsableFreePercent is the free share of root after subtracting a loop file
+// that is already reserved for Incus. A missing or oversize loop falls back
+// to the raw statvfs ratio.
+func UsableFreePercent(total, available, loopAllocated uint64) int {
+	if loopAllocated == 0 || loopAllocated >= total {
+		return percent(available, total)
+	}
+	usableTotal := total - loopAllocated
+	if available >= usableTotal {
+		return 100
+	}
+	return percent(available, usableTotal)
+}
+
+// LoopBackingAllocatedOn sums allocated bytes of Incus pool images on the
+// same device as root. Sparse holes are not counted (st_blocks).
+func LoopBackingAllocatedOn(root string) uint64 {
+	var rootStat syscall.Stat_t
+	if syscall.Stat(root, &rootStat) != nil {
+		return 0
+	}
+	matches, err := filepath.Glob(filepath.Join(root, "var", "lib", "incus", "disks", "*.img"))
+	if err != nil {
+		return 0
+	}
+	var allocated uint64
+	rootDev := uint64(rootStat.Dev)
+	for _, path := range matches {
+		var st syscall.Stat_t
+		if syscall.Stat(path, &st) != nil || uint64(st.Dev) != rootDev {
+			continue
+		}
+		allocated += uint64(st.Blocks) * 512
+	}
+	return allocated
+}
+
+func percent(part, total uint64) int {
+	if total == 0 {
+		return 0
+	}
+	whole := part / total
+	remainder := part % total
+	high, low := bits.Mul64(remainder, 100)
+	fraction, leftover := bits.Div64(high, low, total)
+	if leftover > 0 {
+		fraction++
+	}
+	return int(whole*100 + fraction)
+}

internal/hostdisk/hostdisk_test.go

@@ -0,0 +1,21 @@
+package hostdisk
+
+import "testing"
+
+func TestUsableFreePercentExcludesLoopBackingFile(t *testing.T) {
+	t.Parallel()
+	const gib = 1024 * 1024 * 1024
+	got := UsableFreePercent(309*gib, 73*gib, 200*gib)
+	if got != 67 {
+		t.Fatalf("usable free = %d, want 67", got)
+	}
+	if raw := UsableFreePercent(309*gib, 73*gib, 0); raw != 24 {
+		t.Fatalf("raw free = %d, want 24", raw)
+	}
+	if got := UsableFreePercent(309*gib, 73*gib, 400*gib); got != 24 {
+		t.Fatalf("oversize loop must not invert the percentage: %d", got)
+	}
+	if got := UsableFreePercent(100*gib, 100*gib, 40*gib); got != 100 {
+		t.Fatalf("available covering usable total = %d, want 100", got)
+	}
+}

internal/observabilityrules/rules_test.go

@@ -29,7 +29,7 @@ func TestRepositoryRulesUseCurrentMetricSemantics(t *testing.T) {
 		"provider_retry_error_persistent":   `gha_fleet_provider_retry_deferred_records_by_error_class{error_class=~"identity|intent|provider|timeout|unknown"}`,
 		"compute_pressure_observer_missing": `count(up{service_name="pressure-state"} == 1)`,
 		"compute_pressure_state_stale":      "gha_fleet_pressure_observer_up",
-		"compute_root_disk_low":             "system_filesystem_usage",
+		"compute_root_disk_low":             "gha_fleet_host_root_free_percent",
 		"kernel_slab_unreclaimable":         "gha_fleet_host_slab_unreclaimable_attributed_bytes",
 		"audit_suppression_burst":           `signal_class="audit_suppressed"`,
 		"kernel_workqueue_hog":              `signal_class="kernel_workqueue_hog"`,