From 506fdabe335f8483271cbd24b6b45c5933e59e4a Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Fri, 4 Sep 2026 03:17:40 +0500 Subject: [PATCH 1/2] Exclude the Incus loop file from host disk free-percent. compute_root_disk_low used the OTel root filesystem ratio, which counts the 200 GiB loop-backed thin pool as tenant-used space. With discards=nopassdown that file stays fully allocated while pool data_percent is ~20%, so the ticket fired at 17-18% free of the whole disk. Admission and the ticket now use the observer percent of remaining host space; placement already reads the thin pool. --- CHANGELOG.md | 7 ++++ config/observability-rules.yaml | 12 +++++-- internal/fleetobserve/metrics.go | 2 +- internal/hostprobe/collect_linux.go | 40 ++++++++++++++++++++++- internal/hostprobe/collect_linux_test.go | 21 ++++++++++++ internal/observabilityrules/rules_test.go | 2 +- 6 files changed, 78 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62aec8e5..77949489 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ ## Unreleased +- Compute host disk free-percent and `compute_root_disk_low` from usable root + space, excluding the Incus loop-backed thin-pool file. With + `discards=nopassdown` that file stays fully allocated (~200 GiB on a 309 GiB + root) while pool `data_percent` is ~20%, so the OTel filesystem ratio fired + at 17–18% free of the whole disk. Placement already reads the thin pool; + admission and the ticket now match that fact. + - Treat the authenticated job-start claim as the only repository identity for cache delivery and teardown diagnostics. An organization scale-set runner can be created for one queued repository and receive another job from GitHub; diff --git a/config/observability-rules.yaml b/config/observability-rules.yaml index 24473655..b2357a25 100644 --- a/config/observability-rules.yaml +++ b/config/observability-rules.yaml @@ -145,10 +145,16 @@ rules: - id: compute_root_disk_low severity: ticket query_language: promql - stream_name: system_filesystem_usage - expression: sum by (host_name) (system_filesystem_usage{service_namespace="nddev-github-actions",mountpoint="/",state="free"}) / sum by (host_name) (system_filesystem_usage{service_namespace="nddev-github-actions",mountpoint="/",state=~"free|used"}) + stream_name: gha_fleet_host_root_free_percent + # The OTel host filesystem series counts the Incus loop-backed thin pool + # as ordinary used space on /. That file is a fixed 200 GiB reservation; + # with discards=nopassdown it stays fully allocated while pool data_percent + # is ~20%. The ticket then fired at 17–18% free of the whole root, which + # is the loop file, not tenant data. Placement already reads the thin pool. + # Admission and this rule use the observer percent that excludes the loop. + expression: min by (host_name) (gha_fleet_host_root_free_percent) operator: "<" - threshold: 0.2 + threshold: 20 evaluation_seconds: 300 hold_seconds: 900 # Disk pressure changes slowly and the ticket already carries one exact diff --git a/internal/fleetobserve/metrics.go b/internal/fleetobserve/metrics.go index 59c979f3..c9c2dc8c 100644 --- a/internal/fleetobserve/metrics.go +++ b/internal/fleetobserve/metrics.go @@ -37,7 +37,7 @@ func RenderPrometheus(snapshot Snapshot, now time.Time, maxStaleness time.Durati counter(&output, "gha_fleet_host_oom_kills_total", "Kernel OOM kills observed since host boot.", float64(snapshot.Host.Memory.OOMKillsTotal)) output.WriteString(psimetrics.Render(snapshot.Host.Pressure)) gauge(&output, "gha_fleet_host_root_available_bytes", "Available bytes on the root filesystem.", float64(snapshot.Host.RootFilesystem.AvailableMiB)*1024*1024) - gauge(&output, "gha_fleet_host_root_free_percent", "Free block percentage on the root filesystem.", float64(snapshot.Host.RootFilesystem.FreePercent)) + gauge(&output, "gha_fleet_host_root_free_percent", "Free block percentage of host-usable root space, excluding the Incus loop-backed pool file.", float64(snapshot.Host.RootFilesystem.FreePercent)) gauge(&output, "gha_fleet_host_root_free_inodes_percent", "Free inode percentage on the root filesystem.", float64(snapshot.Host.RootFilesystem.FreeInodesPercent)) gauge(&output, "gha_fleet_host_kvm_present", "Whether /dev/kvm is present in the observer service.", boolFloat(snapshot.Host.KVM.Present)) gauge(&output, "gha_fleet_host_kvm_accessible", "Whether /dev/kvm is accessible to the observer service.", boolFloat(snapshot.Host.KVM.Accessible)) diff --git a/internal/hostprobe/collect_linux.go b/internal/hostprobe/collect_linux.go index 457a14ca..629460c2 100644 --- a/internal/hostprobe/collect_linux.go +++ b/internal/hostprobe/collect_linux.go @@ -380,7 +380,12 @@ func rootFilesystem(root string, runner CommandRunner, ctx context.Context) (Fil } total := uint64(stats.Blocks) * uint64(stats.Bsize) available := uint64(stats.Bavail) * uint64(stats.Bsize) - freePercent := percent(available, total) + var rootStat syscall.Stat_t + loopAllocated := uint64(0) + if syscall.Stat(root, &rootStat) == nil { + loopAllocated = loopBackingAllocatedOn(root, uint64(rootStat.Dev)) + } + freePercent := usableFreePercent(total, available, loopAllocated) freeInodesPercent := 100 if stats.Files > 0 { freeInodesPercent = percent(uint64(stats.Ffree), uint64(stats.Files)) @@ -420,6 +425,39 @@ func rootFilesystem(root string, runner CommandRunner, ctx context.Context) (Fil return filesystem, nil } +// usableFreePercent is the free share of the host disk that jobs, caches and +// logs can actually consume. The Incus loop-backed thin pool is a fixed +// reservation on the same root filesystem; counting it as tenant-used space +// made admission and compute_root_disk_low fire while the pool itself was +// mostly empty. docs/maintenance-windows.md requires both values; this is the +// host-side one. Placement already reads the thin pool. +func usableFreePercent(total, available, loopAllocated uint64) int { + if loopAllocated == 0 || loopAllocated >= total { + return percent(available, total) + } + usableTotal := total - loopAllocated + if available >= usableTotal { + return 100 + } + return percent(available, usableTotal) +} + +func loopBackingAllocatedOn(root string, rootDev uint64) uint64 { + matches, err := filepath.Glob(filepath.Join(root, "var", "lib", "incus", "disks", "*.img")) + if err != nil { + return 0 + } + var allocated uint64 + for _, path := range matches { + var st syscall.Stat_t + if syscall.Stat(path, &st) != nil || uint64(st.Dev) != rootDev { + continue + } + allocated += uint64(st.Blocks) * 512 + } + return allocated +} + func kvmState(root string) KVM { device := filepath.Join(root, "dev", "kvm") present := fileExists(device) diff --git a/internal/hostprobe/collect_linux_test.go b/internal/hostprobe/collect_linux_test.go index 3dd4c5e4..1509b68a 100644 --- a/internal/hostprobe/collect_linux_test.go +++ b/internal/hostprobe/collect_linux_test.go @@ -31,3 +31,24 @@ func TestRunnerProcessesExcludeIncusGuests(t *testing.T) { t.Fatalf("host listeners/workers = %d/%d, want 1/1", listeners, workers) } } + +func TestUsableFreePercentExcludesLoopBackingFile(t *testing.T) { + t.Parallel() + // Live members: 309 GiB root, 73 GiB available, 200 GiB fully allocated + // loop file. Raw statvfs is 23% free and trips the 20% gate; the usable + // share of the remaining 109 GiB is 67%. + const gib = 1024 * 1024 * 1024 + got := usableFreePercent(309*gib, 73*gib, 200*gib) + if got != 67 { + t.Fatalf("usable free = %d, want 67", got) + } + if raw := usableFreePercent(309*gib, 73*gib, 0); raw != 24 { + t.Fatalf("raw free = %d, want 24", raw) + } + if got := usableFreePercent(309*gib, 73*gib, 400*gib); got != 24 { + t.Fatalf("oversize loop must not invert the percentage: %d", got) + } + if got := usableFreePercent(100*gib, 100*gib, 40*gib); got != 100 { + t.Fatalf("available covering usable total = %d, want 100", got) + } +} diff --git a/internal/observabilityrules/rules_test.go b/internal/observabilityrules/rules_test.go index cfb8e341..54f33772 100644 --- a/internal/observabilityrules/rules_test.go +++ b/internal/observabilityrules/rules_test.go @@ -29,7 +29,7 @@ func TestRepositoryRulesUseCurrentMetricSemantics(t *testing.T) { "provider_retry_error_persistent": `gha_fleet_provider_retry_deferred_records_by_error_class{error_class=~"identity|intent|provider|timeout|unknown"}`, "compute_pressure_observer_missing": `count(up{service_name="pressure-state"} == 1)`, "compute_pressure_state_stale": "gha_fleet_pressure_observer_up", - "compute_root_disk_low": "system_filesystem_usage", + "compute_root_disk_low": "gha_fleet_host_root_free_percent", "kernel_slab_unreclaimable": "gha_fleet_host_slab_unreclaimable_attributed_bytes", "audit_suppression_burst": `signal_class="audit_suppressed"`, "kernel_workqueue_hog": `signal_class="kernel_workqueue_hog"`, From f9a2752a4a0c5524c0974abd0344dbf9da710531 Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Fri, 4 Sep 2026 03:23:18 +0500 Subject: [PATCH 2/2] Keep hostprobe unchanged; apply usable disk percent in the observer. internal/hostprobe is part of the provider derivative. Moving the loop-file exclusion into internal/hostdisk and gha-fleet-observer lets the ticket use the corrected percent without a provider release. Admission still uses raw statvfs until the next provider identity. --- CHANGELOG.md | 13 ++-- cmd/gha-fleet-observer/main.go | 10 ++- internal/hostdisk/hostdisk.go | 79 ++++++++++++++++++++++++ internal/hostdisk/hostdisk_test.go | 21 +++++++ internal/hostprobe/collect_linux.go | 40 +----------- internal/hostprobe/collect_linux_test.go | 21 ------- 6 files changed, 117 insertions(+), 67 deletions(-) create mode 100644 internal/hostdisk/hostdisk.go create mode 100644 internal/hostdisk/hostdisk_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 77949489..d58adf4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,13 @@ ## Unreleased -- Compute host disk free-percent and `compute_root_disk_low` from usable root - space, excluding the Incus loop-backed thin-pool file. With - `discards=nopassdown` that file stays fully allocated (~200 GiB on a 309 GiB - root) while pool `data_percent` is ~20%, so the OTel filesystem ratio fired - at 17–18% free of the whole disk. Placement already reads the thin pool; - admission and the ticket now match that fact. +- Compute `compute_root_disk_low` from usable root space, excluding the Incus + loop-backed thin-pool file. With `discards=nopassdown` that file stays fully + allocated (~200 GiB on a 309 GiB root) while pool `data_percent` is ~20%, so + the OTel filesystem ratio fired at 17–18% free of the whole disk. Placement + already reads the thin pool; the observer metric and ticket now match that + fact. Provider admission still uses raw statvfs until the next provider + derivative, because hostprobe is part of that artifact. - Treat the authenticated job-start claim as the only repository identity for cache delivery and teardown diagnostics. An organization scale-set runner can diff --git a/cmd/gha-fleet-observer/main.go b/cmd/gha-fleet-observer/main.go index c5640d04..c2de6053 100644 --- a/cmd/gha-fleet-observer/main.go +++ b/cmd/gha-fleet-observer/main.go @@ -21,6 +21,7 @@ import ( "github.com/NDDev-OpenNetwork/github-actions/internal/fleettrace" providerconfig "github.com/NDDev-OpenNetwork/github-actions/internal/garmproviderincus/config" "github.com/NDDev-OpenNetwork/github-actions/internal/garmproviderincus/provider" + "github.com/NDDev-OpenNetwork/github-actions/internal/hostdisk" "github.com/NDDev-OpenNetwork/github-actions/internal/hostprobe" "github.com/NDDev-OpenNetwork/github-actions/internal/providerjournal" "github.com/NDDev-OpenNetwork/github-actions/internal/providerretry" @@ -189,7 +190,14 @@ func buildCollector(options options) (fleetobserve.Collector, error) { } collector := fleetobserve.Collector{ Config: platform, - Host: hostprobe.Collect, + Host: func(ctx context.Context) (hostprobe.Snapshot, error) { + snapshot, err := hostprobe.Collect(ctx) + if err != nil { + return snapshot, err + } + hostdisk.ApplyUsableRootPercent(&snapshot) + return snapshot, nil + }, Journal: func(ctx context.Context) (providerjournal.Journal, error) { return journalStore.ReadOnly(ctx) }, diff --git a/internal/hostdisk/hostdisk.go b/internal/hostdisk/hostdisk.go new file mode 100644 index 00000000..2d9980d6 --- /dev/null +++ b/internal/hostdisk/hostdisk.go @@ -0,0 +1,79 @@ +// Package hostdisk computes the host-usable root free percent, excluding the +// Incus loop-backed thin-pool file that lives on the same filesystem. +// +// Placement already reads the thin pool. The OTel host filesystem series and +// raw statvfs count that file as ordinary used space, so compute_root_disk_low +// fired at 17–18% free of a 309 GiB root while pool data_percent was ~20%. +package hostdisk + +import ( + "math/bits" + "path/filepath" + "syscall" + + "github.com/NDDev-OpenNetwork/github-actions/internal/hostprobe" +) + +const mebibyte = 1024 * 1024 + +// ApplyUsableRootPercent rewrites snapshot.RootFilesystem.FreePercent so +// metrics and alerts describe remaining host space, not the loop reservation. +func ApplyUsableRootPercent(snapshot *hostprobe.Snapshot) { + if snapshot == nil { + return + } + total := snapshot.RootFilesystem.TotalMiB * mebibyte + available := snapshot.RootFilesystem.AvailableMiB * mebibyte + snapshot.RootFilesystem.FreePercent = UsableFreePercent(total, available, LoopBackingAllocatedOn("/")) +} + +// UsableFreePercent is the free share of root after subtracting a loop file +// that is already reserved for Incus. A missing or oversize loop falls back +// to the raw statvfs ratio. +func UsableFreePercent(total, available, loopAllocated uint64) int { + if loopAllocated == 0 || loopAllocated >= total { + return percent(available, total) + } + usableTotal := total - loopAllocated + if available >= usableTotal { + return 100 + } + return percent(available, usableTotal) +} + +// LoopBackingAllocatedOn sums allocated bytes of Incus pool images on the +// same device as root. Sparse holes are not counted (st_blocks). +func LoopBackingAllocatedOn(root string) uint64 { + var rootStat syscall.Stat_t + if syscall.Stat(root, &rootStat) != nil { + return 0 + } + matches, err := filepath.Glob(filepath.Join(root, "var", "lib", "incus", "disks", "*.img")) + if err != nil { + return 0 + } + var allocated uint64 + rootDev := uint64(rootStat.Dev) + for _, path := range matches { + var st syscall.Stat_t + if syscall.Stat(path, &st) != nil || uint64(st.Dev) != rootDev { + continue + } + allocated += uint64(st.Blocks) * 512 + } + return allocated +} + +func percent(part, total uint64) int { + if total == 0 { + return 0 + } + whole := part / total + remainder := part % total + high, low := bits.Mul64(remainder, 100) + fraction, leftover := bits.Div64(high, low, total) + if leftover > 0 { + fraction++ + } + return int(whole*100 + fraction) +} diff --git a/internal/hostdisk/hostdisk_test.go b/internal/hostdisk/hostdisk_test.go new file mode 100644 index 00000000..61f1db27 --- /dev/null +++ b/internal/hostdisk/hostdisk_test.go @@ -0,0 +1,21 @@ +package hostdisk + +import "testing" + +func TestUsableFreePercentExcludesLoopBackingFile(t *testing.T) { + t.Parallel() + const gib = 1024 * 1024 * 1024 + got := UsableFreePercent(309*gib, 73*gib, 200*gib) + if got != 67 { + t.Fatalf("usable free = %d, want 67", got) + } + if raw := UsableFreePercent(309*gib, 73*gib, 0); raw != 24 { + t.Fatalf("raw free = %d, want 24", raw) + } + if got := UsableFreePercent(309*gib, 73*gib, 400*gib); got != 24 { + t.Fatalf("oversize loop must not invert the percentage: %d", got) + } + if got := UsableFreePercent(100*gib, 100*gib, 40*gib); got != 100 { + t.Fatalf("available covering usable total = %d, want 100", got) + } +} diff --git a/internal/hostprobe/collect_linux.go b/internal/hostprobe/collect_linux.go index 629460c2..457a14ca 100644 --- a/internal/hostprobe/collect_linux.go +++ b/internal/hostprobe/collect_linux.go @@ -380,12 +380,7 @@ func rootFilesystem(root string, runner CommandRunner, ctx context.Context) (Fil } total := uint64(stats.Blocks) * uint64(stats.Bsize) available := uint64(stats.Bavail) * uint64(stats.Bsize) - var rootStat syscall.Stat_t - loopAllocated := uint64(0) - if syscall.Stat(root, &rootStat) == nil { - loopAllocated = loopBackingAllocatedOn(root, uint64(rootStat.Dev)) - } - freePercent := usableFreePercent(total, available, loopAllocated) + freePercent := percent(available, total) freeInodesPercent := 100 if stats.Files > 0 { freeInodesPercent = percent(uint64(stats.Ffree), uint64(stats.Files)) @@ -425,39 +420,6 @@ func rootFilesystem(root string, runner CommandRunner, ctx context.Context) (Fil return filesystem, nil } -// usableFreePercent is the free share of the host disk that jobs, caches and -// logs can actually consume. The Incus loop-backed thin pool is a fixed -// reservation on the same root filesystem; counting it as tenant-used space -// made admission and compute_root_disk_low fire while the pool itself was -// mostly empty. docs/maintenance-windows.md requires both values; this is the -// host-side one. Placement already reads the thin pool. -func usableFreePercent(total, available, loopAllocated uint64) int { - if loopAllocated == 0 || loopAllocated >= total { - return percent(available, total) - } - usableTotal := total - loopAllocated - if available >= usableTotal { - return 100 - } - return percent(available, usableTotal) -} - -func loopBackingAllocatedOn(root string, rootDev uint64) uint64 { - matches, err := filepath.Glob(filepath.Join(root, "var", "lib", "incus", "disks", "*.img")) - if err != nil { - return 0 - } - var allocated uint64 - for _, path := range matches { - var st syscall.Stat_t - if syscall.Stat(path, &st) != nil || uint64(st.Dev) != rootDev { - continue - } - allocated += uint64(st.Blocks) * 512 - } - return allocated -} - func kvmState(root string) KVM { device := filepath.Join(root, "dev", "kvm") present := fileExists(device) diff --git a/internal/hostprobe/collect_linux_test.go b/internal/hostprobe/collect_linux_test.go index 1509b68a..3dd4c5e4 100644 --- a/internal/hostprobe/collect_linux_test.go +++ b/internal/hostprobe/collect_linux_test.go @@ -31,24 +31,3 @@ func TestRunnerProcessesExcludeIncusGuests(t *testing.T) { t.Fatalf("host listeners/workers = %d/%d, want 1/1", listeners, workers) } } - -func TestUsableFreePercentExcludesLoopBackingFile(t *testing.T) { - t.Parallel() - // Live members: 309 GiB root, 73 GiB available, 200 GiB fully allocated - // loop file. Raw statvfs is 23% free and trips the 20% gate; the usable - // share of the remaining 109 GiB is 67%. - const gib = 1024 * 1024 * 1024 - got := usableFreePercent(309*gib, 73*gib, 200*gib) - if got != 67 { - t.Fatalf("usable free = %d, want 67", got) - } - if raw := usableFreePercent(309*gib, 73*gib, 0); raw != 24 { - t.Fatalf("raw free = %d, want 24", raw) - } - if got := usableFreePercent(309*gib, 73*gib, 400*gib); got != 24 { - t.Fatalf("oversize loop must not invert the percentage: %d", got) - } - if got := usableFreePercent(100*gib, 100*gib, 40*gib); got != 100 { - t.Fatalf("available covering usable total = %d, want 100", got) - } -}