diff --git a/CHANGELOG.md b/CHANGELOG.md index 62aec8e5..d58adf4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ ## Unreleased +- Compute `compute_root_disk_low` from usable root space, excluding the Incus + loop-backed thin-pool file. With `discards=nopassdown` that file stays fully + allocated (~200 GiB on a 309 GiB root) while pool `data_percent` is ~20%, so + the OTel filesystem ratio fired at 17–18% free of the whole disk. Placement + already reads the thin pool; the observer metric and ticket now match that + fact. Provider admission still uses raw statvfs until the next provider + derivative, because hostprobe is part of that artifact. + - Treat the authenticated job-start claim as the only repository identity for cache delivery and teardown diagnostics. An organization scale-set runner can be created for one queued repository and receive another job from GitHub; diff --git a/cmd/gha-fleet-observer/main.go b/cmd/gha-fleet-observer/main.go index c5640d04..c2de6053 100644 --- a/cmd/gha-fleet-observer/main.go +++ b/cmd/gha-fleet-observer/main.go @@ -21,6 +21,7 @@ import ( "github.com/NDDev-OpenNetwork/github-actions/internal/fleettrace" providerconfig "github.com/NDDev-OpenNetwork/github-actions/internal/garmproviderincus/config" "github.com/NDDev-OpenNetwork/github-actions/internal/garmproviderincus/provider" + "github.com/NDDev-OpenNetwork/github-actions/internal/hostdisk" "github.com/NDDev-OpenNetwork/github-actions/internal/hostprobe" "github.com/NDDev-OpenNetwork/github-actions/internal/providerjournal" "github.com/NDDev-OpenNetwork/github-actions/internal/providerretry" @@ -189,7 +190,14 @@ func buildCollector(options options) (fleetobserve.Collector, error) { } collector := fleetobserve.Collector{ Config: platform, - Host: hostprobe.Collect, + Host: func(ctx context.Context) (hostprobe.Snapshot, error) { + snapshot, err := hostprobe.Collect(ctx) + if err != nil { + return snapshot, err + } + hostdisk.ApplyUsableRootPercent(&snapshot) + return snapshot, nil + }, Journal: func(ctx context.Context) (providerjournal.Journal, error) { return journalStore.ReadOnly(ctx) }, diff --git a/config/observability-rules.yaml b/config/observability-rules.yaml index 24473655..b2357a25 100644 --- a/config/observability-rules.yaml +++ b/config/observability-rules.yaml @@ -145,10 +145,16 @@ rules: - id: compute_root_disk_low severity: ticket query_language: promql - stream_name: system_filesystem_usage - expression: sum by (host_name) (system_filesystem_usage{service_namespace="nddev-github-actions",mountpoint="/",state="free"}) / sum by (host_name) (system_filesystem_usage{service_namespace="nddev-github-actions",mountpoint="/",state=~"free|used"}) + stream_name: gha_fleet_host_root_free_percent + # The OTel host filesystem series counts the Incus loop-backed thin pool + # as ordinary used space on /. That file is a fixed 200 GiB reservation; + # with discards=nopassdown it stays fully allocated while pool data_percent + # is ~20%. The ticket then fired at 17–18% free of the whole root, which + # is the loop file, not tenant data. Placement already reads the thin pool. + # Admission and this rule use the observer percent that excludes the loop. + expression: min by (host_name) (gha_fleet_host_root_free_percent) operator: "<" - threshold: 0.2 + threshold: 20 evaluation_seconds: 300 hold_seconds: 900 # Disk pressure changes slowly and the ticket already carries one exact diff --git a/internal/fleetobserve/metrics.go b/internal/fleetobserve/metrics.go index 59c979f3..c9c2dc8c 100644 --- a/internal/fleetobserve/metrics.go +++ b/internal/fleetobserve/metrics.go @@ -37,7 +37,7 @@ func RenderPrometheus(snapshot Snapshot, now time.Time, maxStaleness time.Durati counter(&output, "gha_fleet_host_oom_kills_total", "Kernel OOM kills observed since host boot.", float64(snapshot.Host.Memory.OOMKillsTotal)) output.WriteString(psimetrics.Render(snapshot.Host.Pressure)) gauge(&output, "gha_fleet_host_root_available_bytes", "Available bytes on the root filesystem.", float64(snapshot.Host.RootFilesystem.AvailableMiB)*1024*1024) - gauge(&output, "gha_fleet_host_root_free_percent", "Free block percentage on the root filesystem.", float64(snapshot.Host.RootFilesystem.FreePercent)) + gauge(&output, "gha_fleet_host_root_free_percent", "Free block percentage of host-usable root space, excluding the Incus loop-backed pool file.", float64(snapshot.Host.RootFilesystem.FreePercent)) gauge(&output, "gha_fleet_host_root_free_inodes_percent", "Free inode percentage on the root filesystem.", float64(snapshot.Host.RootFilesystem.FreeInodesPercent)) gauge(&output, "gha_fleet_host_kvm_present", "Whether /dev/kvm is present in the observer service.", boolFloat(snapshot.Host.KVM.Present)) gauge(&output, "gha_fleet_host_kvm_accessible", "Whether /dev/kvm is accessible to the observer service.", boolFloat(snapshot.Host.KVM.Accessible)) diff --git a/internal/hostdisk/hostdisk.go b/internal/hostdisk/hostdisk.go new file mode 100644 index 00000000..2d9980d6 --- /dev/null +++ b/internal/hostdisk/hostdisk.go @@ -0,0 +1,79 @@ +// Package hostdisk computes the host-usable root free percent, excluding the +// Incus loop-backed thin-pool file that lives on the same filesystem. +// +// Placement already reads the thin pool. The OTel host filesystem series and +// raw statvfs count that file as ordinary used space, so compute_root_disk_low +// fired at 17–18% free of a 309 GiB root while pool data_percent was ~20%. +package hostdisk + +import ( + "math/bits" + "path/filepath" + "syscall" + + "github.com/NDDev-OpenNetwork/github-actions/internal/hostprobe" +) + +const mebibyte = 1024 * 1024 + +// ApplyUsableRootPercent rewrites snapshot.RootFilesystem.FreePercent so +// metrics and alerts describe remaining host space, not the loop reservation. +func ApplyUsableRootPercent(snapshot *hostprobe.Snapshot) { + if snapshot == nil { + return + } + total := snapshot.RootFilesystem.TotalMiB * mebibyte + available := snapshot.RootFilesystem.AvailableMiB * mebibyte + snapshot.RootFilesystem.FreePercent = UsableFreePercent(total, available, LoopBackingAllocatedOn("/")) +} + +// UsableFreePercent is the free share of root after subtracting a loop file +// that is already reserved for Incus. A missing or oversize loop falls back +// to the raw statvfs ratio. +func UsableFreePercent(total, available, loopAllocated uint64) int { + if loopAllocated == 0 || loopAllocated >= total { + return percent(available, total) + } + usableTotal := total - loopAllocated + if available >= usableTotal { + return 100 + } + return percent(available, usableTotal) +} + +// LoopBackingAllocatedOn sums allocated bytes of Incus pool images on the +// same device as root. Sparse holes are not counted (st_blocks). +func LoopBackingAllocatedOn(root string) uint64 { + var rootStat syscall.Stat_t + if syscall.Stat(root, &rootStat) != nil { + return 0 + } + matches, err := filepath.Glob(filepath.Join(root, "var", "lib", "incus", "disks", "*.img")) + if err != nil { + return 0 + } + var allocated uint64 + rootDev := uint64(rootStat.Dev) + for _, path := range matches { + var st syscall.Stat_t + if syscall.Stat(path, &st) != nil || uint64(st.Dev) != rootDev { + continue + } + allocated += uint64(st.Blocks) * 512 + } + return allocated +} + +func percent(part, total uint64) int { + if total == 0 { + return 0 + } + whole := part / total + remainder := part % total + high, low := bits.Mul64(remainder, 100) + fraction, leftover := bits.Div64(high, low, total) + if leftover > 0 { + fraction++ + } + return int(whole*100 + fraction) +} diff --git a/internal/hostdisk/hostdisk_test.go b/internal/hostdisk/hostdisk_test.go new file mode 100644 index 00000000..61f1db27 --- /dev/null +++ b/internal/hostdisk/hostdisk_test.go @@ -0,0 +1,21 @@ +package hostdisk + +import "testing" + +func TestUsableFreePercentExcludesLoopBackingFile(t *testing.T) { + t.Parallel() + const gib = 1024 * 1024 * 1024 + got := UsableFreePercent(309*gib, 73*gib, 200*gib) + if got != 67 { + t.Fatalf("usable free = %d, want 67", got) + } + if raw := UsableFreePercent(309*gib, 73*gib, 0); raw != 24 { + t.Fatalf("raw free = %d, want 24", raw) + } + if got := UsableFreePercent(309*gib, 73*gib, 400*gib); got != 24 { + t.Fatalf("oversize loop must not invert the percentage: %d", got) + } + if got := UsableFreePercent(100*gib, 100*gib, 40*gib); got != 100 { + t.Fatalf("available covering usable total = %d, want 100", got) + } +} diff --git a/internal/observabilityrules/rules_test.go b/internal/observabilityrules/rules_test.go index cfb8e341..54f33772 100644 --- a/internal/observabilityrules/rules_test.go +++ b/internal/observabilityrules/rules_test.go @@ -29,7 +29,7 @@ func TestRepositoryRulesUseCurrentMetricSemantics(t *testing.T) { "provider_retry_error_persistent": `gha_fleet_provider_retry_deferred_records_by_error_class{error_class=~"identity|intent|provider|timeout|unknown"}`, "compute_pressure_observer_missing": `count(up{service_name="pressure-state"} == 1)`, "compute_pressure_state_stale": "gha_fleet_pressure_observer_up", - "compute_root_disk_low": "system_filesystem_usage", + "compute_root_disk_low": "gha_fleet_host_root_free_percent", "kernel_slab_unreclaimable": "gha_fleet_host_slab_unreclaimable_attributed_bytes", "audit_suppression_burst": `signal_class="audit_suppressed"`, "kernel_workqueue_hog": `signal_class="kernel_workqueue_hog"`,