From de6ebd1e59f41495137aec4dcc7fe0ab02c8bd01 Mon Sep 17 00:00:00 2001 From: Tiberiu Socaci Date: Thu, 10 Sep 2026 15:56:57 +0300 Subject: [PATCH] feat: add host system health monitoring to admin dashboard Signed-off-by: Tiberiu Socaci --- AGENTS.md | 10 +- CHANGELOG.md | 3 + FEATURES.md | 17 ++ TEST-PLAN.md | 42 +++++ docs/OPERATIONS.md | 51 ++++++ public/admin-routes.js | 1 + public/admin-system-health.js | 231 +++++++++++++++++++++++++++ public/app.js | 3 + public/index.html | 2 + public/styles.css | 87 ++++++++++ scripts/system-health-soak.mjs | 115 +++++++++++++ src/db/migrations.js | 25 +++ src/gateway/shutdown.js | 6 + src/gateway/system-health-collect.js | 100 ++++++++++++ src/gateway/system-health-store.js | 123 ++++++++++++++ src/gateway/system-health.js | 98 ++++++++++++ src/server.js | 4 + src/web/routes/admin.js | 2 + src/web/routes/system-health.js | 22 +++ test/admin-navigation.test.js | 1 + test/admin-system-health.test.js | 112 +++++++++++++ test/runtime-lifecycle.test.js | 3 +- test/system-health-api.test.js | 70 ++++++++ test/system-health.test.js | 216 +++++++++++++++++++++++++ 24 files changed, 1341 insertions(+), 3 deletions(-) create mode 100644 public/admin-system-health.js create mode 100644 scripts/system-health-soak.mjs create mode 100644 src/gateway/system-health-collect.js create mode 100644 src/gateway/system-health-store.js create mode 100644 src/gateway/system-health.js create mode 100644 src/web/routes/system-health.js create mode 100644 test/admin-system-health.test.js create mode 100644 test/system-health-api.test.js create mode 100644 test/system-health.test.js diff --git a/AGENTS.md b/AGENTS.md index 38e2cb1..e85fd28 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,7 +76,7 @@ post/edit the reply in the thread (degraded to the surface's capabilities) → u the changed keys), `dead-fields.js` (retired fields stripped on every write). - `src/db/` — `index.js` (the one lazy `node:sqlite` connection: WAL, `busy_timeout`, `foreign_keys`, migrations on open, the one-time legacy JSON import behind `_meta` flags), - `migrations.js` (versioned on `PRAGMA user_version`, currently 24 — append, never edit), + `migrations.js` (versioned on `PRAGMA user_version`, currently 25 — append, never edit), `import-legacy.js`, `fts.js` (the optional FTS5 `channel_memory_fts` index; without FTS5 memory search degrades to a scan). - `src/gateway/run.js` — the run orchestrator: engine adapter selection and precedence (per-run @@ -169,6 +169,11 @@ post/edit the reply in the thread (degraded to the surface's capabilities) → u stays available to every operator): one durable transaction and lock, a detached built-ins-only runner (preflight, smoke, snapshot, install → restart → verify, automatic rollback), results on `/api/health`; `restart_gateway` drains turns, jobs, API runs and update transactions first. +- `src/gateway/system-health.js` + `system-health-collect.js` + `system-health-store.js` — + daemon-side read-only Linux metrics, five-second samples, minute SQLite resource aggregates + (30 days), storage history (186 days, hourly after 30 days), hardware snapshots/change events, + and a data-dependent capacity forecast. Admin routes in `src/web/routes/system-health.js`, + browser page in `public/admin-system-health.js`, isolated canary in `scripts/system-health-soak.mjs`. - Smaller gateway modules: `sessions.js` + `thread-engine.js` (thread key ↔ engine session, per-thread pins), `session-adopt.js` + `session-carry.js` (`/resume` of a local session whose cwd is this channel; host → container carry-over), `stopped-turns.js`, `active-runs.js` @@ -307,7 +312,8 @@ through the control MCP. `skill_revision_files`, `skill_sources`, `skill_templates`, `skill_usage`, `skill_proposals`, `skill_access_tokens`); Composio SDK (`composio_sessions`); licensing (`license_usage`); dashboard data (`usage`, `usage_components`, `usage_requests`, `usage_repair_batches`, - `events` — typed, indexed columns for day/week/month/channel/user rollups); plus `_meta` + `events` — typed, indexed columns for day/week/month/channel/user rollups); system telemetry + (`system_health_resources`, `system_health_storage`, `system_health_hardware`, `system_health_changes`); plus `_meta` (key/value, created at open) and the optional FTS5 `channel_memory_fts`. Config-shaped rows (`channel_meta`, `users`, `bg_jobs`) keep their full record in a JSON `data` blob so every field survives without a migration; `dead-fields.js` strips retired fields on write. diff --git a/CHANGELOG.md b/CHANGELOG.md index bd573e3..d282ead 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog — ChannelGate +- Add System health to the admin UI with real CPU/RAM/load and storage metrics, historical + charts and peaks, storage warnings and capacity estimates, and refreshed hardware inventory. + - Fix MCP configuration selection when a plugin package supplies both engine manifests. Each engine uses its own inline declarations or referenced files, avoiding wrong endpoints and duplicate server errors. diff --git a/FEATURES.md b/FEATURES.md index 15619b3..97cd244 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -1,5 +1,22 @@ # ChannelGate — Features +## System health + +The last admin navigation item, **System health** (`/system-health`), shows daemon-side Linux +CPU, RAM, swap, filesystem capacity and system load. The collector runs every five seconds +independently of open browsers; the page supports pause, refresh and Live/1h/24h/7d/30d ranges. +Minute aggregates retain resource averages and peaks for 30 days. Storage history lasts 186 days, +with older samples reduced to hourly aggregates. The capacity forecast reports insufficient +history until an observed trend supports it; no historical points are generated to fill gaps. +Storage warnings start at 85%, critical at 95%. Peaks and collection status precede hardware. + +Hardware inventory refreshes at startup, every five minutes and on manual refresh, with a bounded +change log. Unavailable hardware fields stay unknown; temperatures, serial numbers, credentials, +and hardware-management writes are excluded. Metrics use the filesystem containing the gateway +runtime root, and the collector runs in the host daemon without widening any engine container's +mounts. All metrics endpoints require the existing admin session. See `docs/OPERATIONS.md` and +the engine-independent System health acceptance gates in `TEST-PLAN.md`. + ## Plugin packages through skill sources Git and folder skill sources recognize Claude/Codex plugin manifests and store a complete package diff --git a/TEST-PLAN.md b/TEST-PLAN.md index 02709f5..3a4482c 100644 --- a/TEST-PLAN.md +++ b/TEST-PLAN.md @@ -1,5 +1,47 @@ # ChannelGate — Test Plan +## System health — engine-independent acceptance + +These cases exercise the daemon collector and authenticated browser, not an engine turn; +Claude/Codex selection cannot affect collection, authorization or rendering. +Automated regressions: `test/system-health.test.js`, `test/system-health-api.test.js`, +`test/admin-system-health.test.js`, and the admin navigation suite. + +- **SYS-01 — live and historical metrics.** Start a disposable Linux admin instance with an empty + runtime root and an admin password. Open `/system-health` after login. CPU's first delta may be + unknown; within two samples require current CPU/RAM/load and filesystem capacity from that + instance. Compare RAM against `/proc/meminfo`, load against `/proc/loadavg`, and capacity against + the runtime root's filesystem. Wait across a minute boundary; reload and restart the fixture. + Require persisted history and real peaks, no invented pre-install points. Select every range, + pause/resume, manually refresh, leave the page and hide the tab. Network evidence must show + five-second polling only while active, visible and unpaused, without overlapping/stale rendering. +- **SYS-02 — history, retention and forecast.** Use the deterministic collector test fixtures + with data spanning at least 187 days and a capacity change. Require resources older than 30 days + purged, storage retained for 186 days with old minute samples downsampled, bounded chart responses + and preserved peaks. Empty/short history must not predict a date. A changed filesystem capacity + must not create a false growth forecast; missing samples must remain gaps. +- **SYS-03 — hardware, error and responsive states.** In a disposable fixture, change synthetic + memory/disk inventory, advance the five-minute clock, then manually refresh. Require new values + and one change event, no duplicate event for an unchanged snapshot. Missing DMI/device files + must appear unavailable. In Chromium at 1440px and 430px verify cards, keyboard chart tooltips, + ranges, warning/critical states and peaks/Collection before hardware, without horizontal page + overflow. Simulate a failed request: require a visible error/stale state and successful retry. +- **SYS-04 — boundary.** With a password absent, expect 403 for all metrics routes; with a password + configured but no session, expect 401, also for a valid run-API key. A logged-in admin can read + metrics; manual refresh requires the CSRF header and allowed Origin. Public `/api/health` must + not expose metrics/hardware. No endpoint accepts arbitrary filesystem paths or shell commands. +- **SYS-05 — 24-hour host canary (release/deployment gate).** Run the isolated soak procedure in + `docs/OPERATIONS.md` on the target Linux host for at least 24 hours. Record candidate commit, + actual elapsed time, collector errors, CPU, RSS, database plus WAL size, persisted sample count, + gaps and hardware visibility. The original design's CPU/RAM/I/O figures are estimates, not pass + evidence. Require no collection/persistence errors, retained samples across restart, bounded + memory/history, and metrics storage below the 100 MiB safety budget. Review measured overhead + before enabling on the served instance. After activation, verify authenticated page and new + samples on the real daemon. A shorter container smoke is not a host soak pass. + +Private deployment QA records and the 24-hour host canary must be recorded separately; an +unexecuted or unavailable check is not passing evidence. + ## Plugin source packages — acceptance gates Automated coverage: `test/skills-plugin-import.test.js`, `test/plugin-runtime.test.js`, diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 01bd93f..314a0cb 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -558,6 +558,57 @@ groups) so interrupted execution is recorded for reconciliation on the next boot turn may be recorded as a plain error before orderly state persistence. Existing installs: add `KillMode=mixed` under `[Service]` and `systemctl --user daemon-reload`. +## System health + +Open **System health**, the last admin navigation item (`/system-health`). The host daemon reads +Linux `/proc` and `/sys` without elevated privileges or subprocesses. It samples every five +seconds even when no browser is open; browsers poll while visible and active. Pause affects only +the page. Refresh also rescans hardware. CPU uses counter deltas (the first sample is unknown), +RAM uses `MemAvailable`, and load is the Linux 1/5/15-minute load average. Storage measures the +filesystem containing the configured gateway runtime root, not the sum of disks in the hardware +inventory. Used bytes exclude free blocks; available bytes exclude filesystem-reserved blocks; +the usage percentage is used / (used + available), as for an unprivileged process. +References: [Linux proc documentation](https://www.kernel.org/doc/html/v6.2/filesystems/proc.html) +and [Node filesystem statistics](https://nodejs.org/docs/latest-v22.x/api/fs.html#class-fsstatfs). + +Resource minute averages and peaks stay for 30 days. Storage stays for 186 days (at least six +calendar months); samples older than 30 days are reduced to hourly observations with preserved +peaks. Responses reduce long resource series to about 240 buckets and storage to daily points. +Empty history and missing metrics are reported without simulated values. Forecasts require at +least seven days of measured history and use only the segment since the last observed capacity +change. The estimate assumes continued growth at the fitted rate, not a guarantee of free space. + +Hardware is read at startup, every five minutes and on refresh. The current snapshot and at most +100 change events are retained; no temperatures, serial numbers, MAC addresses or machine IDs are +collected. DIMM information and some device details may be unavailable without host interfaces; +the page reports unknowns. RAM/disks recognized by Linux appear at the next scan. Distribution +changes appear on rescan; a newly installed kernel appears only after booting it. Hardware details +remain behind the admin session and never appear on the public liveness endpoint. + +Metrics use tables in the existing SQLite database. Retention applies only to these tables, +not audit/usage records. Do not impose a 100 MiB limit on the entire gateway database or force +global WAL truncation: other gateway records and readers share it. Measure incremental metrics +storage using an isolated canary, with 100 MiB for its database plus WAL as a safety budget. + +Before enabling on a served host, run the canary **on that Linux host** from the tested checkout: + +```sh +node scripts/system-health-soak.mjs --output /absolute/new-health-soak-report.json --storage-path /path/to/gateway-runtime-root +``` + +It runs in the foreground for 24 hours, reads the target filesystem, and writes metrics only to +a fresh temporary database. The JSON report updates every 30 seconds and records actual elapsed +time, errors, gaps, CPU, peak RSS, database plus WAL bytes, hardware visibility and persistence +after reopening. The report path must be new; the scratch database is removed at completion. +Use the service manager or a daemon-owned job to keep it alive when a session cannot remain open. +`--duration-seconds 65` is a quick smoke; it explicitly cannot pass the 24-hour gate. A report +with `containerDetected: true` describes the container's visible interfaces and is not evidence +of a host deployment. Even when false, independently verify where the process ran. + +Review the measured overhead rather than treating the prototype estimates as guarantees. After +the host canary passes, use the normal serialized beta landing and safe restart procedure, log in, +and verify current samples, hardware visibility and history persistence on the actual daemon. + ## Retention and log rotation Run `npm run maintenance` daily from the service manager. `CG_RETENTION_DAYS` defaults to 30 and diff --git a/public/admin-routes.js b/public/admin-routes.js index e423edb..7f35d80 100644 --- a/public/admin-routes.js +++ b/public/admin-routes.js @@ -9,6 +9,7 @@ export const ADMIN_VIEWS = Object.freeze({ skills: Object.freeze({ path: "/skills", title: "Skills" }), api: Object.freeze({ path: "/api-docs", title: "HTTP run API" }), settings: Object.freeze({ path: "/settings", title: "Settings" }), + "system-health": Object.freeze({ path: "/system-health", title: "System health" }), }); export const ADMIN_VIEW_PATHS = Object.freeze(Object.values(ADMIN_VIEWS).map(({ path }) => path)); diff --git a/public/admin-system-health.js b/public/admin-system-health.js new file mode 100644 index 0000000..194c76f --- /dev/null +++ b/public/admin-system-health.js @@ -0,0 +1,231 @@ +import { api } from "./admin-api.js"; +import { escapeHtml as esc } from "./admin-view.js"; + +const RANGES = ["live", "1h", "24h", "7d", "30d"]; +const finite = (value) => typeof value === "number" && Number.isFinite(value); +const number = (value, digits = 1) => finite(value) ? value.toFixed(digits) : "—"; +const percent = (value) => finite(value) ? `${number(value)}%` : "—"; +const time = (value) => value && Number.isFinite(new Date(value).getTime()) ? new Date(value).toLocaleString() : "—"; +export function healthBytes(value) { + if (!finite(value)) return "—"; + const unit = value >= 1024 ** 4 ? 4 : value >= 1024 ** 3 ? 3 : value >= 1024 ** 2 ? 2 : value >= 1024 ? 1 : 0; + return `${number(value / 1024 ** unit)} ${["B", "KiB", "MiB", "GiB", "TiB"][unit]}`; +} +export function storageSeverity(value) { return !finite(value) ? "unknown" : value >= 95 ? "critical" : value >= 85 ? "warning" : "normal"; } + +// One cancellable batch at a time. Navigation, visibility, pause and range changes invalidate +// earlier responses even if the transport ignores AbortSignal. Timers begin after completion. +export function createHealthPoller({ request = api, onUpdate, visible = () => !document.hidden, schedule = setTimeout, unschedule = clearTimeout, now = Date.now }) { + let active = false, paused = false, range = "live", generation = 0, timer = null, flight = null, controller = null, pending = false, manual = false, lastFull = 0; + let data = { current: null, history: null, storage: null, hardware: null, errors: {} }; + const publish = () => onUpdate({ ...data, range, paused, loading: !!flight }); + function invalidate() { + generation++; + unschedule(timer); + timer = null; + controller?.abort(); + } + async function run() { + if (!active || !visible() || flight || (!pending && paused)) return; + pending = false; + const version = generation, selectedRange = range, refreshHardware = manual; + manual = false; + const full = refreshHardware || !lastFull || now() - lastFull >= 60_000; + controller = new AbortController(); + const options = { signal: controller.signal }; + const jobs = [["current", "/api/system-health/current", options]]; + if (full) { + jobs.push(["history", `/api/system-health/history?range=${selectedRange}`, options]); + jobs.push(["storage", "/api/system-health/storage", options]); + jobs.push(["hardware", `/api/system-health/hardware${refreshHardware ? "/refresh" : ""}`, { ...options, ...(refreshHardware ? { method: "POST" } : {}) }]); + } + flight = Promise.allSettled(jobs.map(([, url, opts]) => Promise.resolve().then(() => request(url, opts)))); + publish(); + const results = await flight; + flight = null; + if (version === generation && active && visible()) { + const errors = { ...data.errors }; + results.forEach((result, index) => { + const key = jobs[index][0]; + if (result.status === "fulfilled") { data[key] = result.value; delete errors[key]; } + else if (result.reason?.name !== "AbortError") errors[key] = result.reason?.message || "Request failed"; + }); + data.errors = errors; + if (full && results.every((result) => result.status === "fulfilled")) lastFull = now(); + publish(); + } + if (pending) { run(); return; } + if (version !== generation && active && visible()) publish(); + if (active && visible() && !paused) timer = schedule(run, 5_000); + } + function reload({ hardware = false, full = true } = {}) { + invalidate(); + if (full) lastFull = 0; + pending = true; + manual ||= hardware; + run(); + } + return { + enter() { active = true; reload(); }, + leave() { active = false; pending = false; manual = false; invalidate(); }, + visibilityChanged() { invalidate(); if (active && visible() && !paused) { pending = true; lastFull = 0; run(); } }, + togglePause() { paused = !paused; pending = false; manual = false; invalidate(); publish(); if (!paused) reload(); }, + setRange(value) { if (!RANGES.includes(value) || range === value) return; range = value; data.history = null; publish(); reload(); }, + refresh() { reload({ hardware: true }); }, + }; +} + +function meter(value, tone = "") { return `
`; } +function metric(label, value, detail, meterValue, tone = "", state = "") { + return `
${label}${esc(state)}
${esc(value)}
${esc(detail)}
${meter(meterValue, tone)}
`; +} +function pairs(rows) { return `
${rows.map(([label, value]) => `
${esc(label)}
${esc(value ?? "—")}
`).join("")}
`; } +export function forecastCopy(forecast) { + if (!forecast || ["insufficient_data", "insufficient_history"].includes(forecast.status)) return { title: "Building the storage trend", detail: "Not enough measured history for a capacity forecast yet." }; + if (forecast.status === "full") return { title: "Storage is full", detail: "Free space or increase capacity." }; + if (finite(forecast.daysToFull)) return { title: `About ${Math.max(0, Math.round(forecast.daysToFull))} days to capacity`, detail: `${healthBytes(forecast.bytesPerDay)} growth per day over ${number(forecast.observedDays, 0)} measured days. Estimate assumes the same growth and capacity.` }; + if (["stable", "declining", "no_growth"].includes(forecast.status)) return { title: "No projected capacity limit", detail: `Measured usage is stable or declining across ${number(forecast.observedDays, 0)} days. Future growth can change this.` }; + return { title: "Forecast unavailable", detail: "A reliable projection is not available for the collected history." }; +} + +// Time is the horizontal scale: irregular samples and missing observations never become +// evenly spaced synthetic history. Missing metrics break paths instead of plotting zero. +export function healthChart(points = [], { storage = false, start, end } = {}) { + const usable = points.filter((p) => finite(p.timestamp)).sort((a, b) => a.timestamp - b.timestamp); + if (!usable.length) return '

No observations in this period yet.

'; + const from = finite(start) ? start : usable[0].timestamp; + const to = finite(end) && end > from ? end : Math.max(from + 1, usable.at(-1).timestamp); + const width = 720, height = storage ? 170 : 260, left = 38, right = 12, top = 14, bottom = 24; + const x = (t) => left + (t - from) / (to - from) * (width - left - right); + const y = (v) => top + (1 - Math.min(100, Math.max(0, v)) / 100) * (height - top - bottom); + const fields = storage ? [["storagePercent", "disk", "Storage"]] : [["cpuPercent", "cpu", "CPU"], ["memoryPercent", "memory", "RAM"]]; + const grid = [0, 25, 50, 75, 100].map((v) => `${v}%`).join(""); + const paths = fields.map(([field, cls]) => { + let pen = false, previous = null; + const typicalGap = storage ? 3 * 86_400_000 : Math.max(finite(start) ? 120_000 : 15_000, (to - from) / 120); + const path = usable.map((point) => { + if (!finite(point[field])) { pen = false; return ""; } + const move = !pen || previous !== null && point.timestamp - previous > typicalGap; + pen = true; previous = point.timestamp; + return `${move ? "M" : "L"}${x(point.timestamp).toFixed(2)},${y(point[field]).toFixed(2)}`; + }).join(" "); + return ``; + }).join(""); + const targets = usable.map((point) => { + const label = `${time(point.timestamp)} · ${fields.map(([field, , name]) => `${name} ${percent(point[field])}`).join(" · ")}${storage ? ` · ${healthBytes(point.storageUsedBytes)} used` : ""}`; + const dots = fields.filter(([field]) => finite(point[field])).map(([field, cls]) => ``).join(""); + return `${esc(label)}${dots}`; + }).join(""); + return `${grid}${paths}${targets}
${esc(new Date(from).toLocaleDateString())} ${storage ? "" : esc(new Date(from).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" }))}${esc(new Date(to).toLocaleDateString())} ${storage ? "" : esc(new Date(to).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" }))}
`; +} + +function installTooltip(host, points, storage) { + const svg = host.querySelector("svg"), tip = host.querySelector(".sh-tooltip"); + if (!svg || !tip || !points?.length) return; + const targets = [...svg.querySelectorAll("g")]; + targets.forEach((target, i) => { + const show = (event) => { + tip.textContent = target.querySelector("title").textContent; + tip.hidden = false; + const rect = host.getBoundingClientRect(); + tip.style.left = `${Math.max(0, Math.min(rect.width - Math.min(220, rect.width), (event.clientX || rect.left + rect.width / 2) - rect.left))}px`; + tip.style.top = "20px"; + }; + target.addEventListener("pointermove", show); + target.addEventListener("pointerleave", () => { tip.hidden = true; }); + // One focusable chart; arrows navigate observations without hundreds of tab stops. + if (i === 0) { + svg.setAttribute("tabindex", "0"); + svg.setAttribute("aria-label", `${storage ? "Storage" : "CPU and RAM"} usage. Use left and right arrows for observations.`); + } + }); + let selected = targets.length - 1; + svg.addEventListener("keydown", (event) => { + if (!["ArrowLeft", "ArrowRight", "Escape"].includes(event.key)) return; + event.preventDefault(); + if (event.key === "Escape") { tip.hidden = true; return; } + selected = Math.max(0, Math.min(targets.length - 1, selected + (event.key === "ArrowLeft" ? -1 : 1))); + tip.textContent = targets[selected].querySelector("title").textContent; + tip.hidden = false; tip.style.left = "38px"; tip.style.top = "20px"; + }); + svg.addEventListener("blur", () => { tip.hidden = true; }); +} + +function hardwareMarkup(data) { + if (!data?.snapshot) return '

Hardware inventory has not been collected yet.

'; + const h = data.snapshot; + // Hardware values are escaped at the display boundary; missing optional Linux interfaces + // are explicit unknowns and never replaced with the prototype's server specification. + const groups = [ + ["Processor", h.cpuModel, [["Architecture", h.architecture], ["Sockets", h.sockets], ["Physical cores", h.physicalCores], ["Logical CPUs", h.logicalCpus]]], + ["Memory", healthBytes(h.memoryTotalBytes), [["Swap", healthBytes(h.swapTotalBytes)], ["DIMM details", "Not collected"]]], + ["Operating system", h.osName, [["Kernel", h.kernel], ["Filesystem", h.filesystem]]], + ["Mainboard", h.boardName, [["Vendor", h.boardVendor], ["BIOS", [h.biosVendor, h.biosVersion].filter(Boolean).join(" ")]]], + ]; + return `
${groups.map(([label, model, rows]) => `
${label}${esc(model || "Not exposed by Linux")}${pairs(rows)}
`).join("")}
+

Storage devices

${h.disks?.length ? h.disks.map((d) => `
${esc(d.model || d.name)}${esc([d.name, healthBytes(d.sizeBytes), d.rotational === true ? "HDD" : d.rotational === false ? "SSD" : null].filter(Boolean).join(" · "))}
`).join("") : '

No physical disk details exposed by Linux.

'}

Network & graphics

${esc(h.network?.map((n) => typeof n === "string" ? n : n.name).join(", ") || "Network inventory unavailable")}

${esc(h.gpus?.map((g) => typeof g === "string" ? g : [g.vendor, g.device, g.driver].filter(Boolean).join(" ")).join(", ") || "Graphics inventory unavailable")}

+

Scanned ${esc(time(data.collectedAt))}. Inventory refreshes every 5 minutes and on manual refresh. Only devices exposed by Linux are shown.

+ ${data.changes?.length ? `
Recent hardware changes (${data.changes.length})
    ${data.changes.map((change) => `
  • ${esc(time(change.timestamp))} · ${esc(change.fields?.join(", "))}
    • `).join("")}
` : '

No hardware changes recorded.

'}`; +} + +let view = null; +export function setSystemHealthActive(active) { + // Async admin bootstrap can finish after the user has already navigated elsewhere. + active = active && document.getElementById("view-system-health")?.classList.contains("active"); + if (!view && active) view = mountSystemHealth(document.getElementById("view-system-health")); + if (active) view?.enter(); else view?.leave(); +} +function mountSystemHealth(root) { + root.innerHTML = `

Infrastructure

System health

Live host resources, 30-day performance history, and six months of storage trends.

Connecting…
+
+
${RANGES.map((range) => ``).join("")}
Waiting for the first sample…
+

Resource usage

CPURAM

Storage trend

Last six months · recorded observations

Used
Capacity forecast

+

Selected-period peaks

Collection

+

Server hardware

Inventory detected on the gateway host

`; + const $ = (selector) => root.querySelector(selector); + function paint(state) { + const current = state.current, sample = current?.sample, collection = current?.collection; + const stale = !!sample && Date.now() - sample.timestamp > Math.max(15_000, (collection?.sampleIntervalMs || 5_000) * 3); + const errors = Object.entries(state.errors).map(([key, error]) => `${key}: ${error}`); + if (collection?.error) errors.push(`Collector: ${collection.error}`); + if (stale) errors.push("The last sample is stale. Displayed values are from the timestamp below."); + $("#sh-error").hidden = !errors.length; + $("#sh-error").textContent = errors.join(" · "); + $("#sh-status").textContent = state.paused ? "Paused" : errors.length ? "Updates interrupted" : sample ? "Live · every 5s" : state.loading ? "Connecting…" : "Awaiting first sample"; + $("#sh-status").classList.toggle("sh-status-warn", state.paused || !!errors.length); + $("#sh-pause").textContent = state.paused ? "Resume" : "Pause"; + $("#sh-pause").setAttribute("aria-pressed", String(state.paused)); + $("#sh-refresh").disabled = state.loading; + const severity = storageSeverity(sample?.storagePercent); + $("#sh-metrics").innerHTML = metric("CPU usage", percent(sample?.cpuPercent), `${sample?.logicalCpus ?? "—"} logical CPUs`, sample?.cpuPercent) + + metric("Memory", percent(sample?.memoryPercent), `${healthBytes(sample?.memoryUsedBytes)} of ${healthBytes(sample?.memoryTotalBytes)}`, sample?.memoryPercent, "sh-memory") + + metric("Storage", percent(sample?.storagePercent), `${healthBytes(sample?.storageUsedBytes)} used · ${healthBytes(sample?.storageAvailableBytes)} available`, sample?.storagePercent, `sh-${severity}`, severity === "unknown" ? "" : severity) + + metric("System load", number(sample?.load1, 2), `${number(sample?.load5, 2)} / ${number(sample?.load15, 2)} at 5 / 15 min`, sample?.logicalCpus ? sample.load1 / sample.logicalCpus * 100 : null); + const warning = ["warning", "critical"].includes(severity); + $("#sh-storage-alert").hidden = !warning; + $("#sh-storage-alert").classList.toggle("sh-critical", severity === "critical"); + $("#sh-storage-alert").textContent = warning ? `Storage ${severity === "critical" ? "critical" : "above 85%"} · ${percent(sample.storagePercent)} used on ${sample.storagePath || "the gateway filesystem"}. ${healthBytes(sample.storageAvailableBytes)} available. Review disk usage and plan capacity.` : ""; + $("#sh-updated").textContent = sample ? `Sample: ${time(sample.timestamp)}${state.paused ? " · paused" : ""}` : "Waiting for the first sample…"; + root.querySelectorAll("[data-range]").forEach((button) => button.setAttribute("aria-pressed", String(button.dataset.range === state.range))); + const points = state.range === "live" ? current?.recent || [] : state.history?.points || []; + const history = state.range === "live" ? {} : state.history || {}; + $("#sh-period").textContent = state.range === "live" ? "Recent live samples · 5-second observations" : `${state.range} period · aggregated observations`; + $("#sh-resource-chart").innerHTML = healthChart(points, history); + installTooltip($("#sh-resource-chart"), points, false); + $("#sh-storage-chart").innerHTML = healthChart(state.storage?.points || [], { ...state.storage, storage: true }); + installTooltip($("#sh-storage-chart"), state.storage?.points, true); + const forecast = forecastCopy(state.storage?.forecast); + $("#sh-forecast-title").textContent = forecast.title; + $("#sh-forecast-detail").textContent = forecast.detail; + const peaks = state.range === "live" ? Object.fromEntries(["cpuPercent", "memoryPercent", "load1", "storagePercent"].map((field) => { const values = points.map((p) => p[field]).filter(finite); return [field, values.length ? Math.max(...values) : null]; })) : state.history?.peaks; + $("#sh-peaks").innerHTML = pairs([["CPU", percent(peaks?.cpuPercent)], ["RAM", percent(peaks?.memoryPercent)], ["System load (1 min)", number(peaks?.load1, 2)], ["Storage", percent(peaks?.storagePercent)]]); + $("#sh-collection").innerHTML = pairs([["Live sampling", collection ? `${number(collection.sampleIntervalMs / 1000, 0)} seconds` : "—"], ["Persisted aggregates", collection ? `${number(collection.aggregateIntervalMs / 1000, 0)} seconds` : "—"], ["Resource retention", collection ? `${collection.resourceRetentionDays} days` : "—"], ["Storage retention", collection ? `${collection.storageRetentionDays} days` : "—"], ["Last persisted", time(collection?.lastPersistedAt)], ["Swap in use", `${healthBytes(sample?.swapUsedBytes)} / ${healthBytes(sample?.swapTotalBytes)}`]]); + $("#sh-hardware-body").innerHTML = hardwareMarkup(state.hardware); + } + const controller = createHealthPoller({ onUpdate: paint }); + $("#sh-pause").addEventListener("click", controller.togglePause); + $("#sh-refresh").addEventListener("click", controller.refresh); + root.querySelectorAll("[data-range]").forEach((button) => button.addEventListener("click", () => controller.setRange(button.dataset.range))); + document.addEventListener("visibilitychange", controller.visibilityChanged); + return controller; +} diff --git a/public/app.js b/public/app.js index 2ddd762..8cb17b9 100644 --- a/public/app.js +++ b/public/app.js @@ -15,6 +15,7 @@ import { } from "./admin-state.js"; import { activeSectionFor, filterSettings } from "./admin-settings-search.js"; import { api } from "./admin-api.js"; +import { setSystemHealthActive } from "./admin-system-health.js"; import { attachReveal, confirmDialog, escapeHtml, infoDialog, openDialog, paintReveal, passwordDialog, revealSecret, tokenValue } from "./admin-view.js"; import { loadSkills } from "./admin-skills.js"; import { mountSkillAssignmentPicker } from "./skill-assignment-picker.js"; @@ -275,6 +276,7 @@ function capLabelOf(m = {}) { // ── View switching + URL history ──────────────────────────────────────────────── function loadView(name) { + if (name === "system-health") setSystemHealthActive(true); if (name === "channels" && !viewLoaded.channels) { viewLoaded.channels = true; loadConversations().catch(() => {}); } if (name === "users" && !viewLoaded.users) { viewLoaded.users = true; loadUsers().catch(() => {}); } if (name === "settings" && !viewLoaded.settings) { viewLoaded.settings = true; loadSettings().catch(() => {}); } @@ -300,6 +302,7 @@ function setView(name, { history = "push", load = true } = {}) { const path = pathForView(view); if (history === "replace") window.history.replaceState({ view }, "", path); else if (history === "push" && window.location.pathname !== path) window.history.pushState({ view }, "", path); + if (view !== "system-health") setSystemHealthActive(false); if (load) loadView(view); if (view === "settings") enterSettingsView(hash); } diff --git a/public/index.html b/public/index.html index e7904a7..8af1580 100644 --- a/public/index.html +++ b/public/index.html @@ -27,6 +27,7 @@ Skills API Settings + System health
Loading…
@@ -37,6 +38,7 @@
+
diff --git a/public/styles.css b/public/styles.css index 691c29b..7e780a5 100644 --- a/public/styles.css +++ b/public/styles.css @@ -1066,3 +1066,90 @@ button.clear-tok.armed { background: rgba(229, 96, 77, .14); border-color: rgba( .mode-options .togglerow { margin-bottom:10px; } @media (max-width:900px) { .mode-layout { grid-template-columns:1fr; } } @media (max-width:520px) { .mode-layout .cap-cards { grid-template-columns:1fr; } } + +/* System health: host telemetry inside the existing admin shell. */ +#view-system-health { max-width: 1500px; margin-inline: auto; } +.sh-head { align-items: flex-start; gap: 20px; } +.sh-eyebrow { margin: 0 0 5px; color: var(--orange); text-transform: uppercase; letter-spacing: .09em; font-size: 10px; font-weight: 600; } +.sh-actions, .sh-toolbar, .sh-card-head, .sh-metric-top { display: flex; align-items: center; justify-content: space-between; gap: 12px; } +.sh-actions { flex-wrap: wrap; justify-content: flex-end; } +.sh-actions button { white-space: nowrap; } +.sh-status { border: 1px solid rgba(78,199,122,.25); border-radius: 8px; padding: 7px 10px; color: #8de2aa; font-size: 11px; white-space: nowrap; background: rgba(78,199,122,.06); } +.sh-status-warn { color: var(--warn); border-color: rgba(232,176,75,.3); background: rgba(232,176,75,.06); } +.sh-metrics { display: grid; grid-template-columns: repeat(4,minmax(0,1fr)); gap: 12px; } +.sh-metric { min-width: 0; padding: 17px 18px 15px; background: var(--panel); border: 1px solid var(--line-soft); border-radius: 10px; } +.sh-metric-top { color: var(--muted); font-size: 10px; font-weight: 600; text-transform: uppercase; letter-spacing: .05em; } +.sh-metric-top small { font-size: 9px; font-weight: 400; } +.sh-metric-value { display: block; margin-top: 9px; font-size: 31px; font-weight: 600; line-height: 1.2; letter-spacing: -.035em; font-variant-numeric: tabular-nums; } +.sh-metric-detail { margin-top: 7px; color: var(--faint); font-size: 10px; overflow-wrap: anywhere; } +.sh-meter { height: 4px; margin-top: 12px; overflow: hidden; border-radius: 8px; background: var(--line-soft); } +.sh-meter span { display: block; height: 100%; background: var(--light); } +.sh-meter.sh-memory span { background: var(--orange); } +.sh-meter.sh-warning span { background: var(--warn); } +.sh-meter.sh-critical span { background: var(--danger); } +.sh-warning { color: var(--warn); } +.sh-critical { color: var(--danger); } +.sh-notice { margin: 12px 0; padding: 12px 14px; border: 1px solid rgba(232,176,75,.3); border-radius: 9px; background: rgba(232,176,75,.07); color: #f3c86c; font-size: 11px; overflow-wrap: anywhere; } +.sh-notice.sh-critical { border-color: rgba(229,96,77,.4); background: rgba(229,96,77,.08); color: #f79d90; } +.sh-notice[hidden] { display: none; } +.sh-toolbar { margin: 22px 0 12px; flex-wrap: wrap; } +.sh-ranges { display: flex; gap: 3px; padding: 3px; border: 1px solid var(--line-soft); border-radius: 8px; background: var(--panel-dark); } +.sh-ranges button { min-width: 51px; padding: 6px 10px; border: 0; border-radius: 6px; background: transparent; color: var(--faint); font-size: 11px; font-weight: 500; } +.sh-ranges button[aria-pressed=true] { background: var(--panel-2); color: var(--ink); box-shadow: inset 0 -2px 0 var(--orange); } +.sh-charts { display: grid; grid-template-columns: minmax(0,1.7fr) minmax(280px,1fr); gap: 12px; } +.sh-panel { min-width: 0; padding: 17px 18px 15px; border: 1px solid var(--line-soft); border-radius: 10px; background: var(--panel-dark); } +.sh-panel h3 { margin: 0 0 10px; font-size: 13px; font-weight: 600; } +.sh-card-head { align-items: flex-start; margin-bottom: 8px; } +.sh-card-head h3 { margin-bottom: 3px; } +.sh-muted { color: var(--faint); font-size: 10.5px; margin: 4px 0; overflow-wrap: anywhere; } +.sh-resource-panel { grid-row: span 2; } +.sh-legend { display: flex; align-items: center; gap: 14px; font-size: 10px; white-space: nowrap; } +.sh-legend span::before, .sh-legend.sh-disk::before { content: ''; display: inline-block; width: 10px; height: 2px; margin-right: 5px; background: currentColor; vertical-align: middle; } +.sh-cpu { color: var(--light); } +.sh-memory { color: var(--orange); } +.sh-disk { color: var(--warn); } +.sh-chart { position: relative; min-height: 220px; } +.sh-chart-small { min-height: 125px; } +.sh-chart-svg { display: block; width: 100%; height: 260px; overflow: visible; } +.sh-chart-small .sh-chart-svg { height: 150px; } +.sh-grid-line { stroke: var(--line-soft); stroke-width: 1; vector-effect: non-scaling-stroke; } +.sh-axis-label { fill: var(--faint); font: 9px var(--font); } +.sh-line { fill: none; stroke: currentColor; stroke-width: 1.8; stroke-linecap: round; stroke-linejoin: round; vector-effect: non-scaling-stroke; } +.sh-dot { fill: currentColor; } +.sh-axis-foot { display: flex; justify-content: space-between; gap: 8px; margin-left: 38px; color: var(--faint); font-size: 9px; } +.sh-chart-empty { display: flex; min-height: inherit; align-items: center; justify-content: center; color: var(--faint); font-size: 12px; text-align: center; padding: 20px; } +.sh-tooltip { position: absolute; z-index: 3; width: 220px; max-width: 100%; pointer-events: none; border: 1px solid var(--line); border-radius: 7px; background: var(--bg); box-shadow: 0 10px 30px rgba(0,0,0,.25); padding: 10px; font-size: 10px; color: var(--ink); } +.sh-tooltip[hidden] { display: none; } +.sh-forecast { display: flex; flex-direction: column; justify-content: center; } +.sh-kicker { display: block; color: var(--muted); font-size: 9.5px; font-weight: 600; letter-spacing: .07em; text-transform: uppercase; margin-bottom: 8px; } +.sh-detail-grid { display: grid; grid-template-columns: repeat(2,minmax(0,1fr)); gap: 12px; margin: 12px 0; } +.sh-facts { display: grid; grid-template-columns: minmax(0,1fr) minmax(0,1fr); gap: 7px 12px; margin: 0; } +.sh-facts dt { color: var(--faint); font-size: 10.5px; } +.sh-facts dd { margin: 0; text-align: right; font: 10.5px var(--mono); overflow-wrap: anywhere; } +.sh-hardware-grid { display: grid; grid-template-columns: repeat(4,minmax(0,1fr)); border: 1px solid var(--line-soft); border-radius: 9px; overflow: hidden; gap: 1px; background: var(--line-soft); } +.sh-hardware-group { min-width: 0; padding: 14px; background: #172d2d; } +.sh-hardware-group > strong { display: block; min-height: 40px; margin-bottom: 10px; font-size: 12px; font-weight: 500; overflow-wrap: anywhere; } +.sh-hardware-group .sh-facts { padding-top: 9px; border-top: 1px solid var(--line-soft); } +.sh-hardware-group .sh-facts dt, .sh-hardware-group .sh-facts dd { font-size: 9px; } +.sh-device-grid { display: grid; grid-template-columns: repeat(2,minmax(0,1fr)); gap: 20px; margin: 12px 0; } +.sh-device-grid h4 { color: var(--muted); margin: 0 0 8px; font-size: 11px; font-weight: 500; } +.sh-device { display: flex; flex-wrap: wrap; justify-content: space-between; gap: 5px 12px; padding: 5px 0; border-bottom: 1px solid var(--line-soft); overflow-wrap: anywhere; } +.sh-device strong { font-size: 10px; font-weight: 500; } +.sh-device span { color: var(--faint); font-size: 10px; } +.sh-changes { margin-top: 12px; color: var(--muted); font-size: 10px; } +.sh-changes li { margin: 5px 0; overflow-wrap: anywhere; } +@media (max-width: 1200px) { + .sh-metrics, .sh-hardware-grid { grid-template-columns: repeat(2,minmax(0,1fr)); } + .sh-charts { grid-template-columns: 1fr; } + .sh-resource-panel { grid-row: auto; } +} +@media (max-width: 760px) { + .sh-head { flex-direction: column; } + .sh-actions { justify-content: flex-start; gap: 8px; } + .sh-metrics, .sh-detail-grid, .sh-hardware-grid, .sh-device-grid { grid-template-columns: 1fr; } + .sh-ranges { width: 100%; } + .sh-ranges button { min-width: 0; flex: 1; } + .sh-card-head { flex-wrap: wrap; } + .sh-chart-svg { height: 215px; } + .sh-hardware-group > strong { min-height: 0; } +} diff --git a/scripts/system-health-soak.mjs b/scripts/system-health-soak.mjs new file mode 100644 index 0000000..bc5084f --- /dev/null +++ b/scripts/system-health-soak.mjs @@ -0,0 +1,115 @@ +#!/usr/bin/env node +// Foreground canary: a fresh scratch DB, actual Linux reads and measured process overhead. +// It never opens the serving gateway DB, changes hardware or launches an engine. +import { mkdtemp, mkdir, stat, writeFile, rename, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import { DatabaseSync } from "node:sqlite"; +import { runMigrations } from "../src/db/index.js"; +import { createLinuxHealthCollector } from "../src/gateway/system-health-collect.js"; +import { createSystemHealthStore } from "../src/gateway/system-health-store.js"; +import { createSystemHealth } from "../src/gateway/system-health.js"; + +const args = process.argv.slice(2); +const options = {}; +for (let i = 0; i < args.length; i += 2) { + if (!["--duration-seconds", "--output", "--storage-path"].includes(args[i]) || !args[i + 1]) { + throw new Error("Usage: node scripts/system-health-soak.mjs --output /absolute/report.json [--duration-seconds 86400] [--storage-path /path/on/target/filesystem]"); + } + options[args[i]] = args[i + 1]; +} +const seconds = Number(options["--duration-seconds"] ?? 86400); +if (!Number.isFinite(seconds) || seconds < 5 || seconds > 604800) throw new Error("Duration must be 5–604800 seconds"); +if (!options["--output"] || !path.isAbsolute(options["--output"])) throw new Error("An absolute --output report path is required"); +const output = options["--output"]; +await mkdir(path.dirname(output), { recursive: true }); +// Refuse to overwrite another canary's evidence. +await writeFile(output, "{}\n", { flag: "wx", mode: 0o600 }); +const scratch = await mkdtemp(path.join(tmpdir(), "cg-health-soak-")); +const file = path.join(scratch, "metrics.db"); +const storagePath = path.resolve(options["--storage-path"] || scratch); +const db = new DatabaseSync(file); +db.exec("PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000"); +runMigrations(db); +const startedAt = Date.now(), startedMono = performance.now(), startedCpu = process.cpuUsage(); +let samples = 0, lastSampleAt = null, maxGapMs = 0, peakRssBytes = 0, peakDatabaseWalBytes = 0; +let interrupted = false, finishing = false, reportTimer; +const errors = new Set(); +let containerDetected = false; +for (const name of ["/run/.containerenv", "/.dockerenv"]) { + if (await stat(name).then(() => true, () => false)) containerDetected = true; +} +const linux = createLinuxHealthCollector({ storagePath }); +const collector = { + hardware: linux.hardware, + collect: async () => { + const result = await linux.collect(); + samples++; + if (lastSampleAt != null) maxGapMs = Math.max(maxGapMs, result.sample.timestamp - lastSampleAt); + lastSampleAt = result.sample.timestamp; + if (result.error) errors.add(result.error); + return result; + }, +}; +const store = createSystemHealthStore(db); +const service = createSystemHealth({ collector, store }); +const bytes = async (name) => (await stat(name).catch(() => null))?.size || 0; +async function report(complete = false) { + const elapsedSeconds = (performance.now() - startedMono) / 1000; + const usage = process.cpuUsage(startedCpu); + peakRssBytes = Math.max(peakRssBytes, process.memoryUsage().rss, process.resourceUsage().maxRSS * 1024); + peakDatabaseWalBytes = Math.max(peakDatabaseWalBytes, await bytes(file) + await bytes(`${file}-wal`)); + if (service.current().collection.error) errors.add(service.current().collection.error); + const minuteRows = db.prepare("SELECT COUNT(*) AS n FROM system_health_resources").get().n; + const hardware = service.hardware(); + const completed24Hours = elapsedSeconds >= 86400; + const checksPassed = samples >= Math.floor(elapsedSeconds / 5) * 0.95 && maxGapMs <= 30000 && + errors.size === 0 && peakDatabaseWalBytes < 100 * 1024 ** 2 && !interrupted; + const result = { startedAt, completedAt: complete ? Date.now() : null, elapsedSeconds, + mode: seconds >= 86400 ? "soak" : "smoke", containerDetected, + scope: "Local process namespace; host execution must be independently verified", storagePath, + samples, minuteRows, maxGapMs, averageCpuPercentOfOneCore: (usage.user + usage.system) / (elapsedSeconds * 10000), + peakRssBytes, peakDatabaseWalBytes, errors: [...errors].slice(0, 10), interrupted, + completed24Hours, checksPassed, soakPassed: complete && completed24Hours && checksPassed, + hardware: hardware.snapshot, hardwareCollectedAt: hardware.collectedAt }; + await writeFile(`${output}.tmp`, `${JSON.stringify(result, null, 2)}\n`, { mode: 0o600 }); + await rename(`${output}.tmp`, output); + return result; +} +let reportActive = null; +function checkpoint() { + if (finishing || reportActive) return; + reportActive = report().catch(() => { errors.add("Canary report write failed"); }).finally(() => { reportActive = null; }); +} +try { + service.start(); + reportTimer = setInterval(checkpoint, 30000); + console.log(`System health ${seconds >= 86400 ? "soak" : "smoke"} started; foreground duration ${seconds}s; report ${output}`); + await new Promise((resolve) => { + const finishTimer = setTimeout(resolve, seconds * 1000); + const stop = () => { interrupted = true; clearTimeout(finishTimer); resolve(); }; + process.once("SIGINT", stop); + process.once("SIGTERM", stop); + }); + finishing = true; + clearInterval(reportTimer); + await reportActive; + await service.stop(); + const result = await report(true); + db.close(); + const reopened = new DatabaseSync(file); + result.restartPersistenceVerified = reopened.prepare("SELECT COUNT(*) AS n FROM system_health_resources").get().n === result.minuteRows; + reopened.close(); + if (!result.restartPersistenceVerified) { result.checksPassed = false; result.soakPassed = false; } + await writeFile(output, `${JSON.stringify(result, null, 2)}\n`, { mode: 0o600 }); + console.log(JSON.stringify({ complete: true, mode: result.mode, elapsedSeconds: result.elapsedSeconds, + samples: result.samples, minuteRows: result.minuteRows, checksPassed: result.checksPassed, + soakPassed: result.soakPassed, containerDetected, report: output })); + process.exitCode = result.checksPassed ? 0 : 1; +} finally { + clearInterval(reportTimer); + await service.stop(); + try { db.close(); } catch { /* already closed */ } + await rm(scratch, { recursive: true, force: true }); +} diff --git a/src/db/migrations.js b/src/db/migrations.js index 5e86f43..9564268 100644 --- a/src/db/migrations.js +++ b/src/db/migrations.js @@ -786,4 +786,29 @@ export const migrations = [ `); }, }, + { + version: 25, + up(db) { + db.exec(` + CREATE TABLE system_health_resources ( + timestamp INTEGER PRIMARY KEY, + cpu REAL, cpu_peak REAL, memory REAL, memory_peak REAL, load REAL, load_peak REAL, + storage REAL, storage_peak REAL, + cpu_count INTEGER NOT NULL, memory_count INTEGER NOT NULL, load_count INTEGER NOT NULL, storage_count INTEGER NOT NULL + ); + CREATE TABLE system_health_storage ( + timestamp INTEGER NOT NULL, resolution INTEGER NOT NULL, + used REAL, total REAL, available REAL, percent REAL, peak REAL, + PRIMARY KEY(timestamp, resolution) + ); + CREATE TABLE system_health_hardware ( + id INTEGER PRIMARY KEY CHECK(id = 1), collected_at INTEGER NOT NULL, snapshot TEXT NOT NULL + ); + CREATE TABLE system_health_changes ( + id INTEGER PRIMARY KEY, timestamp INTEGER NOT NULL, fields TEXT NOT NULL + ); + CREATE INDEX idx_system_health_changes_time ON system_health_changes(timestamp); + `); + }, + }, ]; diff --git a/src/gateway/shutdown.js b/src/gateway/shutdown.js index 7411a7f..36026fa 100644 --- a/src/gateway/shutdown.js +++ b/src/gateway/shutdown.js @@ -2,6 +2,7 @@ import { setTimeout as delay } from "node:timers/promises"; import { shutdownPool, poolStats } from "../engines/session-pool.js"; import { shutdownEngineChildren, forceKillEngineChildren, engineProcessStats } from "../engines/process-registry.js"; import { runQueue } from "../slack/message-lifecycle.js"; +import { stopSystemHealth } from "./system-health.js"; // Deferred import: src/ee/ is the proprietary licensing layer and shutdown.js is on the boot path // for every command-line entry point, including ones that never open the database. @@ -96,6 +97,7 @@ export async function performShutdown({ pollMs = DEFAULT_POLL_MS, killAfterMs = DEFAULT_KILL_AFTER_MS, disconnectTimeoutMs = DEFAULT_DISCONNECT_MS, + stopMonitoring = stopSystemHealth, getActivity = runtimeActivity, sweepWarm = shutdownPool, sweepCold = shutdownEngineChildren, @@ -133,6 +135,10 @@ export async function performShutdown({ // phase; after a timeout it also tells interrupted turns to preserve their durable recovery row. markForce(); + // Both signal stops and admin-requested restarts flush the last partial metrics minute. + // A stalled filesystem read must not hold up the process shutdown deadline. + await waitBounded(Promise.resolve().then(stopMonitoring).catch(() => {}), 1_000); + // Sweep even after a clean drain: idle persistent sessions still own detached Claude/MCP process // groups and must never be re-parented to PID 1 when the daemon exits. const cold = sweepCold({ killAfterMs }); diff --git a/src/gateway/system-health-collect.js b/src/gateway/system-health-collect.js new file mode 100644 index 0000000..f3e3972 --- /dev/null +++ b/src/gateway/system-health-collect.js @@ -0,0 +1,100 @@ +// Daemon-only read-only Linux telemetry. No subprocesses, device opens, serials or temperatures. +import * as fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +const number = (v) => v != null && v !== "" && Number.isFinite(Number(v)) ? Number(v) : null; +const label = (value) => value == null ? null : value.slice(0, 256); +const percent = (used, total) => used != null && total > 0 ? Math.max(0, Math.min(100, used / total * 100)) : null; +export function parseCpuStat(text) { + const values = text?.match(/^cpu\s+(.+)$/m)?.[1].trim().split(/\s+/).slice(0, 8).map(Number); + if (!values || values.length < 4 || values.some((v) => !Number.isFinite(v) || v < 0)) return null; + return { total: values.reduce((a, b) => a + b, 0), idle: values[3] + (values[4] || 0), counters: values }; +} +export function cpuUsage(previous, current) { + if (!previous || !current || current.total <= previous.total || current.idle < previous.idle) return null; + // Linux documents decreasing iowait counters; treat any reset as a new baseline. + if (current.counters?.some((v, i) => v < (previous.counters?.[i] ?? 0))) return null; + return percent(current.total - previous.total - (current.idle - previous.idle), current.total - previous.total); +} +export function parseMeminfo(text) { + const values = Object.fromEntries([...String(text || "").matchAll(/^(\w+):\s+(\d+)\s+kB$/gm)].map((m) => [m[1], Number(m[2]) * 1024])); + const total = values.MemTotal ?? null; + const used = total != null && values.MemAvailable != null ? Math.max(0, total - values.MemAvailable) : null; + const swap = values.SwapTotal ?? null; + return { memoryTotalBytes: total, memoryUsedBytes: used, memoryPercent: percent(used, total), swapTotalBytes: swap, + swapUsedBytes: swap != null && values.SwapFree != null ? Math.max(0, swap - values.SwapFree) : null }; +} +export function storageStats(stats, storagePath) { + const total = Number(stats.blocks) * Number(stats.bsize); + const free = Number(stats.bfree) * Number(stats.bsize); + const available = Number(stats.bavail) * Number(stats.bsize); + if (![total, free, available].every(Number.isFinite) || total <= 0) throw new Error("invalid filesystem statistics"); + const used = Math.max(0, total - free); + // Include reserved blocks in the denominator exactly like df's user-visible capacity. + return { storageUsedBytes: used, storageTotalBytes: total, storageAvailableBytes: Math.max(0, available), + storagePercent: percent(used, used + Math.max(0, available)), storagePath }; +} + +export function createLinuxHealthCollector({ io = fs, now = Date.now, storagePath, system = os } = {}) { + let previousCpu = null; + const read = async (file) => { try { return (await io.readFile(file, "utf8")).trim().slice(0, 262144); } catch { return null; } }; + const entries = async (dir) => { try { return (await io.readdir(dir)).sort().slice(0, 128); } catch { return []; } }; + const linkName = async (file) => { try { return path.basename(await io.readlink(file)); } catch { return null; } }; + async function collect() { + const [stat, mem, load, disk] = await Promise.all([read("/proc/stat"), read("/proc/meminfo"), read("/proc/loadavg"), + io.statfs(storagePath).then((value) => storageStats(value, storagePath)).catch(() => null)]); + const cpu = parseCpuStat(stat); + const memory = parseMeminfo(mem); + const loads = load?.split(/\s+/).slice(0, 3).map((value) => { const n = number(value); return n != null && n >= 0 ? n : null; }) || []; + const logicalCpus = stat?.match(/^cpu\d+\s/gm)?.length || null; + const errors = [!cpu && "CPU", memory.memoryUsedBytes == null && "memory", (loads.length !== 3 || loads.some((v) => v == null || v < 0)) && "load", !disk && "storage"].filter(Boolean); + const sample = { timestamp: now(), cpuPercent: cpuUsage(previousCpu, cpu), ...memory, + load1: loads[0] ?? null, load5: loads[1] ?? null, load15: loads[2] ?? null, logicalCpus, + storageUsedBytes: null, storageTotalBytes: null, storageAvailableBytes: null, storagePercent: null, storagePath, ...disk }; + previousCpu = cpu; + return { sample, error: errors.length ? `Unavailable telemetry: ${errors.join(", ")}` : null }; + } + async function hardware() { + const [cpuText, cpuStat, mem, osRelease, mounts, names, pciNames, networkNames, ...dmi] = await Promise.all([ + read("/proc/cpuinfo"), read("/proc/stat"), read("/proc/meminfo"), read("/etc/os-release"), read("/proc/self/mountinfo"), + entries("/sys/block"), entries("/sys/bus/pci/devices"), entries("/sys/class/net"), + ...["board_vendor", "board_name", "bios_vendor", "bios_version"].map((name) => read(`/sys/class/dmi/id/${name}`)), + ]); + const cpus = (cpuText || "").split(/\n\s*\n/).filter(Boolean).map((block) => Object.fromEntries(block.split("\n").filter((l) => l.includes(":")).map((l) => { const i = l.indexOf(":"); return [l.slice(0, i).trim(), l.slice(i + 1).trim()]; }))); + const logicalCpus = cpuStat?.match(/^cpu\d+\s/gm)?.length || null; + if (!logicalCpus && !mem) throw new Error("Host inventory sources unavailable"); + const completeTopology = logicalCpus != null && cpus.length === logicalCpus; + const sockets = new Set(cpus.map((c) => c["physical id"]).filter((v) => v != null)); + const cores = new Set(cpus.filter((c) => c["core id"] != null).map((c) => `${c["physical id"] ?? "0"}:${c["core id"]}`)); + const disks = await Promise.all(names.filter((n) => !/^(loop|ram|zram)/.test(n)).map(async (name) => { + const [model, size, rotational] = await Promise.all([read(`/sys/block/${name}/device/model`), read(`/sys/block/${name}/size`), read(`/sys/block/${name}/queue/rotational`)]); + return { name, model: label(model), sizeBytes: number(size) == null ? null : number(size) * 512, rotational: rotational === "1" ? true : rotational === "0" ? false : null }; + })); + const gpus = (await Promise.all(pciNames.map(async (name) => { + const base = `/sys/bus/pci/devices/${name}`; + if (!(await read(`${base}/class`))?.startsWith("0x03")) return null; + const [vendor, device, driver] = await Promise.all([read(`${base}/vendor`), read(`${base}/device`), linkName(`${base}/driver`)]); + return { vendor, device, driver }; + }))).filter(Boolean); + const network = (await Promise.all(networkNames.filter((n) => n !== "lo").map(async (name) => { + const driver = await linkName(`/sys/class/net/${name}/device/driver`); + return driver ? { name, driver } : null; + }))).filter(Boolean); + const decode = (s) => s.replace(/\\([0-7]{3})/g, (_, oct) => String.fromCharCode(parseInt(oct, 8))); + const mount = (mounts || "").split("\n").map((line) => { + const [head, tail] = line.split(" - "); + return { mountpoint: decode(head.split(" ")[4] || ""), filesystem: tail?.split(" ")[0] || null }; + }).filter((m) => m.mountpoint && (storagePath === m.mountpoint || storagePath.startsWith(m.mountpoint === "/" ? "/" : `${m.mountpoint}/`))) + .sort((a, b) => b.mountpoint.length - a.mountpoint.length)[0]; + const memory = parseMeminfo(mem); + return { cpuModel: label(cpus[0]?.["model name"] || cpus[0]?.["Processor"] || null), sockets: completeTopology ? sockets.size || null : null, + physicalCores: completeTopology ? cores.size || null : null, logicalCpus, architecture: system.arch(), + memoryTotalBytes: memory.memoryTotalBytes, swapTotalBytes: memory.swapTotalBytes, + disks, gpus, network, boardVendor: label(dmi[0]), boardName: label(dmi[1]), biosVendor: label(dmi[2]), biosVersion: label(dmi[3]), + osName: osRelease?.match(/^PRETTY_NAME=(.*)$/m)?.[1].replace(/^"|"$/g, "") || null, + kernel: system.release(), storagePath, filesystem: mount?.filesystem || null, + unavailableSources: [!logicalCpus && "CPU", !mem && "memory", !osRelease && "OS release", !mounts && "mounts"].filter(Boolean) }; + } + return { collect, hardware }; +} diff --git a/src/gateway/system-health-store.js b/src/gateway/system-health-store.js new file mode 100644 index 0000000..5f7f5de --- /dev/null +++ b/src/gateway/system-health-store.js @@ -0,0 +1,123 @@ +// Bounded minute history in the gateway's existing SQLite database. +export const DAY_MS = 86_400_000; +export const HEALTH_POLICY = Object.freeze({ sampleIntervalMs: 5000, aggregateIntervalMs: 60_000, + hardwareIntervalMs: 300_000, resourceRetentionDays: 30, storageRetentionDays: 186 }); +const finite = (values) => values.filter(Number.isFinite); +const mean = (values) => { const v = finite(values); return v.length ? v.reduce((a, b) => a + b, 0) / v.length : null; }; +const peak = (values) => { const v = finite(values); return v.length ? Math.max(...v) : null; }; + +export function storageForecast(points) { + const valid = points.filter((p) => Number.isFinite(p.storageUsedBytes) && p.storageTotalBytes > 0); + const latest = valid.at(-1); + const empty = { status: "insufficient_history", daysToFull: null, bytesPerDay: null, observedDays: 0 }; + if (!latest) return empty; + // Use only the segment since the most recent capacity change. A replacement disk is not growth. + let start = valid.length - 1; + while (start > 0 && valid[start - 1].storageTotalBytes === latest.storageTotalBytes) start--; + const segment = valid.slice(start); + const observedDays = (latest.timestamp - segment[0].timestamp) / DAY_MS; + const available = latest.storageAvailableBytes ?? Math.max(0, latest.storageTotalBytes - latest.storageUsedBytes); + if (available <= 0) return { ...empty, status: "full", daysToFull: 0, observedDays }; + if (observedDays < 7 || segment.length < 8) return { ...empty, observedDays }; + const xs = segment.map((p) => (p.timestamp - segment[0].timestamp) / DAY_MS); + const ys = segment.map((p) => p.storageUsedBytes); + const mx = mean(xs), my = mean(ys); + const divisor = xs.reduce((sum, x) => sum + (x - mx) ** 2, 0); + const slope = xs.reduce((sum, x, i) => sum + (x - mx) * (ys[i] - my), 0) / divisor; + if (!Number.isFinite(slope)) return { ...empty, observedDays }; + if (slope <= 0) return { status: "stable", daysToFull: null, bytesPerDay: slope, observedDays }; + return { status: "growing", daysToFull: available / slope, bytesPerDay: slope, observedDays }; +} + +export function createSystemHealthStore(db, { now = Date.now } = {}) { + function persist(samples) { + if (!samples.length) return; + const timestamp = Math.floor(samples[0].timestamp / 60_000) * 60_000; + const values = (field) => samples.map((s) => s[field]); + const fields = ["cpuPercent", "memoryPercent", "load1", "storagePercent"]; + db.exec("BEGIN IMMEDIATE"); + try { + const columns = ["cpu", "memory", "load", "storage"]; + const updates = columns.map((key) => ` + ${key} = CASE WHEN ${key}_count + excluded.${key}_count > 0 THEN + (COALESCE(${key}, 0) * ${key}_count + COALESCE(excluded.${key}, 0) * excluded.${key}_count) + / (${key}_count + excluded.${key}_count) ELSE NULL END, + ${key}_peak = NULLIF(MAX(COALESCE(${key}_peak, -1), COALESCE(excluded.${key}_peak, -1)), -1), + ${key}_count = ${key}_count + excluded.${key}_count`).join(","); + db.prepare(`INSERT INTO system_health_resources VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(timestamp) DO UPDATE SET ${updates}`) + .run(timestamp, ...fields.flatMap((key) => [mean(values(key)), peak(values(key))]), + ...fields.map((key) => finite(values(key)).length)); + const latest = samples.filter((s) => s.storageTotalBytes > 0 && Number.isFinite(s.storageUsedBytes)).at(-1); + if (latest) db.prepare(`INSERT INTO system_health_storage VALUES (?, 60, ?, ?, ?, ?, ?) + ON CONFLICT(timestamp, resolution) DO UPDATE SET used = excluded.used, total = excluded.total, + available = excluded.available, percent = excluded.percent, + peak = NULLIF(MAX(COALESCE(peak, -1), COALESCE(excluded.peak, -1)), -1)`) + .run(timestamp, latest.storageUsedBytes, latest.storageTotalBytes, latest.storageAvailableBytes, latest.storagePercent, peak(values("storagePercent"))); + db.exec("COMMIT"); + } catch (error) { db.exec("ROLLBACK"); throw error; } + return timestamp; + } + function maintain() { + const current = now(); + // Compact only complete hours before the minute-retention boundary. Keeping the newest actual + // observation (rather than averaging capacity) preserves disk-resize boundaries for forecasting. + const cutoff = Math.floor((current - HEALTH_POLICY.resourceRetentionDays * DAY_MS) / 3_600_000) * 3_600_000; + db.exec("BEGIN IMMEDIATE"); + try { + db.prepare(`INSERT OR REPLACE INTO system_health_storage (timestamp, resolution, used, total, available, percent, peak) + SELECT s.timestamp, 3600, s.used, s.total, s.available, s.percent, g.peak + FROM system_health_storage s JOIN ( + SELECT MAX(timestamp) AS latest, MAX(peak) AS peak FROM system_health_storage + WHERE resolution = 60 AND timestamp < ? GROUP BY CAST(timestamp / 3600000 AS INTEGER) + ) g ON s.timestamp = g.latest AND s.resolution = 60`).run(cutoff); + db.prepare("DELETE FROM system_health_storage WHERE resolution = 60 AND timestamp < ?").run(cutoff); + db.prepare("DELETE FROM system_health_resources WHERE timestamp < ?").run(current - HEALTH_POLICY.resourceRetentionDays * DAY_MS); + db.prepare("DELETE FROM system_health_storage WHERE timestamp < ?").run(current - HEALTH_POLICY.storageRetentionDays * DAY_MS); + db.prepare("DELETE FROM system_health_changes WHERE timestamp < ?").run(current - HEALTH_POLICY.storageRetentionDays * DAY_MS); + db.exec("DELETE FROM system_health_changes WHERE id NOT IN (SELECT id FROM system_health_changes ORDER BY id DESC LIMIT 100)"); + db.exec("COMMIT"); + } catch (error) { db.exec("ROLLBACK"); throw error; } + } + function history(range = "1h") { + const spans = { "1h": 3_600_000, "24h": DAY_MS, "7d": 7 * DAY_MS, "30d": 30 * DAY_MS }; + if (!Object.hasOwn(spans, range)) throw new RangeError("Invalid system health range"); + const end = now(), start = end - spans[range], bucket = Math.max(60_000, Math.ceil(spans[range] / 240 / 60_000) * 60_000); + const points = db.prepare(`SELECT MAX(timestamp) AS timestamp, AVG(cpu) AS cpuPercent, AVG(memory) AS memoryPercent, + AVG(load) AS load1, AVG(storage) AS storagePercent FROM system_health_resources WHERE timestamp >= ? AND timestamp <= ? + GROUP BY CAST(timestamp / ? AS INTEGER) ORDER BY timestamp`).all(start, end, bucket); + const peaks = db.prepare(`SELECT MAX(cpu_peak) AS cpuPercent, MAX(memory_peak) AS memoryPercent, + MAX(load_peak) AS load1, MAX(storage_peak) AS storagePercent FROM system_health_resources WHERE timestamp >= ? AND timestamp <= ?`).get(start, end); + return { points, peaks, start, end }; + } + function storage() { + const end = now(), start = end - HEALTH_POLICY.storageRetentionDays * DAY_MS; + const points = db.prepare(`SELECT s.timestamp, s.used AS storageUsedBytes, s.total AS storageTotalBytes, + s.available AS storageAvailableBytes, s.percent AS storagePercent FROM system_health_storage s JOIN ( + SELECT MAX(timestamp) AS latest FROM system_health_storage WHERE timestamp >= ? AND timestamp <= ? + GROUP BY CAST(timestamp / 86400000 AS INTEGER) + ) g ON s.timestamp = g.latest ORDER BY s.timestamp`).all(start, end); + return { points, forecast: storageForecast(points), start, end }; + } + function hardware() { + const row = db.prepare("SELECT collected_at, snapshot FROM system_health_hardware WHERE id = 1").get(); + const changes = db.prepare("SELECT timestamp, fields FROM system_health_changes ORDER BY id DESC LIMIT 100").all() + .map((r) => ({ timestamp: r.timestamp, fields: JSON.parse(r.fields) })); + return { snapshot: row ? JSON.parse(row.snapshot) : null, collectedAt: row?.collected_at ?? null, changes }; + } + function saveHardware(snapshot) { + const previous = hardware().snapshot; + const fields = previous ? Object.keys(snapshot).filter((key) => JSON.stringify(snapshot[key]) !== JSON.stringify(previous[key])) : []; + const timestamp = now(); + db.exec("BEGIN IMMEDIATE"); + try { + db.prepare("INSERT OR REPLACE INTO system_health_hardware VALUES (1, ?, ?)").run(timestamp, JSON.stringify(snapshot)); + if (fields.length) db.prepare("INSERT INTO system_health_changes(timestamp, fields) VALUES (?, ?)").run(timestamp, JSON.stringify(fields)); + db.exec("DELETE FROM system_health_changes WHERE id NOT IN (SELECT id FROM system_health_changes ORDER BY id DESC LIMIT 100)"); + db.exec("COMMIT"); + } catch (error) { db.exec("ROLLBACK"); throw error; } + return hardware(); + } + const lastPersistedAt = () => db.prepare("SELECT MAX(timestamp) AS timestamp FROM system_health_resources").get().timestamp; + return { persist, maintain, history, storage, hardware, saveHardware, lastPersistedAt }; +} diff --git a/src/gateway/system-health.js b/src/gateway/system-health.js new file mode 100644 index 0000000..cdc6079 --- /dev/null +++ b/src/gateway/system-health.js @@ -0,0 +1,98 @@ +import { getDb } from "../db/index.js"; +import { gatewayRoot } from "../config/paths.js"; +import { createLinuxHealthCollector } from "./system-health-collect.js"; +import { createSystemHealthStore, HEALTH_POLICY } from "./system-health-store.js"; + +export function createSystemHealth({ collector, store, now = Date.now, schedule = setInterval, unschedule = clearInterval } = {}) { + let sample = null, recent = [], pending = [], active = null, refreshing = null; + let timer = null, started = false, stopped = false, lastHardwareAt = null, lastMaintenanceAt = null; + let lastPersistedAt = store.lastPersistedAt(), collectionError = null, persistenceError = null, hardwareError = null; + async function refreshHardware() { + if (refreshing) return refreshing; + // Coalesce repeated clicks/pollers without rescanning sysfs on every admin request. + if (lastHardwareAt != null && now() - lastHardwareAt < 5000) return store.hardware(); + refreshing = (async () => { + try { + const snapshot = await collector.hardware(); + const result = store.saveHardware(snapshot); + lastHardwareAt = now(); hardwareError = snapshot.unavailableSources?.length ? `Hardware sources unavailable: ${snapshot.unavailableSources.join(", ")}` : null; + return result; + } catch { + hardwareError = "Hardware inventory unavailable"; + throw new Error(hardwareError); + } finally { refreshing = null; } + })(); + return refreshing; + } + function flush() { + if (!pending.length) return; + lastPersistedAt = store.persist(pending); + pending = []; + persistenceError = null; + } + async function tick() { + if (stopped || active) return active; + active = (async () => { + try { + const result = await collector.collect(); + sample = result.sample; + collectionError = result.error; + recent.push(sample); + recent = recent.filter((s) => s.timestamp >= now() - 15 * 60_000).slice(-180); + if (pending.length && Math.floor(sample.timestamp / 60_000) !== Math.floor(pending[0].timestamp / 60_000)) { + try { flush(); } catch { persistenceError = "Metrics persistence unavailable"; pending = []; } + } + pending.push(sample); + pending = pending.slice(-12); + if (lastMaintenanceAt == null || now() - lastMaintenanceAt >= 3_600_000) { + try { store.maintain(); lastMaintenanceAt = now(); } catch { persistenceError = "Metrics retention unavailable"; } + } + if (lastHardwareAt == null || now() - lastHardwareAt >= HEALTH_POLICY.hardwareIntervalMs) { + // Inventory work never holds up the five-second metrics loop. + void refreshHardware().catch(() => {}); + } + } catch { collectionError = "System telemetry unavailable"; } + finally { active = null; } + })(); + return active; + } + function current() { + return { sample, recent, collection: { ...HEALTH_POLICY, lastSampleAt: sample?.timestamp ?? null, + lastPersistedAt, error: [collectionError, persistenceError, hardwareError].filter(Boolean).join("; ") || null } }; + } + function history(range = "live") { + if (range !== "live") return store.history(range); + const end = now(), start = end - 15 * 60_000; + const points = recent.filter((s) => s.timestamp >= start && s.timestamp <= end); + const peaks = Object.fromEntries(["cpuPercent", "memoryPercent", "load1", "storagePercent"].map((key) => { + const values = points.map((s) => s[key]).filter(Number.isFinite); + return [key, values.length ? Math.max(...values) : null]; + })); + return { points, peaks, start, end }; + } + function start() { + if (started) return; + started = true; stopped = false; + timer = schedule(() => void tick(), HEALTH_POLICY.sampleIntervalMs); + timer.unref?.(); + void tick(); + } + async function stop() { + stopped = true; + if (timer) unschedule(timer); + timer = null; started = false; + await active; + await refreshing?.catch(() => {}); + try { flush(); } catch { persistenceError = "Metrics persistence unavailable"; } + } + return { start, stop, tick, current, history, storage: store.storage, hardware: store.hardware, refreshHardware }; +} +let service; +export function getSystemHealth() { + service ||= createSystemHealth({ collector: createLinuxHealthCollector({ storagePath: gatewayRoot() }), store: createSystemHealthStore(getDb()) }); + return service; +} +export function startSystemHealth() { + try { getSystemHealth().start(); } catch { console.error("[system-health] collector unavailable; gateway continues"); } +} +export async function stopSystemHealth() { await service?.stop(); } diff --git a/src/server.js b/src/server.js index 29dff05..ae60748 100644 --- a/src/server.js +++ b/src/server.js @@ -48,6 +48,7 @@ import { autoRepairCodexUsageHistory } from "./gateway/usage-repair.js"; import { postNotice } from "./platforms/notify.js"; import { startLicenseVerification } from "./ee/license.js"; import { startUsageReporting } from "./ee/limits.js"; +import { startSystemHealth } from "./gateway/system-health.js"; installConsoleRedaction(); @@ -168,6 +169,9 @@ async function main() { // relying on create-time modes. const { failed: hardenFailed } = hardenRuntimeFiles(); assertRuntimeHardening({ failed: hardenFailed }); + // Host metrics belong to the daemon, never to an engine container. Sampling is asynchronous + // and failure is reported on the admin page without holding up chat or startup. + void Promise.resolve().then(() => startSystemHealth()).catch(() => console.error("[system-health] collector startup failed")); // Brand-new install (nothing an operator wrote in settings.json — the installer's own // pre-boot keys don't count) → mint an admin password rather than leaving the whole API open. diff --git a/src/web/routes/admin.js b/src/web/routes/admin.js index 416c738..09ccff4 100644 --- a/src/web/routes/admin.js +++ b/src/web/routes/admin.js @@ -20,6 +20,7 @@ import { createSchedulesRouter } from "./schedules.js"; import { createChannelsRouter } from "./channels.js"; import { createUsersRouter } from "./users.js"; import { createSkillsRouter } from "./skills.js"; +import { createSystemHealthRouter } from "./system-health.js"; export function createAdminRouter({ slack, @@ -33,6 +34,7 @@ export function createAdminRouter({ router.use(createSettingsRouter({ slack, transports, instanceId, startGatewayUpdate, restartCoordinator })); router.use(createObservabilityRouter()); + router.use(createSystemHealthRouter()); router.use(createApprovalsRouter({ slack })); router.use(createSchedulesRouter()); router.use(createChannelsRouter({ slack, testMakeToolbox })); diff --git a/src/web/routes/system-health.js b/src/web/routes/system-health.js new file mode 100644 index 0000000..5b5a6d5 --- /dev/null +++ b/src/web/routes/system-health.js @@ -0,0 +1,22 @@ +import { Router } from "express"; +import { getSystemHealth } from "../../gateway/system-health.js"; + +// Mounted only behind the existing admin authentication stack; never on public /api/health. +export function createSystemHealthRouter({ getService = getSystemHealth } = {}) { + const router = Router(); + router.use("/system-health", (_req, res, next) => { res.set("Cache-Control", "no-store"); next(); }); + const handler = (action) => async (req, res) => { + try { res.json(await action(getService(), req)); } + catch { res.status(503).json({ error: "System health data unavailable" }); } + }; + router.get("/system-health/current", handler((service) => service.current())); + router.get("/system-health/history", (req, res, next) => { + const range = req.query.range ?? "live"; + if (typeof range !== "string" || !["live", "1h", "24h", "7d", "30d"].includes(range)) return res.status(400).json({ error: "Invalid system health range" }); + return handler((service) => service.history(range))(req, res, next); + }); + router.get("/system-health/storage", handler((service) => service.storage())); + router.get("/system-health/hardware", handler((service) => service.hardware())); + router.post("/system-health/hardware/refresh", handler((service) => service.refreshHardware())); + return router; +} diff --git a/test/admin-navigation.test.js b/test/admin-navigation.test.js index 10479a3..71db407 100644 --- a/test/admin-navigation.test.js +++ b/test/admin-navigation.test.js @@ -25,6 +25,7 @@ const EXPECTED = { skills: "/skills", api: "/api-docs", settings: "/settings", + "system-health": "/system-health", }; test("every admin view has a canonical path and round-trips from it", () => { diff --git a/test/admin-system-health.test.js b/test/admin-system-health.test.js new file mode 100644 index 0000000..13dd6b2 --- /dev/null +++ b/test/admin-system-health.test.js @@ -0,0 +1,112 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { createHealthPoller, forecastCopy, healthBytes, healthChart, storageSeverity } from "../public/admin-system-health.js"; + +const settle = () => new Promise((resolve) => setImmediate(resolve)); +function harness(request) { + const calls = [], updates = [], timers = new Map(); + let timerId = 0, isVisible = true; + const controller = createHealthPoller({ + request: (url, options) => { calls.push({ url, options }); return request?.(url, options) ?? Promise.resolve({ url }); }, + onUpdate: (state) => updates.push(structuredClone(state)), + visible: () => isVisible, + schedule: (fn, ms) => { const id = ++timerId; timers.set(id, { fn, ms }); return id; }, + unschedule: (id) => timers.delete(id), + }); + return { controller, calls, updates, timers, setVisible(value) { isVisible = value; controller.visibilityChanged(); }, tick() { const [id, timer] = timers.entries().next().value; timers.delete(id); timer.fn(); } }; +} + +test("health polling stops off-page/hidden/paused, and manual refresh scans hardware while paused", async () => { + const h = harness(); + h.controller.enter(); await settle(); + assert.equal(h.calls.length, 4); + assert.deepEqual([...h.timers.values()].map((timer) => timer.ms), [5000]); + h.tick(); await settle(); + assert.equal(h.calls.length, 5); + h.controller.togglePause(); + assert.equal(h.timers.size, 0); + h.controller.refresh(); await settle(); + assert.equal(h.calls.length, 9); + assert.equal(h.calls.at(-1).url, "/api/system-health/hardware/refresh"); + assert.equal(h.calls.at(-1).options.method, "POST"); + assert.equal(h.timers.size, 0); + h.controller.togglePause(); await settle(); + h.setVisible(false); assert.equal(h.timers.size, 0); + const count = h.calls.length; + h.setVisible(true); await settle(); assert.equal(h.calls.length, count + 4); + h.controller.leave(); assert.equal(h.timers.size, 0); + h.setVisible(false); h.setVisible(true); await settle(); + assert.equal(h.calls.length, count + 4); +}); + +test("range change never overlaps requests or accepts an outdated response", async () => { + const releases = []; + const h = harness((url) => new Promise((resolve) => releases.push(() => resolve({ url, marker: "old" })))); + h.controller.enter(); await settle(); + assert.equal(h.calls.length, 4); + h.controller.setRange("24h"); await settle(); + assert.equal(h.calls.length, 4); + assert.ok(h.calls.every((call) => call.options.signal.aborted)); + releases.splice(0).forEach((release) => release()); await settle(); + assert.equal(h.calls.length, 8); + assert.match(h.calls[5].url, /range=24h/); + assert.equal(h.updates.at(-1).current, null); + assert.equal(h.updates.at(-1).history, null); + releases.splice(0).forEach((release) => release()); await settle(); + assert.match(h.updates.at(-1).history.url, /range=24h/); + h.controller.leave(); +}); + +test("pausing an in-flight request discards late results and restores refresh", async () => { + const releases = []; + const h = harness(() => new Promise((resolve) => releases.push(() => resolve({ sample: "late" })))); + h.controller.enter(); await settle(); + h.controller.togglePause(); + releases.forEach((release) => release()); await settle(); + assert.equal(h.updates.at(-1).current, null); + assert.equal(h.updates.at(-1).loading, false); + assert.equal(h.updates.at(-1).paused, true); + assert.equal(h.timers.size, 0); + h.controller.leave(); +}); + +test("one failed section does not hide successful data and is retried", async () => { + let failing = true; + const h = harness((url) => failing && url.endsWith("/storage") ? Promise.reject(new Error("storage unavailable")) : Promise.resolve({ url })); + h.controller.enter(); await settle(); + assert.equal(h.updates.at(-1).errors.storage, "storage unavailable"); + assert.ok(h.updates.at(-1).current); + failing = false; + h.tick(); await settle(); + assert.equal(h.updates.at(-1).errors.storage, undefined); + assert.ok(h.updates.at(-1).storage); + h.controller.leave(); +}); + +test("hourly minute observations connect, missing values break lines, tooltip input is escaped", () => { + const start = Date.UTC(2026, 0, 1), end = start + 3_600_000; + const html = healthChart([ + { timestamp: start, cpuPercent: 10, memoryPercent: 30 }, + { timestamp: start + 60_000, cpuPercent: 20, memoryPercent: 31 }, + { timestamp: start + 120_000, cpuPercent: null, memoryPercent: 32 }, + { timestamp: start + 180_000, cpuPercent: 15, memoryPercent: 33 }, + ], { start, end }); + const cpu = html.match(/class="sh-line sh-cpu" d="([^"]+)"/)[1]; + assert.equal((cpu.match(/M/g) || []).length, 2); + assert.equal((cpu.match(/L/g) || []).length, 1); + assert.match(html, /CPU 10\.0%/); + assert.doesNotMatch(healthChart([{ timestamp: start, cpuPercent: '' }]), /