From f5f3a24b604000d4f3492cd40812c16eac921fdb Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 21:56:12 +0000 Subject: [PATCH] Back off analyser polling while the queue is empty The analyser asked for work every 30 seconds even when there was none, so the website never went quiet long enough for Railway to sleep it. Double the wait between empty polls from IDLE_SLEEP_MIN (60s) up to IDLE_SLEEP_MAX (1800s), resetting as soon as an app is assigned, and retry transient curl failures for up to a minute so requests survive the server waking up. The analyser's last check-in was only held in memory, so a woken process reported it offline until the next ping. Store it in an analyser_heartbeat row and read it back once per process. Claude-Session: https://claude.ai/code/session_01Fi3H98SycMK13rrr8ySSU1 --- analyser/.env.example | 2 ++ analyser/processQueue.sh | 27 ++++++++++++++- migrations/015_analyser_heartbeat.sql | 7 ++++ models/Apps.js | 15 +++++++++ raspberry-pi-analyser.md | 4 +++ routes/index.js | 29 ++++++++++++++-- test/analyserHeartbeat.test.js | 48 +++++++++++++++++++++++++++ 7 files changed, 128 insertions(+), 4 deletions(-) create mode 100644 migrations/015_analyser_heartbeat.sql create mode 100644 test/analyserHeartbeat.test.js diff --git a/analyser/.env.example b/analyser/.env.example index 1b8e220..e1f15ac 100644 --- a/analyser/.env.example +++ b/analyser/.env.example @@ -27,6 +27,8 @@ MAX_APP_SIZE_BYTES=3000000000 MAX_ATTEMPT_DOWNLOAD_BYTES=3000000000 CONSECUTIVE_FAILURE_LIMIT=5 CIRCUIT_BREAKER_SLEEP=3600 +IDLE_SLEEP_MIN=60 +IDLE_SLEEP_MAX=1800 DOWNLOAD_WATCHDOG_INTERVAL=5 NETWORK_INTERFACE= diff --git a/analyser/processQueue.sh b/analyser/processQueue.sh index 94db701..d17ad25 100755 --- a/analyser/processQueue.sh +++ b/analyser/processQueue.sh @@ -44,6 +44,8 @@ fi : "${IOS_STAGING_CLEANUP_TIMEOUT:=30}" : "${CONSECUTIVE_FAILURE_LIMIT:=5}" : "${CIRCUIT_BREAKER_SLEEP:=3600}" +: "${IDLE_SLEEP_MIN:=60}" +: "${IDLE_SLEEP_MAX:=1800}" : "${DOWNLOAD_WATCHDOG_INTERVAL:=5}" : "${NETWORK_INTERFACE:=}" : "${DAILY_BYTES_FILE:=./daily-download-bytes.txt}" @@ -113,6 +115,10 @@ fi curl_auth_config=$(mktemp) chmod 600 "$curl_auth_config" printf 'header = "Authorization: Bearer %s"\n' "$UPLOAD_PASSWORD" > "$curl_auth_config" +# The server may be asleep (Railway stops idle services) and answer its first +# request with a 502 or refuse the connection while it starts, so every call +# retries transient failures for up to a minute. +printf 'retry = 5\nretry-delay = 5\nretry-max-time = 60\nretry-connrefused\n' >> "$curl_auth_config" queue_headers=$(mktemp) chmod 600 "$queue_headers" trap 'rm -f "$curl_auth_config" "$queue_headers"' EXIT @@ -784,6 +790,20 @@ process_app() [ "$success" -eq 1 ] } +# An empty queue is polled with a doubling delay, so the server sees no traffic +# for long enough to sleep. Any assigned app resets it. +idle_sleep="$IDLE_SLEEP_MIN" + +idle_backoff() +{ + echo "Queue empty. Checking again in ${idle_sleep}s" + sleep "$idle_sleep" + idle_sleep=$((idle_sleep * 2)) + if [ "$idle_sleep" -gt "$IDLE_SLEEP_MAX" ]; then + idle_sleep="$IDLE_SLEEP_MAX" + fi +} + if [ -n "$ONLY_APP_ID" ]; then reset_daily_counter @@ -846,5 +866,10 @@ while true; do exit 0 fi - sleepabit + if [ "$appId" == "" ] ; then + idle_backoff + else + idle_sleep="$IDLE_SLEEP_MIN" + sleepabit + fi done diff --git a/migrations/015_analyser_heartbeat.sql b/migrations/015_analyser_heartbeat.sql new file mode 100644 index 0000000..d1fae9b --- /dev/null +++ b/migrations/015_analyser_heartbeat.sql @@ -0,0 +1,7 @@ +-- When the analyser last checked in. Kept in the database so a restarted or +-- woken web process still knows the analyser is online. + +CREATE TABLE IF NOT EXISTS analyser_heartbeat ( + id boolean PRIMARY KEY DEFAULT true CHECK (id), + last_seen timestamptz NOT NULL +); diff --git a/models/Apps.js b/models/Apps.js index ab11857..de446d2 100644 --- a/models/Apps.js +++ b/models/Apps.js @@ -503,6 +503,19 @@ const getSiteDataSignature = async () => { }; } +const recordAnalyserHeartbeat = async () => { + await pool.query(` + INSERT INTO analyser_heartbeat (id, last_seen) VALUES (true, NOW()) + ON CONFLICT (id) DO UPDATE SET last_seen = EXCLUDED.last_seen + `); +} + +// Milliseconds since the epoch, or 0 if the analyser has never checked in. +const lastAnalyserHeartbeat = async () => { + const result = await pool.query('SELECT last_seen FROM analyser_heartbeat'); + return result.rows.length > 0 ? result.rows[0].last_seen.getTime() : 0; +} + const countAnalysed = async () => { const result = await pool.query("SELECT COUNT(*) FROM apps WHERE status = 'analysed'"); return parseInt(result.rows[0].count, 10); @@ -519,6 +532,8 @@ module.exports = { nextApp, updateAnalysis, getAllApps, + recordAnalyserHeartbeat, + lastAnalyserHeartbeat, getSiteDataSignature, healthCheck, deriveAnalysisState, diff --git a/raspberry-pi-analyser.md b/raspberry-pi-analyser.md index d2a9a09..8af3238 100644 --- a/raspberry-pi-analyser.md +++ b/raspberry-pi-analyser.md @@ -85,9 +85,13 @@ MAX_APP_SIZE_BYTES=1800000000 MAX_ATTEMPT_DOWNLOAD_BYTES=1800000000 CONSECUTIVE_FAILURE_LIMIT=5 CIRCUIT_BREAKER_SLEEP=3600 +IDLE_SLEEP_MIN=60 +IDLE_SLEEP_MAX=1800 DOWNLOAD_WATCHDOG_INTERVAL=5 ``` +When the queue is empty the analyser waits `IDLE_SLEEP_MIN` seconds before asking again, doubling the wait up to `IDLE_SLEEP_MAX`, and goes back to the minimum as soon as it gets an app. This lets a hosting platform that stops idle services, such as Railway, put the website to sleep while there is nothing to analyse. A newly requested app can therefore wait up to `IDLE_SLEEP_MAX` before analysis starts. Requests to the server retry for up to a minute, so the analyser rides out the server waking up. + Set `NETWORK_INTERFACE` only if auto-detection picks the wrong interface. On a Pi this is often `eth0` or `wlan0`. ## Use RAM-Backed IPA Storage diff --git a/routes/index.js b/routes/index.js index 8192329..98755bb 100644 --- a/routes/index.js +++ b/routes/index.js @@ -30,7 +30,25 @@ const APPS_PER_PAGE = 50; const MAX_SITEMAP_URLS = 50000; const MAX_SITEMAP_BYTES = 50 * 1024 * 1024; +const ANALYSER_ONLINE_MS = 1000*60*60; let lastPing = 0; // unix timestamp +let lastPingLoad = null; + +// lastPing starts at zero in a fresh process, and on a platform that stops +// idle services every wake is one, so the last check-in is read back from the +// database once. A failed read is retried a minute later rather than on every +// request. +function loadLastPing() { + if (!lastPingLoad) { + lastPingLoad = Apps.lastAnalyserHeartbeat().then((stored) => { + lastPing = Math.max(lastPing, stored); + }, (err) => { + console.error('Could not read analyser heartbeat:', err.message); + setTimeout(() => { lastPingLoad = null; }, 60 * 1000).unref(); + }); + } + return lastPingLoad; +} function requireValidAppId(req, res, next) { if (!isValidAppId(req.params.appId)) @@ -66,8 +84,10 @@ function redirectPublic(res, status, path) { // ping from analyser in past hour? router.use(function (req, res, next) { - res.locals.analyserOnline = lastPing > Date.now() - 1000*60*60; - next(); + loadLastPing().then(() => { + res.locals.analyserOnline = lastPing > Date.now() - ANALYSER_ONLINE_MS; + next(); + }); }); function thirdPartyTrackerNames(analysis) { @@ -423,7 +443,7 @@ router.get('/healthz', asyncHandler(async (req, res) => { })); router.get('/healthz/analyser', (req, res) => { - const online = lastPing > Date.now() - 1000*60*60; + const online = lastPing > Date.now() - ANALYSER_ONLINE_MS; res.status(online ? 200 : 503).json({ ok: online }); }); @@ -789,6 +809,9 @@ router.get('/queue', asyncHandler(async (req, res) => { // enable analyser to report online status router.get('/ping', (req, res) => { lastPing = Date.now(); + Apps.recordAnalyserHeartbeat().catch((err) => { + console.error('Could not record analyser heartbeat:', err.message); + }); res.send("online"); }); diff --git a/test/analyserHeartbeat.test.js b/test/analyserHeartbeat.test.js new file mode 100644 index 0000000..3c957fe --- /dev/null +++ b/test/analyserHeartbeat.test.js @@ -0,0 +1,48 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); + +// Runs in its own process, so the server's one-off heartbeat read happens +// against the stubs below. +test('analyser status survives a restart through the stored heartbeat', async () => { + process.env.UPLOAD_PASSWORD = 'test-secret'; + const Apps = require('../models/Apps'); + const original = { + lastAnalyserHeartbeat: Apps.lastAnalyserHeartbeat, + recordAnalyserHeartbeat: Apps.recordAnalyserHeartbeat, + }; + let reads = 0; + let writes = 0; + Apps.lastAnalyserHeartbeat = async () => { + reads++; + return Date.now() - 10 * 60 * 1000; + }; + Apps.recordAnalyserHeartbeat = async () => { + writes++; + }; + + const app = require('../server'); + const server = await new Promise((resolve) => { + const instance = app.listen(0, () => resolve(instance)); + }); + + try { + const base = `http://127.0.0.1:${server.address().port}`; + const first = await fetch(`${base}/healthz/analyser`); + const second = await fetch(`${base}/healthz/analyser`); + assert.equal(first.status, 200); + assert.equal(second.status, 200); + assert.equal(reads, 1); + + const ping = await fetch(`${base}/ping`, { + headers: { authorization: 'Bearer test-secret' } + }); + assert.equal(ping.status, 200); + assert.equal(writes, 1); + } finally { + Apps.lastAnalyserHeartbeat = original.lastAnalyserHeartbeat; + Apps.recordAnalyserHeartbeat = original.recordAnalyserHeartbeat; + await new Promise((resolve, reject) => { + server.close((err) => err ? reject(err) : resolve()); + }); + } +});