diff --git a/analyser/.env.example b/analyser/.env.example index 1b8e220..e1f15ac 100644 --- a/analyser/.env.example +++ b/analyser/.env.example @@ -27,6 +27,8 @@ MAX_APP_SIZE_BYTES=3000000000 MAX_ATTEMPT_DOWNLOAD_BYTES=3000000000 CONSECUTIVE_FAILURE_LIMIT=5 CIRCUIT_BREAKER_SLEEP=3600 +IDLE_SLEEP_MIN=60 +IDLE_SLEEP_MAX=1800 DOWNLOAD_WATCHDOG_INTERVAL=5 NETWORK_INTERFACE= diff --git a/analyser/processQueue.sh b/analyser/processQueue.sh index 94db701..d17ad25 100755 --- a/analyser/processQueue.sh +++ b/analyser/processQueue.sh @@ -44,6 +44,8 @@ fi : "${IOS_STAGING_CLEANUP_TIMEOUT:=30}" : "${CONSECUTIVE_FAILURE_LIMIT:=5}" : "${CIRCUIT_BREAKER_SLEEP:=3600}" +: "${IDLE_SLEEP_MIN:=60}" +: "${IDLE_SLEEP_MAX:=1800}" : "${DOWNLOAD_WATCHDOG_INTERVAL:=5}" : "${NETWORK_INTERFACE:=}" : "${DAILY_BYTES_FILE:=./daily-download-bytes.txt}" @@ -113,6 +115,10 @@ fi curl_auth_config=$(mktemp) chmod 600 "$curl_auth_config" printf 'header = "Authorization: Bearer %s"\n' "$UPLOAD_PASSWORD" > "$curl_auth_config" +# The server may be asleep (Railway stops idle services) and answer its first +# request with a 502 or refuse the connection while it starts, so every call +# retries transient failures for up to a minute. +printf 'retry = 5\nretry-delay = 5\nretry-max-time = 60\nretry-connrefused\n' >> "$curl_auth_config" queue_headers=$(mktemp) chmod 600 "$queue_headers" trap 'rm -f "$curl_auth_config" "$queue_headers"' EXIT @@ -784,6 +790,20 @@ process_app() [ "$success" -eq 1 ] } +# An empty queue is polled with a doubling delay, so the server sees no traffic +# for long enough to sleep. Any assigned app resets it. +idle_sleep="$IDLE_SLEEP_MIN" + +idle_backoff() +{ + echo "Queue empty. Checking again in ${idle_sleep}s" + sleep "$idle_sleep" + idle_sleep=$((idle_sleep * 2)) + if [ "$idle_sleep" -gt "$IDLE_SLEEP_MAX" ]; then + idle_sleep="$IDLE_SLEEP_MAX" + fi +} + if [ -n "$ONLY_APP_ID" ]; then reset_daily_counter @@ -846,5 +866,10 @@ while true; do exit 0 fi - sleepabit + if [ "$appId" == "" ] ; then + idle_backoff + else + idle_sleep="$IDLE_SLEEP_MIN" + sleepabit + fi done diff --git a/migrations/015_analyser_heartbeat.sql b/migrations/015_analyser_heartbeat.sql new file mode 100644 index 0000000..d1fae9b --- /dev/null +++ b/migrations/015_analyser_heartbeat.sql @@ -0,0 +1,7 @@ +-- When the analyser last checked in. Kept in the database so a restarted or +-- woken web process still knows the analyser is online. + +CREATE TABLE IF NOT EXISTS analyser_heartbeat ( + id boolean PRIMARY KEY DEFAULT true CHECK (id), + last_seen timestamptz NOT NULL +); diff --git a/models/Apps.js b/models/Apps.js index ab11857..de446d2 100644 --- a/models/Apps.js +++ b/models/Apps.js @@ -503,6 +503,19 @@ const getSiteDataSignature = async () => { }; } +const recordAnalyserHeartbeat = async () => { + await pool.query(` + INSERT INTO analyser_heartbeat (id, last_seen) VALUES (true, NOW()) + ON CONFLICT (id) DO UPDATE SET last_seen = EXCLUDED.last_seen + `); +} + +// Milliseconds since the epoch, or 0 if the analyser has never checked in. +const lastAnalyserHeartbeat = async () => { + const result = await pool.query('SELECT last_seen FROM analyser_heartbeat'); + return result.rows.length > 0 ? result.rows[0].last_seen.getTime() : 0; +} + const countAnalysed = async () => { const result = await pool.query("SELECT COUNT(*) FROM apps WHERE status = 'analysed'"); return parseInt(result.rows[0].count, 10); @@ -519,6 +532,8 @@ module.exports = { nextApp, updateAnalysis, getAllApps, + recordAnalyserHeartbeat, + lastAnalyserHeartbeat, getSiteDataSignature, healthCheck, deriveAnalysisState, diff --git a/raspberry-pi-analyser.md b/raspberry-pi-analyser.md index d2a9a09..8af3238 100644 --- a/raspberry-pi-analyser.md +++ b/raspberry-pi-analyser.md @@ -85,9 +85,13 @@ MAX_APP_SIZE_BYTES=1800000000 MAX_ATTEMPT_DOWNLOAD_BYTES=1800000000 CONSECUTIVE_FAILURE_LIMIT=5 CIRCUIT_BREAKER_SLEEP=3600 +IDLE_SLEEP_MIN=60 +IDLE_SLEEP_MAX=1800 DOWNLOAD_WATCHDOG_INTERVAL=5 ``` +When the queue is empty the analyser waits `IDLE_SLEEP_MIN` seconds before asking again, doubling the wait up to `IDLE_SLEEP_MAX`, and goes back to the minimum as soon as it gets an app. This lets a hosting platform that stops idle services, such as Railway, put the website to sleep while there is nothing to analyse. A newly requested app can therefore wait up to `IDLE_SLEEP_MAX` before analysis starts. Requests to the server retry for up to a minute, so the analyser rides out the server waking up. + Set `NETWORK_INTERFACE` only if auto-detection picks the wrong interface. On a Pi this is often `eth0` or `wlan0`. ## Use RAM-Backed IPA Storage diff --git a/routes/index.js b/routes/index.js index 8192329..98755bb 100644 --- a/routes/index.js +++ b/routes/index.js @@ -30,7 +30,25 @@ const APPS_PER_PAGE = 50; const MAX_SITEMAP_URLS = 50000; const MAX_SITEMAP_BYTES = 50 * 1024 * 1024; +const ANALYSER_ONLINE_MS = 1000*60*60; let lastPing = 0; // unix timestamp +let lastPingLoad = null; + +// lastPing starts at zero in a fresh process, and on a platform that stops +// idle services every wake is one, so the last check-in is read back from the +// database once. A failed read is retried a minute later rather than on every +// request. +function loadLastPing() { + if (!lastPingLoad) { + lastPingLoad = Apps.lastAnalyserHeartbeat().then((stored) => { + lastPing = Math.max(lastPing, stored); + }, (err) => { + console.error('Could not read analyser heartbeat:', err.message); + setTimeout(() => { lastPingLoad = null; }, 60 * 1000).unref(); + }); + } + return lastPingLoad; +} function requireValidAppId(req, res, next) { if (!isValidAppId(req.params.appId)) @@ -66,8 +84,10 @@ function redirectPublic(res, status, path) { // ping from analyser in past hour? router.use(function (req, res, next) { - res.locals.analyserOnline = lastPing > Date.now() - 1000*60*60; - next(); + loadLastPing().then(() => { + res.locals.analyserOnline = lastPing > Date.now() - ANALYSER_ONLINE_MS; + next(); + }); }); function thirdPartyTrackerNames(analysis) { @@ -423,7 +443,7 @@ router.get('/healthz', asyncHandler(async (req, res) => { })); router.get('/healthz/analyser', (req, res) => { - const online = lastPing > Date.now() - 1000*60*60; + const online = lastPing > Date.now() - ANALYSER_ONLINE_MS; res.status(online ? 200 : 503).json({ ok: online }); }); @@ -789,6 +809,9 @@ router.get('/queue', asyncHandler(async (req, res) => { // enable analyser to report online status router.get('/ping', (req, res) => { lastPing = Date.now(); + Apps.recordAnalyserHeartbeat().catch((err) => { + console.error('Could not record analyser heartbeat:', err.message); + }); res.send("online"); }); diff --git a/test/analyserHeartbeat.test.js b/test/analyserHeartbeat.test.js new file mode 100644 index 0000000..3c957fe --- /dev/null +++ b/test/analyserHeartbeat.test.js @@ -0,0 +1,48 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); + +// Runs in its own process, so the server's one-off heartbeat read happens +// against the stubs below. +test('analyser status survives a restart through the stored heartbeat', async () => { + process.env.UPLOAD_PASSWORD = 'test-secret'; + const Apps = require('../models/Apps'); + const original = { + lastAnalyserHeartbeat: Apps.lastAnalyserHeartbeat, + recordAnalyserHeartbeat: Apps.recordAnalyserHeartbeat, + }; + let reads = 0; + let writes = 0; + Apps.lastAnalyserHeartbeat = async () => { + reads++; + return Date.now() - 10 * 60 * 1000; + }; + Apps.recordAnalyserHeartbeat = async () => { + writes++; + }; + + const app = require('../server'); + const server = await new Promise((resolve) => { + const instance = app.listen(0, () => resolve(instance)); + }); + + try { + const base = `http://127.0.0.1:${server.address().port}`; + const first = await fetch(`${base}/healthz/analyser`); + const second = await fetch(`${base}/healthz/analyser`); + assert.equal(first.status, 200); + assert.equal(second.status, 200); + assert.equal(reads, 1); + + const ping = await fetch(`${base}/ping`, { + headers: { authorization: 'Bearer test-secret' } + }); + assert.equal(ping.status, 200); + assert.equal(writes, 1); + } finally { + Apps.lastAnalyserHeartbeat = original.lastAnalyserHeartbeat; + Apps.recordAnalyserHeartbeat = original.recordAnalyserHeartbeat; + await new Promise((resolve, reject) => { + server.close((err) => err ? reject(err) : resolve()); + }); + } +});