Back off analyser polling while the queue is empty

analyser/.env.example

@@ -27,6 +27,8 @@ MAX_APP_SIZE_BYTES=3000000000
 MAX_ATTEMPT_DOWNLOAD_BYTES=3000000000
 CONSECUTIVE_FAILURE_LIMIT=5
 CIRCUIT_BREAKER_SLEEP=3600
+IDLE_SLEEP_MIN=60
+IDLE_SLEEP_MAX=1800
 DOWNLOAD_WATCHDOG_INTERVAL=5
 NETWORK_INTERFACE=
 

analyser/processQueue.sh

@@ -44,6 +44,8 @@ fi
 : "${IOS_STAGING_CLEANUP_TIMEOUT:=30}"
 : "${CONSECUTIVE_FAILURE_LIMIT:=5}"
 : "${CIRCUIT_BREAKER_SLEEP:=3600}"
+: "${IDLE_SLEEP_MIN:=60}"
+: "${IDLE_SLEEP_MAX:=1800}"
 : "${DOWNLOAD_WATCHDOG_INTERVAL:=5}"
 : "${NETWORK_INTERFACE:=}"
 : "${DAILY_BYTES_FILE:=./daily-download-bytes.txt}"
@@ -113,6 +115,10 @@ fi
 curl_auth_config=$(mktemp)
 chmod 600 "$curl_auth_config"
 printf 'header = "Authorization: Bearer %s"\n' "$UPLOAD_PASSWORD" > "$curl_auth_config"
+# The server may be asleep (Railway stops idle services) and answer its first
+# request with a 502 or refuse the connection while it starts, so every call
+# retries transient failures for up to a minute.
+printf 'retry = 5\nretry-delay = 5\nretry-max-time = 60\nretry-connrefused\n' >> "$curl_auth_config"
 queue_headers=$(mktemp)
 chmod 600 "$queue_headers"
 trap 'rm -f "$curl_auth_config" "$queue_headers"' EXIT
@@ -784,6 +790,20 @@ process_app()
 	[ "$success" -eq 1 ]
 }
 
+# An empty queue is polled with a doubling delay, so the server sees no traffic
+# for long enough to sleep. Any assigned app resets it.
+idle_sleep="$IDLE_SLEEP_MIN"
+
+idle_backoff()
+{
+	echo "Queue empty. Checking again in ${idle_sleep}s"
+	sleep "$idle_sleep"
+	idle_sleep=$((idle_sleep * 2))
+	if [ "$idle_sleep" -gt "$IDLE_SLEEP_MAX" ]; then
+		idle_sleep="$IDLE_SLEEP_MAX"
+	fi
+}
+
 if [ -n "$ONLY_APP_ID" ]; then
 	reset_daily_counter
 
@@ -846,5 +866,10 @@ while true; do
 		exit 0
 	fi
 
-	sleepabit
+	if [ "$appId" == "" ] ; then
+		idle_backoff
+	else
+		idle_sleep="$IDLE_SLEEP_MIN"
+		sleepabit
+	fi
 done

migrations/015_analyser_heartbeat.sql

@@ -0,0 +1,7 @@
+-- When the analyser last checked in. Kept in the database so a restarted or
+-- woken web process still knows the analyser is online.
+
+CREATE TABLE IF NOT EXISTS analyser_heartbeat (
+    id boolean PRIMARY KEY DEFAULT true CHECK (id),
+    last_seen timestamptz NOT NULL
+);

models/Apps.js

@@ -503,6 +503,19 @@ const getSiteDataSignature = async () => {
     };
 }
 
+const recordAnalyserHeartbeat = async () => {
+    await pool.query(`
+        INSERT INTO analyser_heartbeat (id, last_seen) VALUES (true, NOW())
+        ON CONFLICT (id) DO UPDATE SET last_seen = EXCLUDED.last_seen
+    `);
+}
+
+// Milliseconds since the epoch, or 0 if the analyser has never checked in.
+const lastAnalyserHeartbeat = async () => {
+    const result = await pool.query('SELECT last_seen FROM analyser_heartbeat');
+    return result.rows.length > 0 ? result.rows[0].last_seen.getTime() : 0;
+}
+
 const countAnalysed = async () => {
     const result = await pool.query("SELECT COUNT(*) FROM apps WHERE status = 'analysed'");
     return parseInt(result.rows[0].count, 10);
@@ -519,6 +532,8 @@ module.exports = {
     nextApp,
     updateAnalysis,
     getAllApps,
+    recordAnalyserHeartbeat,
+    lastAnalyserHeartbeat,
     getSiteDataSignature,
     healthCheck,
     deriveAnalysisState,

raspberry-pi-analyser.md

@@ -85,9 +85,13 @@ MAX_APP_SIZE_BYTES=1800000000
 MAX_ATTEMPT_DOWNLOAD_BYTES=1800000000
 CONSECUTIVE_FAILURE_LIMIT=5
 CIRCUIT_BREAKER_SLEEP=3600
+IDLE_SLEEP_MIN=60
+IDLE_SLEEP_MAX=1800
 DOWNLOAD_WATCHDOG_INTERVAL=5
 ```
 
+When the queue is empty the analyser waits `IDLE_SLEEP_MIN` seconds before asking again, doubling the wait up to `IDLE_SLEEP_MAX`, and goes back to the minimum as soon as it gets an app. This lets a hosting platform that stops idle services, such as Railway, put the website to sleep while there is nothing to analyse. A newly requested app can therefore wait up to `IDLE_SLEEP_MAX` before analysis starts. Requests to the server retry for up to a minute, so the analyser rides out the server waking up.
+
 Set `NETWORK_INTERFACE` only if auto-detection picks the wrong interface. On a Pi this is often `eth0` or `wlan0`.
 
 ## Use RAM-Backed IPA Storage

routes/index.js

@@ -30,7 +30,25 @@ const APPS_PER_PAGE = 50;
 const MAX_SITEMAP_URLS = 50000;
 const MAX_SITEMAP_BYTES = 50 * 1024 * 1024;
 
+const ANALYSER_ONLINE_MS = 1000*60*60;
 let lastPing = 0; // unix timestamp
+let lastPingLoad = null;
+
+// lastPing starts at zero in a fresh process, and on a platform that stops
+// idle services every wake is one, so the last check-in is read back from the
+// database once. A failed read is retried a minute later rather than on every
+// request.
+function loadLastPing() {
+  if (!lastPingLoad) {
+    lastPingLoad = Apps.lastAnalyserHeartbeat().then((stored) => {
+      lastPing = Math.max(lastPing, stored);
+    }, (err) => {
+      console.error('Could not read analyser heartbeat:', err.message);
+      setTimeout(() => { lastPingLoad = null; }, 60 * 1000).unref();
+    });
+  }
+  return lastPingLoad;
+}
 
 function requireValidAppId(req, res, next) {
   if (!isValidAppId(req.params.appId))
@@ -66,8 +84,10 @@ function redirectPublic(res, status, path) {
 
 // ping from analyser in past hour?
 router.use(function (req, res, next) {
-  res.locals.analyserOnline = lastPing > Date.now() - 1000*60*60;
-  next();
+  loadLastPing().then(() => {
+    res.locals.analyserOnline = lastPing > Date.now() - ANALYSER_ONLINE_MS;
+    next();
+  });
 });
 
 function thirdPartyTrackerNames(analysis) {
@@ -423,7 +443,7 @@ router.get('/healthz', asyncHandler(async (req, res) => {
 }));
 
 router.get('/healthz/analyser', (req, res) => {
-  const online = lastPing > Date.now() - 1000*60*60;
+  const online = lastPing > Date.now() - ANALYSER_ONLINE_MS;
   res.status(online ? 200 : 503).json({ ok: online });
 });
 
@@ -789,6 +809,9 @@ router.get('/queue', asyncHandler(async (req, res) => {
 // enable analyser to report online status
 router.get('/ping', (req, res) => {
     lastPing = Date.now();
+    Apps.recordAnalyserHeartbeat().catch((err) => {
+      console.error('Could not record analyser heartbeat:', err.message);
+    });
 
     res.send("online");
 });

test/analyserHeartbeat.test.js

@@ -0,0 +1,48 @@
+const assert = require('node:assert/strict');
+const test = require('node:test');
+
+// Runs in its own process, so the server's one-off heartbeat read happens
+// against the stubs below.
+test('analyser status survives a restart through the stored heartbeat', async () => {
+  process.env.UPLOAD_PASSWORD = 'test-secret';
+  const Apps = require('../models/Apps');
+  const original = {
+    lastAnalyserHeartbeat: Apps.lastAnalyserHeartbeat,
+    recordAnalyserHeartbeat: Apps.recordAnalyserHeartbeat,
+  };
+  let reads = 0;
+  let writes = 0;
+  Apps.lastAnalyserHeartbeat = async () => {
+    reads++;
+    return Date.now() - 10 * 60 * 1000;
+  };
+  Apps.recordAnalyserHeartbeat = async () => {
+    writes++;
+  };
+
+  const app = require('../server');
+  const server = await new Promise((resolve) => {
+    const instance = app.listen(0, () => resolve(instance));
+  });
+
+  try {
+    const base = `http://127.0.0.1:${server.address().port}`;
+    const first = await fetch(`${base}/healthz/analyser`);
+    const second = await fetch(`${base}/healthz/analyser`);
+    assert.equal(first.status, 200);
+    assert.equal(second.status, 200);
+    assert.equal(reads, 1);
+
+    const ping = await fetch(`${base}/ping`, {
+      headers: { authorization: 'Bearer test-secret' }
+    });
+    assert.equal(ping.status, 200);
+    assert.equal(writes, 1);
+  } finally {
+    Apps.lastAnalyserHeartbeat = original.lastAnalyserHeartbeat;
+    Apps.recordAnalyserHeartbeat = original.recordAnalyserHeartbeat;
+    await new Promise((resolve, reject) => {
+      server.close((err) => err ? reject(err) : resolve());
+    });
+  }
+});