From b9fd949eeb7777f241171c8d65fbe019c66bed3d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 07:54:10 +0000 Subject: [PATCH] Wake the analyser on demand instead of polling the website The analyser called /ping and /queue every 30 seconds, so the website, and through it the database, never went quiet long enough for Railway to sleep them. The analyser now waits while the queue is empty and only contacts the website when there is work: - /queue reports, via X-Analysis-Next-Due-Seconds, when the next app falls due on its own (a stale analysis, a retryable failure, an abandoned claim), from a query mirroring nextApp's conditions. - Queueing an app (the request form or queue-refetch) posts a bare "wake" to an ntfy topic (NTFY_TOPIC, optionally NTFY_TOKEN), which the idle analyser subscribes to. A lost wake-up only delays the analysis. - The analyser asks at least daily (MAX_IDLE_WAIT), and every /queue call records a check-in, stored in a new analyser_heartbeat row so a woken web process still knows the analyser is online. The separate /ping endpoint is removed and the online window widens to 26 hours. - curl retries transient failures for up to a minute, so requests survive the website waking up. Claude-Session: https://claude.ai/code/session_01Fi3H98SycMK13rrr8ySSU1 --- README.md | 14 ++- analyser/.env.example | 13 ++- analyser/processQueue.sh | 84 ++++++++++++++--- lib/analyserWake.js | 61 +++++++++++++ migrations/015_analyser_heartbeat.sql | 7 ++ models/Apps.js | 46 ++++++++++ raspberry-pi-analyser.md | 2 + routes/index.js | 61 ++++++++++--- scripts/queue-refetch.js | 2 + server.js | 1 - test/analyserQueue.test.js | 124 ++++++++++++++++++++++++++ test/analyserWake.test.js | 61 +++++++++++++ test/auth.test.js | 16 +++- 13 files changed, 459 insertions(+), 33 deletions(-) create mode 100644 lib/analyserWake.js create mode 100644 migrations/015_analyser_heartbeat.sql create mode 100644 test/analyserQueue.test.js create mode 100644 test/analyserWake.test.js diff --git a/README.md b/README.md index ffda933..65c0b20 100644 --- a/README.md +++ b/README.md @@ -100,11 +100,18 @@ APP_STORE_CACHE_RETENTION_DAYS=90 PORT=3000 CLOUDFLARE_API_TOKEN=change-me CLOUDFLARE_ZONE_ID=change-me +NTFY_TOPIC= +NTFY_TOKEN= ``` `BODY_LIMIT` applies to authenticated analyser JSON and text uploads. `PUBLIC_FORM_BODY_LIMIT` is the smaller limit for the public analysis request form. `APP_STORE_CACHE_RETENTION_DAYS` controls how long cached App Store metadata is kept. +`NTFY_TOPIC` names the [ntfy](https://ntfy.sh) topic the website posts to when an +app is queued, which wakes an idle analyser straight away; use the same value in +`analyser/.env` (see [Queue And Operations](#queue-and-operations)). +`NTFY_TOKEN` is optional: an ntfy account access token, so publishing is not +subject to ntfy.sh's per-IP limits for anonymous use. Set `SITE_URL` in production to the public origin, for example `SITE_URL=https://ios.trackercontrol.org`. Canonical links, Open Graph URLs, @@ -262,14 +269,15 @@ For maintenance, use `sudo bash /opt/tracker-control-ios/scripts/manage-raspi-ip The website exposes analyser endpoints: -- `GET /queue` returns the next app to process in the response body and its one-use assignment token in the `X-Analysis-Claim-Token` response header. -- `GET /ping` marks the analyser online. +- `GET /queue` returns the next app to process in the response body and its one-use assignment token in the `X-Analysis-Claim-Token` response header. With the queue empty the body is empty, and `X-Analysis-Next-Due-Seconds` says when the next app falls due on its own (a stale analysis, a retryable failure, an abandoned claim); the header is absent if none ever will. Each call also records that the analyser is online. - `POST /uploadAnalysis` stores successful analysis results when sent with the assignment's `X-Analysis-Claim-Token` header. - `POST /reportAnalysisFailure` stores failed analysis results when sent with the assignment's `X-Analysis-Claim-Token` header. Analyzer requests authenticate with `Authorization: Bearer $UPLOAD_PASSWORD`. Authentication and assignment tokens serve different purposes: the bearer token authorises the analyser, while the assignment token prevents an expired worker from overwriting a newer result. +The analyser does not poll. With the queue empty it waits until the next app falls due, or until the website posts to the `NTFY_TOPIC` ntfy topic after queueing an app, whichever comes first, and asks again at least once a day (`MAX_IDLE_WAIT`). The website and its database therefore only see traffic when there is work or a visitor, so a host that stops idle services, such as Railway, can let them sleep. A failed or missing wake-up only delays an analysis until the analyser's next check. + ### Claim-token rollout The claim-token migration deliberately requeues every in-flight assignment that predates tokens. Use a coordinated deployment: @@ -284,7 +292,7 @@ The `/queue` response body remains a plain bundle ID, but older analyser scripts Health checks: - `GET /healthz` returns `200` when the website can reach PostgreSQL. -- `GET /healthz/analyser` returns `200` when the analyser has pinged in the last hour. +- `GET /healthz/analyser` returns `200` when the analyser has asked for work in the last 26 hours. Useful maintenance commands: diff --git a/analyser/.env.example b/analyser/.env.example index 1b8e220..15f07d7 100644 --- a/analyser/.env.example +++ b/analyser/.env.example @@ -1,4 +1,4 @@ -# Web server that exposes /queue, /ping, /uploadAnalysis, and /reportAnalysisFailure. +# Web server that exposes /queue, /uploadAnalysis, and /reportAnalysisFailure. SERVER=https://your-railway-app.example # Must match UPLOAD_PASSWORD on the web server. @@ -27,6 +27,17 @@ MAX_APP_SIZE_BYTES=3000000000 MAX_ATTEMPT_DOWNLOAD_BYTES=3000000000 CONSECUTIVE_FAILURE_LIMIT=5 CIRCUIT_BREAKER_SLEEP=3600 + +# Waiting for work. The website posts to this ntfy topic (https://ntfy.sh) when +# an app is queued; use the same NTFY_TOPIC (and NTFY_TOKEN, if any) as the web +# server. Pick a long random name, e.g. `openssl rand -hex 16`. Without a topic +# the analyser only checks when the next app falls due, or after MAX_IDLE_WAIT. +NTFY_TOPIC= +NTFY_TOKEN= +NTFY_SERVER=https://ntfy.sh +MIN_IDLE_WAIT=60 +MAX_IDLE_WAIT=86400 +QUEUE_RETRY_WAIT=300 DOWNLOAD_WATCHDOG_INTERVAL=5 NETWORK_INTERFACE= diff --git a/analyser/processQueue.sh b/analyser/processQueue.sh index 94db701..1771715 100755 --- a/analyser/processQueue.sh +++ b/analyser/processQueue.sh @@ -44,6 +44,12 @@ fi : "${IOS_STAGING_CLEANUP_TIMEOUT:=30}" : "${CONSECUTIVE_FAILURE_LIMIT:=5}" : "${CIRCUIT_BREAKER_SLEEP:=3600}" +: "${NTFY_TOPIC:=}" +: "${NTFY_SERVER:=https://ntfy.sh}" +: "${NTFY_TOKEN:=}" +: "${MIN_IDLE_WAIT:=60}" +: "${MAX_IDLE_WAIT:=86400}" +: "${QUEUE_RETRY_WAIT:=300}" : "${DOWNLOAD_WATCHDOG_INTERVAL:=5}" : "${NETWORK_INTERFACE:=}" : "${DAILY_BYTES_FILE:=./daily-download-bytes.txt}" @@ -113,6 +119,10 @@ fi curl_auth_config=$(mktemp) chmod 600 "$curl_auth_config" printf 'header = "Authorization: Bearer %s"\n' "$UPLOAD_PASSWORD" > "$curl_auth_config" +# The server may be asleep (Railway stops idle services) and answer its first +# request with a 502 or refuse the connection while it starts, so every call +# retries transient failures for up to a minute. +printf 'retry = 5\nretry-delay = 5\nretry-max-time = 60\nretry-connrefused\n' >> "$curl_auth_config" queue_headers=$(mktemp) chmod 600 "$queue_headers" trap 'rm -f "$curl_auth_config" "$queue_headers"' EXIT @@ -434,6 +444,7 @@ fetch_queue_app() : > "$queue_headers" appId=$(curl -sS -K "$curl_auth_config" -D "$queue_headers" "$queueUrl" --fail) || return 1 claimToken=$(awk 'tolower($1) == "x-analysis-claim-token:" { gsub(/\r/, "", $2); print $2; exit }' "$queue_headers") + nextDueSeconds=$(awk 'tolower($1) == "x-analysis-next-due-seconds:" { gsub(/\r/, "", $2); print $2; exit }' "$queue_headers") if [ -n "$appId" ] && [ -z "$claimToken" ]; then echo "Queue response for $appId did not include an analysis claim token." >&2 @@ -784,12 +795,58 @@ process_app() [ "$success" -eq 1 ] } +# The website is never polled: with the queue empty, wait until the next app +# falls due on its own (the server says when) or until the website announces +# a newly queued app on the ntfy topic, whichever comes first. Every /queue +# call also counts as a check-in, and MAX_IDLE_WAIT bounds the wait so the +# website keeps seeing the analyser at least daily. +wait_for_work() +{ + local wait="$MAX_IDLE_WAIT" started elapsed line wake_pid woken=0 + + if [[ "$nextDueSeconds" =~ ^[0-9]+$ ]] && [ "$nextDueSeconds" -lt "$wait" ]; then + wait="$nextDueSeconds" + fi + if [ "$wait" -lt "$MIN_IDLE_WAIT" ]; then + wait="$MIN_IDLE_WAIT" + fi + started=$(date +%s) + + if [ -n "$NTFY_TOPIC" ]; then + echo "Queue empty. Waiting up to ${wait}s for a wake-up." + # since= replays a wake-up sent between the /queue check and this + # subscription. ntfy sends keepalives as empty lines. + exec {wake_fd}< <(exec curl -sN --max-time "$wait" \ + ${NTFY_TOKEN:+-H "Authorization: Bearer $NTFY_TOKEN"} \ + "${NTFY_SERVER%/}/$NTFY_TOPIC/raw?since=$queue_checked_at") + wake_pid=$! + while IFS= read -r -u "$wake_fd" line; do + if [ -n "$line" ]; then + woken=1 + break + fi + done + kill "$wake_pid" 2>/dev/null + exec {wake_fd}<&- + else + echo "Queue empty. Checking again in ${wait}s (set NTFY_TOPIC to be woken for new requests)." + fi + + if [ "$woken" -eq 1 ]; then + echo "Woken up: an app was queued." + return + fi + + # Also covers ntfy being unreachable, so the website is not hammered. + elapsed=$(( $(date +%s) - started )) + if [ "$elapsed" -lt "$wait" ]; then + sleep $(( wait - elapsed )) + fi +} + if [ -n "$ONLY_APP_ID" ]; then reset_daily_counter - echo "Reporting online status apps to install" - curl -s -K "$curl_auth_config" "$SERVER/ping" - if ! check_daily_limit; then exit 1 fi @@ -814,9 +871,6 @@ fi while true; do reset_daily_counter - echo "Reporting online status apps to install" - curl -s -K "$curl_auth_config" "$SERVER/ping" - if ! check_daily_limit; then sleep "$CIRCUIT_BREAKER_SLEEP" continue @@ -830,18 +884,24 @@ while true; do fi echo "Fetching apps to install" + queue_checked_at=$(date +%s) + nextDueSeconds="" if ! fetch_queue_app "$SERVER/queue"; then - echo "Failed to fetch a queue assignment." - appId="" - claimToken="" + echo "Failed to fetch a queue assignment. Retrying in ${QUEUE_RETRY_WAIT}s." + [ "$RUN_ONCE" = "1" ] && exit 1 + sleep "$QUEUE_RETRY_WAIT" + continue fi if [ "$appId" == "" ] ; then - echo "No app to process.." - else - process_app "$appId" "$claimToken" + echo "No app to process.." + [ "$RUN_ONCE" = "1" ] && exit 0 + wait_for_work + continue fi + process_app "$appId" "$claimToken" + if [ "$RUN_ONCE" = "1" ]; then exit 0 fi diff --git a/lib/analyserWake.js b/lib/analyserWake.js new file mode 100644 index 0000000..a448755 --- /dev/null +++ b/lib/analyserWake.js @@ -0,0 +1,61 @@ +// Tells a waiting analyser that an app was queued, through an ntfy topic +// (https://ntfy.sh) that analyser/processQueue.sh subscribes to while idle. +// The analyser never polls the website, so without this a newly requested app +// waits until the analyser's next scheduled check. The message carries no +// data; it only ends the analyser's wait. Inert unless NTFY_TOPIC is set. +// ntfy.sh rate-limits anonymous publishing per IP address, which a shared +// hosting egress address can exhaust; NTFY_TOKEN (an ntfy account access +// token) publishes under the account's own limits instead. + +const DEFAULT_SERVER = 'https://ntfy.sh'; +const TIMEOUT_MS = 3000; +// ntfy topic names: letters, digits, dashes and underscores, at most 64. +const TOPIC_PATTERN = /^[A-Za-z0-9_-]{1,64}$/; + +function wakeUrl({ + topic = process.env.NTFY_TOPIC, + server = process.env.NTFY_SERVER +} = {}) { + topic = (topic || '').trim(); + if (!topic) return null; + if (!TOPIC_PATTERN.test(topic)) { + console.error('NTFY_TOPIC must be 1-64 letters, digits, dashes or underscores; analyser wake-ups are disabled.'); + return null; + } + const base = ((server || '').trim() || DEFAULT_SERVER).replace(/\/+$/, ''); + return `${base}/${topic}`; +} + +/** + * Best effort: resolves to true if the notification was accepted, false + * otherwise. Never throws, so callers need not await it. A missed wake-up only + * delays the analysis until the analyser's next scheduled check. + */ +async function wakeAnalyser({ + fetchImpl = fetch, + token = process.env.NTFY_TOKEN, + ...options +} = {}) { + const url = wakeUrl(options); + if (!url) return false; + + token = (token || '').trim(); + try { + const response = await fetchImpl(url, { + method: 'POST', + body: 'wake', + headers: token ? { authorization: `Bearer ${token}` } : {}, + signal: AbortSignal.timeout(TIMEOUT_MS) + }); + if (!response.ok) { + console.error('Analyser wake-up rejected:', response.status); + return false; + } + return true; + } catch (err) { + console.error('Analyser wake-up failed:', err.message); + return false; + } +} + +module.exports = { wakeAnalyser, wakeUrl }; diff --git a/migrations/015_analyser_heartbeat.sql b/migrations/015_analyser_heartbeat.sql new file mode 100644 index 0000000..d1fae9b --- /dev/null +++ b/migrations/015_analyser_heartbeat.sql @@ -0,0 +1,7 @@ +-- When the analyser last checked in. Kept in the database so a restarted or +-- woken web process still knows the analyser is online. + +CREATE TABLE IF NOT EXISTS analyser_heartbeat ( + id boolean PRIMARY KEY DEFAULT true CHECK (id), + last_seen timestamptz NOT NULL +); diff --git a/models/Apps.js b/models/Apps.js index 3daaafd..af14c04 100644 --- a/models/Apps.js +++ b/models/Apps.js @@ -518,6 +518,49 @@ const getSiteDataSignature = async () => { }; } +// When the analyser next has work if nothing new is queued: the earliest moment +// an app becomes claimable by nextApp through the passage of time alone (a +// stale analysis, a retryable failure, an abandoned claim). Mirrors the +// candidate conditions in nextApp. Returns a Date, or null if no app will ever +// become due without being queued. +const nextDueAt = async () => { + const result = await pool.query(` + SELECT MIN(due) AS due + FROM ( + SELECT CASE + WHEN status = 'processing' + THEN processing_started + ($3::int * INTERVAL '1 minute') + WHEN (status = 'analysed' + OR (status = 'failed' AND failure_retryable)) + AND analysisversion IS NOT DISTINCT FROM $1 + THEN analysed + ($2::int * INTERVAL '1 day') + END AS due + FROM apps + WHERE ${appIdSqlPredicate(4, 5)} + ) candidates + `, [ + currentAnalysisVersion, + staleAnalysisDays, + processingTimeoutMinutes, + APP_ID_PATTERN_SOURCE, + MAX_APP_ID_LENGTH + ]); + return result.rows[0].due; +} + +const recordAnalyserHeartbeat = async () => { + await pool.query(` + INSERT INTO analyser_heartbeat (id, last_seen) VALUES (true, NOW()) + ON CONFLICT (id) DO UPDATE SET last_seen = EXCLUDED.last_seen + `); +} + +// Milliseconds since the epoch, or 0 if the analyser has never checked in. +const lastAnalyserHeartbeat = async () => { + const result = await pool.query('SELECT last_seen FROM analyser_heartbeat'); + return result.rows.length > 0 ? result.rows[0].last_seen.getTime() : 0; +} + const countAnalysed = async () => { const result = await pool.query("SELECT COUNT(*) FROM apps WHERE status = 'analysed'"); return parseInt(result.rows[0].count, 10); @@ -532,6 +575,9 @@ module.exports = { countAnalysed, addApp, nextApp, + nextDueAt, + recordAnalyserHeartbeat, + lastAnalyserHeartbeat, updateAnalysis, getAllApps, getSiteDataSignature, diff --git a/raspberry-pi-analyser.md b/raspberry-pi-analyser.md index d2a9a09..13c476d 100644 --- a/raspberry-pi-analyser.md +++ b/raspberry-pi-analyser.md @@ -88,6 +88,8 @@ CIRCUIT_BREAKER_SLEEP=3600 DOWNLOAD_WATCHDOG_INTERVAL=5 ``` +The analyser does not poll the website. When the queue is empty it waits until the next app falls due (the website says when), until the website announces a newly queued app on an [ntfy](https://ntfy.sh) topic, or for `MAX_IDLE_WAIT` seconds (a day by default), whichever comes first. Set the same long random topic name as `NTFY_TOPIC` here and on the web server, for example from `openssl rand -hex 16`; without it, a newly requested app waits for the next of the other two. Requests to the website retry for up to a minute, so the analyser rides out the website waking up from sleep. + Set `NETWORK_INTERFACE` only if auto-detection picks the wrong interface. On a Pi this is often `eth0` or `wlan0`. ## Use RAM-Backed IPA Storage diff --git a/routes/index.js b/routes/index.js index 2f3d6d6..87876a3 100644 --- a/routes/index.js +++ b/routes/index.js @@ -13,6 +13,7 @@ const { buildReportMetadata, buildListingDetails } = require('../lib/appMetadata const { siteBaseUrl } = require('../lib/siteUrl'); const cloudflare = require('../lib/cloudflarePurge'); const { canonicalUrl } = require('../lib/i18n'); +const { wakeAnalyser } = require('../lib/analyserWake'); // Taken from https://reports.exodus-privacy.eu.org/api/trackers const exodusTrackers = JSON.parse(fs.readFileSync('./exodusTrackers.json', 'utf-8')) @@ -30,7 +31,28 @@ const APPS_PER_PAGE = 50; const MAX_SITEMAP_URLS = 50000; const MAX_SITEMAP_BYTES = 50 * 1024 * 1024; +// The analyser only contacts the website when it has work or a scheduled +// check, at least daily plus the length of one analysis, so it counts as +// online for 26 hours. +const ANALYSER_ONLINE_MS = 26 * 60 * 60 * 1000; let lastPing = 0; // unix timestamp +let lastPingLoad = null; + +// lastPing starts at zero in a fresh process, and on a platform that stops +// idle services every wake is one, so the last check-in is read back from the +// database once. A failed read is retried a minute later rather than on every +// request. +function loadLastPing() { + if (!lastPingLoad) { + lastPingLoad = Apps.lastAnalyserHeartbeat().then((stored) => { + lastPing = Math.max(lastPing, stored); + }, (err) => { + console.error('Could not read analyser heartbeat:', err.message); + setTimeout(() => { lastPingLoad = null; }, 60 * 1000).unref(); + }); + } + return lastPingLoad; +} function requireValidAppId(req, res, next) { if (!isValidAppId(req.params.appId)) @@ -64,10 +86,12 @@ function redirectPublic(res, status, path) { return res.redirect(status, local(path)); } -// ping from analyser in past hour? +// has the analyser checked in recently? router.use(function (req, res, next) { - res.locals.analyserOnline = lastPing > Date.now() - 1000*60*60; - next(); + loadLastPing().then(() => { + res.locals.analyserOnline = lastPing > Date.now() - ANALYSER_ONLINE_MS; + next(); + }); }); function thirdPartyTrackerNames(analysis) { @@ -440,7 +464,7 @@ router.get('/healthz', asyncHandler(async (req, res) => { })); router.get('/healthz/analyser', (req, res) => { - const online = lastPing > Date.now() - 1000*60*60; + const online = lastPing > Date.now() - ANALYSER_ONLINE_MS; res.status(online ? 200 : 503).json({ ok: online }); }); @@ -674,6 +698,7 @@ router.post('/analysis/:appId', }); } + wakeAnalyser(); return redirectPublic(res, 303, `/analysis/${details.appId}`); })); @@ -791,11 +816,29 @@ router.get('/queue', asyncHandler(async (req, res) => { if (requestedAppId && !isValidAppId(requestedAppId)) return res.status(400).send('Please provide a valid App Store bundle ID.'); + lastPing = Date.now(); + Apps.recordAnalyserHeartbeat().catch((err) => { + console.error('Could not record analyser heartbeat:', err.message); + }); + let app = await Apps.nextApp(requestedAppId); console.log(app ? app.appid : null); - if (!app) + if (!app) { + // The analyser waits for a wake-up (see lib/analyserWake.js) or until the + // next app falls due on its own, whichever comes first. Without the + // header it falls back to its own maximum wait. + try { + const due = await Apps.nextDueAt(); + if (due) { + const seconds = Math.max(0, Math.ceil((due.getTime() - Date.now()) / 1000)); + res.set('X-Analysis-Next-Due-Seconds', String(seconds)); + } + } catch (err) { + console.error('Could not compute the next due analysis:', err.message); + } return res.send(); + } // Keep the body as the bundle ID for deployed analysers while newer // clients carry the per-assignment token in a separate header. @@ -803,13 +846,6 @@ router.get('/queue', asyncHandler(async (req, res) => { res.send(app.appid); })); -// enable analyser to report online status -router.get('/ping', (req, res) => { - lastPing = Date.now(); - - res.send("online"); -}); - // upload analysis results router.post('/uploadAnalysis', asyncHandler(async (req, res) => { if (!req.query.appId || !req.query.analysisVersion) @@ -1004,7 +1040,6 @@ router.get('/robots.txt', (req, res) => { 'Disallow: /search', 'Disallow: /request/', 'Disallow: /queue', - 'Disallow: /ping', 'Disallow: /healthz', '', `Sitemap: ${siteBaseUrl(req)}/sitemap.xml`, diff --git a/scripts/queue-refetch.js b/scripts/queue-refetch.js index 73f882b..8a5d980 100644 --- a/scripts/queue-refetch.js +++ b/scripts/queue-refetch.js @@ -4,6 +4,7 @@ const path = require('path'); const dotenv = require('dotenv'); const { Client } = require('pg'); const { buildAnalysisProvenanceSourceSql } = require('../models/Apps'); +const { wakeAnalyser } = require('../lib/analyserWake'); dotenv.config({ path: path.join(__dirname, '..', '.env') }); dotenv.config({ path: path.join(__dirname, '..', 'analyser', '.env') }); @@ -155,6 +156,7 @@ async function main() { console.log(`Queued ${rows.length} apps for refetch.`); printRows(rows); await client.end(); + if (rows.length > 0) await wakeAnalyser(); } main().catch((error) => { diff --git a/server.js b/server.js index c77cd5c..cc0e669 100644 --- a/server.js +++ b/server.js @@ -38,7 +38,6 @@ app.disable('x-powered-by') const os = require('os'); const analyserPaths = new Set([ '/queue', - '/ping', '/uploadanalysis', '/reportanalysisfailure' ]); diff --git a/test/analyserQueue.test.js b/test/analyserQueue.test.js new file mode 100644 index 0000000..ab48334 --- /dev/null +++ b/test/analyserQueue.test.js @@ -0,0 +1,124 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); + +// Runs in its own process, so the server's one-off heartbeat read happens +// against the stubs below. +process.env.UPLOAD_PASSWORD = 'test-secret'; +process.env.NTFY_TOPIC = 'tc-test-topic'; +const Apps = require('../models/Apps'); +const store = require('../lib/appStore'); + +const state = { reads: 0, writes: 0, nextApp: null, nextDue: null }; +Apps.lastAnalyserHeartbeat = async () => { + state.reads++; + return Date.now() - 20 * 60 * 60 * 1000; +}; +Apps.recordAnalyserHeartbeat = async () => { state.writes++; }; +Apps.nextApp = async () => state.nextApp; +Apps.nextDueAt = async () => state.nextDue; + +const wakeUps = []; +const realFetch = globalThis.fetch; +globalThis.fetch = async (url, options) => { + if (String(url).startsWith('https://ntfy.sh/')) { + wakeUps.push({ url: String(url), body: options.body }); + return { ok: true, status: 200 }; + } + return realFetch(url, options); +}; + +const app = require('../server'); + +async function withServer(run) { + const server = await new Promise((resolve) => { + const instance = app.listen(0, () => resolve(instance)); + }); + try { + await run(`http://127.0.0.1:${server.address().port}`); + } finally { + await new Promise((resolve, reject) => { + server.close((err) => err ? reject(err) : resolve()); + }); + } +} + +const auth = { authorization: 'Bearer test-secret' }; + +test('a stored check-in from the last day keeps the analyser online across restarts', async () => { + await withServer(async (base) => { + assert.equal((await fetch(`${base}/healthz/analyser`)).status, 200); + assert.equal((await fetch(`${base}/healthz/analyser`)).status, 200); + }); + assert.equal(state.reads, 1); +}); + +test('an empty queue tells the analyser when the next app falls due', async () => { + const originalConsoleLog = console.log; + console.log = () => {}; + try { + await withServer(async (base) => { + state.nextApp = null; + state.nextDue = new Date(Date.now() + 3600 * 1000); + const due = await fetch(`${base}/queue`, { headers: auth }); + assert.equal(due.status, 200); + assert.equal(await due.text(), ''); + const seconds = Number(due.headers.get('x-analysis-next-due-seconds')); + assert.ok(seconds > 3590 && seconds <= 3600, String(seconds)); + + state.nextDue = new Date(Date.now() - 1000); + const overdue = await fetch(`${base}/queue`, { headers: auth }); + assert.equal(overdue.headers.get('x-analysis-next-due-seconds'), '0'); + + state.nextDue = null; + const never = await fetch(`${base}/queue`, { headers: auth }); + assert.equal(never.headers.get('x-analysis-next-due-seconds'), null); + + state.nextApp = { appid: 'com.example.app', analysis_claim_token: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' }; + state.nextDue = new Date(Date.now() + 3600 * 1000); + const claimed = await fetch(`${base}/queue`, { headers: auth }); + assert.equal(await claimed.text(), 'com.example.app'); + assert.equal(claimed.headers.get('x-analysis-next-due-seconds'), null); + }); + } finally { + console.log = originalConsoleLog; + } + assert.equal(state.writes, 4); +}); + +test('the separate ping endpoint is gone', async () => { + await withServer(async (base) => { + const response = await fetch(`${base}/ping`, { headers: { ...auth, accept: 'text/plain' } }); + assert.equal(response.status, 404); + }); +}); + +test('queueing an app wakes the analyser', async () => { + const original = { + findApp: Apps.findApp, + findCachedAppStoreResult: Apps.findCachedAppStoreResult, + addAppAndStorefront: Apps.addAppAndStorefront, + storeApp: store.app, + }; + Apps.findApp = async () => null; + Apps.findCachedAppStoreResult = async () => null; + Apps.addAppAndStorefront = async () => ({ rowCount: 1 }); + store.app = async () => ({ appId: 'com.example.Wake', title: 'Wake', free: true }); + wakeUps.length = 0; + + try { + await withServer(async (base) => { + const response = await fetch(`${base}/analysis/com.example.wake`, { + method: 'POST', + headers: { 'content-type': 'application/x-www-form-urlencoded' }, + redirect: 'manual' + }); + assert.equal(response.status, 303); + }); + } finally { + Apps.findApp = original.findApp; + Apps.findCachedAppStoreResult = original.findCachedAppStoreResult; + Apps.addAppAndStorefront = original.addAppAndStorefront; + store.app = original.storeApp; + } + assert.deepEqual(wakeUps, [{ url: 'https://ntfy.sh/tc-test-topic', body: 'wake' }]); +}); diff --git a/test/analyserWake.test.js b/test/analyserWake.test.js new file mode 100644 index 0000000..62c8205 --- /dev/null +++ b/test/analyserWake.test.js @@ -0,0 +1,61 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { wakeAnalyser, wakeUrl } = require('../lib/analyserWake'); + +test('wake-ups are disabled without a valid topic', () => { + const originalConsoleError = console.error; + console.error = () => {}; + try { + assert.equal(wakeUrl({ topic: '' }), null); + assert.equal(wakeUrl({ topic: ' ' }), null); + assert.equal(wakeUrl({ topic: 'a/b' }), null); + assert.equal(wakeUrl({ topic: 'x'.repeat(65) }), null); + } finally { + console.error = originalConsoleError; + } +}); + +test('the topic URL defaults to ntfy.sh and accepts a self-hosted server', () => { + assert.equal(wakeUrl({ topic: 'tc-ios_42' }), 'https://ntfy.sh/tc-ios_42'); + assert.equal(wakeUrl({ topic: 'tc', server: 'https://ntfy.example.test//' }), 'https://ntfy.example.test/tc'); +}); + +test('a wake-up posts a bare message to the topic', async () => { + const calls = []; + const fetchImpl = async (url, options) => { + calls.push({ url, method: options.method, body: options.body, headers: options.headers }); + return { ok: true, status: 200 }; + }; + assert.equal(await wakeAnalyser({ topic: 'tc', token: '', fetchImpl }), true); + assert.equal(await wakeAnalyser({ topic: 'tc', token: 'tk_secret', fetchImpl }), true); + assert.deepEqual(calls, [ + { url: 'https://ntfy.sh/tc', method: 'POST', body: 'wake', headers: {} }, + { url: 'https://ntfy.sh/tc', method: 'POST', body: 'wake', headers: { authorization: 'Bearer tk_secret' } }, + ]); +}); + +test('a failed wake-up is reported but never thrown', async () => { + const originalConsoleError = console.error; + console.error = () => {}; + try { + assert.equal(await wakeAnalyser({ + topic: 'tc', + fetchImpl: async () => { throw new Error('offline'); } + }), false); + assert.equal(await wakeAnalyser({ + topic: 'tc', + fetchImpl: async () => ({ ok: false, status: 429 }) + }), false); + } finally { + console.error = originalConsoleError; + } +}); + +test('nothing is sent when no topic is configured', async () => { + let called = false; + assert.equal(await wakeAnalyser({ + topic: '', + fetchImpl: async () => { called = true; } + }), false); + assert.equal(called, false); +}); diff --git a/test/auth.test.js b/test/auth.test.js index d73abfb..48b6929 100644 --- a/test/auth.test.js +++ b/test/auth.test.js @@ -3,6 +3,15 @@ const test = require('node:test'); test('analyser endpoints accept bearer auth and reject missing auth', async () => { process.env.UPLOAD_PASSWORD = 'test-secret'; + const Apps = require('../models/Apps'); + const original = { + nextApp: Apps.nextApp, + nextDueAt: Apps.nextDueAt, + recordAnalyserHeartbeat: Apps.recordAnalyserHeartbeat, + }; + Apps.nextApp = async () => null; + Apps.nextDueAt = async () => null; + Apps.recordAnalyserHeartbeat = async () => {}; const app = require('../server'); const server = await new Promise((resolve) => { @@ -11,16 +20,17 @@ test('analyser endpoints accept bearer auth and reject missing auth', async () = try { const base = `http://127.0.0.1:${server.address().port}`; - const headerResponse = await fetch(`${base}/ping`, { + const headerResponse = await fetch(`${base}/queue`, { headers: { authorization: 'Bearer test-secret' } }); - const queryResponse = await fetch(`${base}/ping?password=test-secret`); - const missingResponse = await fetch(`${base}/ping`); + const queryResponse = await fetch(`${base}/queue?password=test-secret`); + const missingResponse = await fetch(`${base}/queue`); assert.equal(headerResponse.status, 200); assert.equal(queryResponse.status, 400); assert.equal(missingResponse.status, 400); } finally { + Object.assign(Apps, original); await new Promise((resolve, reject) => { server.close((err) => err ? reject(err) : resolve()); });