diff --git a/README.md b/README.md index ffda933..65c0b20 100644 --- a/README.md +++ b/README.md @@ -100,11 +100,18 @@ APP_STORE_CACHE_RETENTION_DAYS=90 PORT=3000 CLOUDFLARE_API_TOKEN=change-me CLOUDFLARE_ZONE_ID=change-me +NTFY_TOPIC= +NTFY_TOKEN= ``` `BODY_LIMIT` applies to authenticated analyser JSON and text uploads. `PUBLIC_FORM_BODY_LIMIT` is the smaller limit for the public analysis request form. `APP_STORE_CACHE_RETENTION_DAYS` controls how long cached App Store metadata is kept. +`NTFY_TOPIC` names the [ntfy](https://ntfy.sh) topic the website posts to when an +app is queued, which wakes an idle analyser straight away; use the same value in +`analyser/.env` (see [Queue And Operations](#queue-and-operations)). +`NTFY_TOKEN` is optional: an ntfy account access token, so publishing is not +subject to ntfy.sh's per-IP limits for anonymous use. Set `SITE_URL` in production to the public origin, for example `SITE_URL=https://ios.trackercontrol.org`. Canonical links, Open Graph URLs, @@ -262,14 +269,15 @@ For maintenance, use `sudo bash /opt/tracker-control-ios/scripts/manage-raspi-ip The website exposes analyser endpoints: -- `GET /queue` returns the next app to process in the response body and its one-use assignment token in the `X-Analysis-Claim-Token` response header. -- `GET /ping` marks the analyser online. +- `GET /queue` returns the next app to process in the response body and its one-use assignment token in the `X-Analysis-Claim-Token` response header. With the queue empty the body is empty, and `X-Analysis-Next-Due-Seconds` says when the next app falls due on its own (a stale analysis, a retryable failure, an abandoned claim); the header is absent if none ever will. Each call also records that the analyser is online. - `POST /uploadAnalysis` stores successful analysis results when sent with the assignment's `X-Analysis-Claim-Token` header. - `POST /reportAnalysisFailure` stores failed analysis results when sent with the assignment's `X-Analysis-Claim-Token` header. Analyzer requests authenticate with `Authorization: Bearer $UPLOAD_PASSWORD`. Authentication and assignment tokens serve different purposes: the bearer token authorises the analyser, while the assignment token prevents an expired worker from overwriting a newer result. +The analyser does not poll. With the queue empty it waits until the next app falls due, or until the website posts to the `NTFY_TOPIC` ntfy topic after queueing an app, whichever comes first, and asks again at least once a day (`MAX_IDLE_WAIT`). The website and its database therefore only see traffic when there is work or a visitor, so a host that stops idle services, such as Railway, can let them sleep. A failed or missing wake-up only delays an analysis until the analyser's next check. + ### Claim-token rollout The claim-token migration deliberately requeues every in-flight assignment that predates tokens. Use a coordinated deployment: @@ -284,7 +292,7 @@ The `/queue` response body remains a plain bundle ID, but older analyser scripts Health checks: - `GET /healthz` returns `200` when the website can reach PostgreSQL. -- `GET /healthz/analyser` returns `200` when the analyser has pinged in the last hour. +- `GET /healthz/analyser` returns `200` when the analyser has asked for work in the last 26 hours. Useful maintenance commands: diff --git a/analyser/.env.example b/analyser/.env.example index 1b8e220..15f07d7 100644 --- a/analyser/.env.example +++ b/analyser/.env.example @@ -1,4 +1,4 @@ -# Web server that exposes /queue, /ping, /uploadAnalysis, and /reportAnalysisFailure. +# Web server that exposes /queue, /uploadAnalysis, and /reportAnalysisFailure. SERVER=https://your-railway-app.example # Must match UPLOAD_PASSWORD on the web server. @@ -27,6 +27,17 @@ MAX_APP_SIZE_BYTES=3000000000 MAX_ATTEMPT_DOWNLOAD_BYTES=3000000000 CONSECUTIVE_FAILURE_LIMIT=5 CIRCUIT_BREAKER_SLEEP=3600 + +# Waiting for work. The website posts to this ntfy topic (https://ntfy.sh) when +# an app is queued; use the same NTFY_TOPIC (and NTFY_TOKEN, if any) as the web +# server. Pick a long random name, e.g. `openssl rand -hex 16`. Without a topic +# the analyser only checks when the next app falls due, or after MAX_IDLE_WAIT. +NTFY_TOPIC= +NTFY_TOKEN= +NTFY_SERVER=https://ntfy.sh +MIN_IDLE_WAIT=60 +MAX_IDLE_WAIT=86400 +QUEUE_RETRY_WAIT=300 DOWNLOAD_WATCHDOG_INTERVAL=5 NETWORK_INTERFACE= diff --git a/analyser/processQueue.sh b/analyser/processQueue.sh index 94db701..1771715 100755 --- a/analyser/processQueue.sh +++ b/analyser/processQueue.sh @@ -44,6 +44,12 @@ fi : "${IOS_STAGING_CLEANUP_TIMEOUT:=30}" : "${CONSECUTIVE_FAILURE_LIMIT:=5}" : "${CIRCUIT_BREAKER_SLEEP:=3600}" +: "${NTFY_TOPIC:=}" +: "${NTFY_SERVER:=https://ntfy.sh}" +: "${NTFY_TOKEN:=}" +: "${MIN_IDLE_WAIT:=60}" +: "${MAX_IDLE_WAIT:=86400}" +: "${QUEUE_RETRY_WAIT:=300}" : "${DOWNLOAD_WATCHDOG_INTERVAL:=5}" : "${NETWORK_INTERFACE:=}" : "${DAILY_BYTES_FILE:=./daily-download-bytes.txt}" @@ -113,6 +119,10 @@ fi curl_auth_config=$(mktemp) chmod 600 "$curl_auth_config" printf 'header = "Authorization: Bearer %s"\n' "$UPLOAD_PASSWORD" > "$curl_auth_config" +# The server may be asleep (Railway stops idle services) and answer its first +# request with a 502 or refuse the connection while it starts, so every call +# retries transient failures for up to a minute. +printf 'retry = 5\nretry-delay = 5\nretry-max-time = 60\nretry-connrefused\n' >> "$curl_auth_config" queue_headers=$(mktemp) chmod 600 "$queue_headers" trap 'rm -f "$curl_auth_config" "$queue_headers"' EXIT @@ -434,6 +444,7 @@ fetch_queue_app() : > "$queue_headers" appId=$(curl -sS -K "$curl_auth_config" -D "$queue_headers" "$queueUrl" --fail) || return 1 claimToken=$(awk 'tolower($1) == "x-analysis-claim-token:" { gsub(/\r/, "", $2); print $2; exit }' "$queue_headers") + nextDueSeconds=$(awk 'tolower($1) == "x-analysis-next-due-seconds:" { gsub(/\r/, "", $2); print $2; exit }' "$queue_headers") if [ -n "$appId" ] && [ -z "$claimToken" ]; then echo "Queue response for $appId did not include an analysis claim token." >&2 @@ -784,12 +795,58 @@ process_app() [ "$success" -eq 1 ] } +# The website is never polled: with the queue empty, wait until the next app +# falls due on its own (the server says when) or until the website announces +# a newly queued app on the ntfy topic, whichever comes first. Every /queue +# call also counts as a check-in, and MAX_IDLE_WAIT bounds the wait so the +# website keeps seeing the analyser at least daily. +wait_for_work() +{ + local wait="$MAX_IDLE_WAIT" started elapsed line wake_pid woken=0 + + if [[ "$nextDueSeconds" =~ ^[0-9]+$ ]] && [ "$nextDueSeconds" -lt "$wait" ]; then + wait="$nextDueSeconds" + fi + if [ "$wait" -lt "$MIN_IDLE_WAIT" ]; then + wait="$MIN_IDLE_WAIT" + fi + started=$(date +%s) + + if [ -n "$NTFY_TOPIC" ]; then + echo "Queue empty. Waiting up to ${wait}s for a wake-up." + # since= replays a wake-up sent between the /queue check and this + # subscription. ntfy sends keepalives as empty lines. + exec {wake_fd}< <(exec curl -sN --max-time "$wait" \ + ${NTFY_TOKEN:+-H "Authorization: Bearer $NTFY_TOKEN"} \ + "${NTFY_SERVER%/}/$NTFY_TOPIC/raw?since=$queue_checked_at") + wake_pid=$! + while IFS= read -r -u "$wake_fd" line; do + if [ -n "$line" ]; then + woken=1 + break + fi + done + kill "$wake_pid" 2>/dev/null + exec {wake_fd}<&- + else + echo "Queue empty. Checking again in ${wait}s (set NTFY_TOPIC to be woken for new requests)." + fi + + if [ "$woken" -eq 1 ]; then + echo "Woken up: an app was queued." + return + fi + + # Also covers ntfy being unreachable, so the website is not hammered. + elapsed=$(( $(date +%s) - started )) + if [ "$elapsed" -lt "$wait" ]; then + sleep $(( wait - elapsed )) + fi +} + if [ -n "$ONLY_APP_ID" ]; then reset_daily_counter - echo "Reporting online status apps to install" - curl -s -K "$curl_auth_config" "$SERVER/ping" - if ! check_daily_limit; then exit 1 fi @@ -814,9 +871,6 @@ fi while true; do reset_daily_counter - echo "Reporting online status apps to install" - curl -s -K "$curl_auth_config" "$SERVER/ping" - if ! check_daily_limit; then sleep "$CIRCUIT_BREAKER_SLEEP" continue @@ -830,18 +884,24 @@ while true; do fi echo "Fetching apps to install" + queue_checked_at=$(date +%s) + nextDueSeconds="" if ! fetch_queue_app "$SERVER/queue"; then - echo "Failed to fetch a queue assignment." - appId="" - claimToken="" + echo "Failed to fetch a queue assignment. Retrying in ${QUEUE_RETRY_WAIT}s." + [ "$RUN_ONCE" = "1" ] && exit 1 + sleep "$QUEUE_RETRY_WAIT" + continue fi if [ "$appId" == "" ] ; then - echo "No app to process.." - else - process_app "$appId" "$claimToken" + echo "No app to process.." + [ "$RUN_ONCE" = "1" ] && exit 0 + wait_for_work + continue fi + process_app "$appId" "$claimToken" + if [ "$RUN_ONCE" = "1" ]; then exit 0 fi diff --git a/lib/analyserWake.js b/lib/analyserWake.js new file mode 100644 index 0000000..a448755 --- /dev/null +++ b/lib/analyserWake.js @@ -0,0 +1,61 @@ +// Tells a waiting analyser that an app was queued, through an ntfy topic +// (https://ntfy.sh) that analyser/processQueue.sh subscribes to while idle. +// The analyser never polls the website, so without this a newly requested app +// waits until the analyser's next scheduled check. The message carries no +// data; it only ends the analyser's wait. Inert unless NTFY_TOPIC is set. +// ntfy.sh rate-limits anonymous publishing per IP address, which a shared +// hosting egress address can exhaust; NTFY_TOKEN (an ntfy account access +// token) publishes under the account's own limits instead. + +const DEFAULT_SERVER = 'https://ntfy.sh'; +const TIMEOUT_MS = 3000; +// ntfy topic names: letters, digits, dashes and underscores, at most 64. +const TOPIC_PATTERN = /^[A-Za-z0-9_-]{1,64}$/; + +function wakeUrl({ + topic = process.env.NTFY_TOPIC, + server = process.env.NTFY_SERVER +} = {}) { + topic = (topic || '').trim(); + if (!topic) return null; + if (!TOPIC_PATTERN.test(topic)) { + console.error('NTFY_TOPIC must be 1-64 letters, digits, dashes or underscores; analyser wake-ups are disabled.'); + return null; + } + const base = ((server || '').trim() || DEFAULT_SERVER).replace(/\/+$/, ''); + return `${base}/${topic}`; +} + +/** + * Best effort: resolves to true if the notification was accepted, false + * otherwise. Never throws, so callers need not await it. A missed wake-up only + * delays the analysis until the analyser's next scheduled check. + */ +async function wakeAnalyser({ + fetchImpl = fetch, + token = process.env.NTFY_TOKEN, + ...options +} = {}) { + const url = wakeUrl(options); + if (!url) return false; + + token = (token || '').trim(); + try { + const response = await fetchImpl(url, { + method: 'POST', + body: 'wake', + headers: token ? { authorization: `Bearer ${token}` } : {}, + signal: AbortSignal.timeout(TIMEOUT_MS) + }); + if (!response.ok) { + console.error('Analyser wake-up rejected:', response.status); + return false; + } + return true; + } catch (err) { + console.error('Analyser wake-up failed:', err.message); + return false; + } +} + +module.exports = { wakeAnalyser, wakeUrl }; diff --git a/migrations/015_analyser_heartbeat.sql b/migrations/015_analyser_heartbeat.sql new file mode 100644 index 0000000..d1fae9b --- /dev/null +++ b/migrations/015_analyser_heartbeat.sql @@ -0,0 +1,7 @@ +-- When the analyser last checked in. Kept in the database so a restarted or +-- woken web process still knows the analyser is online. + +CREATE TABLE IF NOT EXISTS analyser_heartbeat ( + id boolean PRIMARY KEY DEFAULT true CHECK (id), + last_seen timestamptz NOT NULL +); diff --git a/models/Apps.js b/models/Apps.js index 3daaafd..af14c04 100644 --- a/models/Apps.js +++ b/models/Apps.js @@ -518,6 +518,49 @@ const getSiteDataSignature = async () => { }; } +// When the analyser next has work if nothing new is queued: the earliest moment +// an app becomes claimable by nextApp through the passage of time alone (a +// stale analysis, a retryable failure, an abandoned claim). Mirrors the +// candidate conditions in nextApp. Returns a Date, or null if no app will ever +// become due without being queued. +const nextDueAt = async () => { + const result = await pool.query(` + SELECT MIN(due) AS due + FROM ( + SELECT CASE + WHEN status = 'processing' + THEN processing_started + ($3::int * INTERVAL '1 minute') + WHEN (status = 'analysed' + OR (status = 'failed' AND failure_retryable)) + AND analysisversion IS NOT DISTINCT FROM $1 + THEN analysed + ($2::int * INTERVAL '1 day') + END AS due + FROM apps + WHERE ${appIdSqlPredicate(4, 5)} + ) candidates + `, [ + currentAnalysisVersion, + staleAnalysisDays, + processingTimeoutMinutes, + APP_ID_PATTERN_SOURCE, + MAX_APP_ID_LENGTH + ]); + return result.rows[0].due; +} + +const recordAnalyserHeartbeat = async () => { + await pool.query(` + INSERT INTO analyser_heartbeat (id, last_seen) VALUES (true, NOW()) + ON CONFLICT (id) DO UPDATE SET last_seen = EXCLUDED.last_seen + `); +} + +// Milliseconds since the epoch, or 0 if the analyser has never checked in. +const lastAnalyserHeartbeat = async () => { + const result = await pool.query('SELECT last_seen FROM analyser_heartbeat'); + return result.rows.length > 0 ? result.rows[0].last_seen.getTime() : 0; +} + const countAnalysed = async () => { const result = await pool.query("SELECT COUNT(*) FROM apps WHERE status = 'analysed'"); return parseInt(result.rows[0].count, 10); @@ -532,6 +575,9 @@ module.exports = { countAnalysed, addApp, nextApp, + nextDueAt, + recordAnalyserHeartbeat, + lastAnalyserHeartbeat, updateAnalysis, getAllApps, getSiteDataSignature, diff --git a/raspberry-pi-analyser.md b/raspberry-pi-analyser.md index d2a9a09..13c476d 100644 --- a/raspberry-pi-analyser.md +++ b/raspberry-pi-analyser.md @@ -88,6 +88,8 @@ CIRCUIT_BREAKER_SLEEP=3600 DOWNLOAD_WATCHDOG_INTERVAL=5 ``` +The analyser does not poll the website. When the queue is empty it waits until the next app falls due (the website says when), until the website announces a newly queued app on an [ntfy](https://ntfy.sh) topic, or for `MAX_IDLE_WAIT` seconds (a day by default), whichever comes first. Set the same long random topic name as `NTFY_TOPIC` here and on the web server, for example from `openssl rand -hex 16`; without it, a newly requested app waits for the next of the other two. Requests to the website retry for up to a minute, so the analyser rides out the website waking up from sleep. + Set `NETWORK_INTERFACE` only if auto-detection picks the wrong interface. On a Pi this is often `eth0` or `wlan0`. ## Use RAM-Backed IPA Storage diff --git a/routes/index.js b/routes/index.js index 2f3d6d6..87876a3 100644 --- a/routes/index.js +++ b/routes/index.js @@ -13,6 +13,7 @@ const { buildReportMetadata, buildListingDetails } = require('../lib/appMetadata const { siteBaseUrl } = require('../lib/siteUrl'); const cloudflare = require('../lib/cloudflarePurge'); const { canonicalUrl } = require('../lib/i18n'); +const { wakeAnalyser } = require('../lib/analyserWake'); // Taken from https://reports.exodus-privacy.eu.org/api/trackers const exodusTrackers = JSON.parse(fs.readFileSync('./exodusTrackers.json', 'utf-8')) @@ -30,7 +31,28 @@ const APPS_PER_PAGE = 50; const MAX_SITEMAP_URLS = 50000; const MAX_SITEMAP_BYTES = 50 * 1024 * 1024; +// The analyser only contacts the website when it has work or a scheduled +// check, at least daily plus the length of one analysis, so it counts as +// online for 26 hours. +const ANALYSER_ONLINE_MS = 26 * 60 * 60 * 1000; let lastPing = 0; // unix timestamp +let lastPingLoad = null; + +// lastPing starts at zero in a fresh process, and on a platform that stops +// idle services every wake is one, so the last check-in is read back from the +// database once. A failed read is retried a minute later rather than on every +// request. +function loadLastPing() { + if (!lastPingLoad) { + lastPingLoad = Apps.lastAnalyserHeartbeat().then((stored) => { + lastPing = Math.max(lastPing, stored); + }, (err) => { + console.error('Could not read analyser heartbeat:', err.message); + setTimeout(() => { lastPingLoad = null; }, 60 * 1000).unref(); + }); + } + return lastPingLoad; +} function requireValidAppId(req, res, next) { if (!isValidAppId(req.params.appId)) @@ -64,10 +86,12 @@ function redirectPublic(res, status, path) { return res.redirect(status, local(path)); } -// ping from analyser in past hour? +// has the analyser checked in recently? router.use(function (req, res, next) { - res.locals.analyserOnline = lastPing > Date.now() - 1000*60*60; - next(); + loadLastPing().then(() => { + res.locals.analyserOnline = lastPing > Date.now() - ANALYSER_ONLINE_MS; + next(); + }); }); function thirdPartyTrackerNames(analysis) { @@ -440,7 +464,7 @@ router.get('/healthz', asyncHandler(async (req, res) => { })); router.get('/healthz/analyser', (req, res) => { - const online = lastPing > Date.now() - 1000*60*60; + const online = lastPing > Date.now() - ANALYSER_ONLINE_MS; res.status(online ? 200 : 503).json({ ok: online }); }); @@ -674,6 +698,7 @@ router.post('/analysis/:appId', }); } + wakeAnalyser(); return redirectPublic(res, 303, `/analysis/${details.appId}`); })); @@ -791,11 +816,29 @@ router.get('/queue', asyncHandler(async (req, res) => { if (requestedAppId && !isValidAppId(requestedAppId)) return res.status(400).send('Please provide a valid App Store bundle ID.'); + lastPing = Date.now(); + Apps.recordAnalyserHeartbeat().catch((err) => { + console.error('Could not record analyser heartbeat:', err.message); + }); + let app = await Apps.nextApp(requestedAppId); console.log(app ? app.appid : null); - if (!app) + if (!app) { + // The analyser waits for a wake-up (see lib/analyserWake.js) or until the + // next app falls due on its own, whichever comes first. Without the + // header it falls back to its own maximum wait. + try { + const due = await Apps.nextDueAt(); + if (due) { + const seconds = Math.max(0, Math.ceil((due.getTime() - Date.now()) / 1000)); + res.set('X-Analysis-Next-Due-Seconds', String(seconds)); + } + } catch (err) { + console.error('Could not compute the next due analysis:', err.message); + } return res.send(); + } // Keep the body as the bundle ID for deployed analysers while newer // clients carry the per-assignment token in a separate header. @@ -803,13 +846,6 @@ router.get('/queue', asyncHandler(async (req, res) => { res.send(app.appid); })); -// enable analyser to report online status -router.get('/ping', (req, res) => { - lastPing = Date.now(); - - res.send("online"); -}); - // upload analysis results router.post('/uploadAnalysis', asyncHandler(async (req, res) => { if (!req.query.appId || !req.query.analysisVersion) @@ -1004,7 +1040,6 @@ router.get('/robots.txt', (req, res) => { 'Disallow: /search', 'Disallow: /request/', 'Disallow: /queue', - 'Disallow: /ping', 'Disallow: /healthz', '', `Sitemap: ${siteBaseUrl(req)}/sitemap.xml`, diff --git a/scripts/queue-refetch.js b/scripts/queue-refetch.js index 73f882b..8a5d980 100644 --- a/scripts/queue-refetch.js +++ b/scripts/queue-refetch.js @@ -4,6 +4,7 @@ const path = require('path'); const dotenv = require('dotenv'); const { Client } = require('pg'); const { buildAnalysisProvenanceSourceSql } = require('../models/Apps'); +const { wakeAnalyser } = require('../lib/analyserWake'); dotenv.config({ path: path.join(__dirname, '..', '.env') }); dotenv.config({ path: path.join(__dirname, '..', 'analyser', '.env') }); @@ -155,6 +156,7 @@ async function main() { console.log(`Queued ${rows.length} apps for refetch.`); printRows(rows); await client.end(); + if (rows.length > 0) await wakeAnalyser(); } main().catch((error) => { diff --git a/server.js b/server.js index c77cd5c..cc0e669 100644 --- a/server.js +++ b/server.js @@ -38,7 +38,6 @@ app.disable('x-powered-by') const os = require('os'); const analyserPaths = new Set([ '/queue', - '/ping', '/uploadanalysis', '/reportanalysisfailure' ]); diff --git a/test/analyserQueue.test.js b/test/analyserQueue.test.js new file mode 100644 index 0000000..ab48334 --- /dev/null +++ b/test/analyserQueue.test.js @@ -0,0 +1,124 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); + +// Runs in its own process, so the server's one-off heartbeat read happens +// against the stubs below. +process.env.UPLOAD_PASSWORD = 'test-secret'; +process.env.NTFY_TOPIC = 'tc-test-topic'; +const Apps = require('../models/Apps'); +const store = require('../lib/appStore'); + +const state = { reads: 0, writes: 0, nextApp: null, nextDue: null }; +Apps.lastAnalyserHeartbeat = async () => { + state.reads++; + return Date.now() - 20 * 60 * 60 * 1000; +}; +Apps.recordAnalyserHeartbeat = async () => { state.writes++; }; +Apps.nextApp = async () => state.nextApp; +Apps.nextDueAt = async () => state.nextDue; + +const wakeUps = []; +const realFetch = globalThis.fetch; +globalThis.fetch = async (url, options) => { + if (String(url).startsWith('https://ntfy.sh/')) { + wakeUps.push({ url: String(url), body: options.body }); + return { ok: true, status: 200 }; + } + return realFetch(url, options); +}; + +const app = require('../server'); + +async function withServer(run) { + const server = await new Promise((resolve) => { + const instance = app.listen(0, () => resolve(instance)); + }); + try { + await run(`http://127.0.0.1:${server.address().port}`); + } finally { + await new Promise((resolve, reject) => { + server.close((err) => err ? reject(err) : resolve()); + }); + } +} + +const auth = { authorization: 'Bearer test-secret' }; + +test('a stored check-in from the last day keeps the analyser online across restarts', async () => { + await withServer(async (base) => { + assert.equal((await fetch(`${base}/healthz/analyser`)).status, 200); + assert.equal((await fetch(`${base}/healthz/analyser`)).status, 200); + }); + assert.equal(state.reads, 1); +}); + +test('an empty queue tells the analyser when the next app falls due', async () => { + const originalConsoleLog = console.log; + console.log = () => {}; + try { + await withServer(async (base) => { + state.nextApp = null; + state.nextDue = new Date(Date.now() + 3600 * 1000); + const due = await fetch(`${base}/queue`, { headers: auth }); + assert.equal(due.status, 200); + assert.equal(await due.text(), ''); + const seconds = Number(due.headers.get('x-analysis-next-due-seconds')); + assert.ok(seconds > 3590 && seconds <= 3600, String(seconds)); + + state.nextDue = new Date(Date.now() - 1000); + const overdue = await fetch(`${base}/queue`, { headers: auth }); + assert.equal(overdue.headers.get('x-analysis-next-due-seconds'), '0'); + + state.nextDue = null; + const never = await fetch(`${base}/queue`, { headers: auth }); + assert.equal(never.headers.get('x-analysis-next-due-seconds'), null); + + state.nextApp = { appid: 'com.example.app', analysis_claim_token: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' }; + state.nextDue = new Date(Date.now() + 3600 * 1000); + const claimed = await fetch(`${base}/queue`, { headers: auth }); + assert.equal(await claimed.text(), 'com.example.app'); + assert.equal(claimed.headers.get('x-analysis-next-due-seconds'), null); + }); + } finally { + console.log = originalConsoleLog; + } + assert.equal(state.writes, 4); +}); + +test('the separate ping endpoint is gone', async () => { + await withServer(async (base) => { + const response = await fetch(`${base}/ping`, { headers: { ...auth, accept: 'text/plain' } }); + assert.equal(response.status, 404); + }); +}); + +test('queueing an app wakes the analyser', async () => { + const original = { + findApp: Apps.findApp, + findCachedAppStoreResult: Apps.findCachedAppStoreResult, + addAppAndStorefront: Apps.addAppAndStorefront, + storeApp: store.app, + }; + Apps.findApp = async () => null; + Apps.findCachedAppStoreResult = async () => null; + Apps.addAppAndStorefront = async () => ({ rowCount: 1 }); + store.app = async () => ({ appId: 'com.example.Wake', title: 'Wake', free: true }); + wakeUps.length = 0; + + try { + await withServer(async (base) => { + const response = await fetch(`${base}/analysis/com.example.wake`, { + method: 'POST', + headers: { 'content-type': 'application/x-www-form-urlencoded' }, + redirect: 'manual' + }); + assert.equal(response.status, 303); + }); + } finally { + Apps.findApp = original.findApp; + Apps.findCachedAppStoreResult = original.findCachedAppStoreResult; + Apps.addAppAndStorefront = original.addAppAndStorefront; + store.app = original.storeApp; + } + assert.deepEqual(wakeUps, [{ url: 'https://ntfy.sh/tc-test-topic', body: 'wake' }]); +}); diff --git a/test/analyserWake.test.js b/test/analyserWake.test.js new file mode 100644 index 0000000..62c8205 --- /dev/null +++ b/test/analyserWake.test.js @@ -0,0 +1,61 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { wakeAnalyser, wakeUrl } = require('../lib/analyserWake'); + +test('wake-ups are disabled without a valid topic', () => { + const originalConsoleError = console.error; + console.error = () => {}; + try { + assert.equal(wakeUrl({ topic: '' }), null); + assert.equal(wakeUrl({ topic: ' ' }), null); + assert.equal(wakeUrl({ topic: 'a/b' }), null); + assert.equal(wakeUrl({ topic: 'x'.repeat(65) }), null); + } finally { + console.error = originalConsoleError; + } +}); + +test('the topic URL defaults to ntfy.sh and accepts a self-hosted server', () => { + assert.equal(wakeUrl({ topic: 'tc-ios_42' }), 'https://ntfy.sh/tc-ios_42'); + assert.equal(wakeUrl({ topic: 'tc', server: 'https://ntfy.example.test//' }), 'https://ntfy.example.test/tc'); +}); + +test('a wake-up posts a bare message to the topic', async () => { + const calls = []; + const fetchImpl = async (url, options) => { + calls.push({ url, method: options.method, body: options.body, headers: options.headers }); + return { ok: true, status: 200 }; + }; + assert.equal(await wakeAnalyser({ topic: 'tc', token: '', fetchImpl }), true); + assert.equal(await wakeAnalyser({ topic: 'tc', token: 'tk_secret', fetchImpl }), true); + assert.deepEqual(calls, [ + { url: 'https://ntfy.sh/tc', method: 'POST', body: 'wake', headers: {} }, + { url: 'https://ntfy.sh/tc', method: 'POST', body: 'wake', headers: { authorization: 'Bearer tk_secret' } }, + ]); +}); + +test('a failed wake-up is reported but never thrown', async () => { + const originalConsoleError = console.error; + console.error = () => {}; + try { + assert.equal(await wakeAnalyser({ + topic: 'tc', + fetchImpl: async () => { throw new Error('offline'); } + }), false); + assert.equal(await wakeAnalyser({ + topic: 'tc', + fetchImpl: async () => ({ ok: false, status: 429 }) + }), false); + } finally { + console.error = originalConsoleError; + } +}); + +test('nothing is sent when no topic is configured', async () => { + let called = false; + assert.equal(await wakeAnalyser({ + topic: '', + fetchImpl: async () => { called = true; } + }), false); + assert.equal(called, false); +}); diff --git a/test/auth.test.js b/test/auth.test.js index d73abfb..48b6929 100644 --- a/test/auth.test.js +++ b/test/auth.test.js @@ -3,6 +3,15 @@ const test = require('node:test'); test('analyser endpoints accept bearer auth and reject missing auth', async () => { process.env.UPLOAD_PASSWORD = 'test-secret'; + const Apps = require('../models/Apps'); + const original = { + nextApp: Apps.nextApp, + nextDueAt: Apps.nextDueAt, + recordAnalyserHeartbeat: Apps.recordAnalyserHeartbeat, + }; + Apps.nextApp = async () => null; + Apps.nextDueAt = async () => null; + Apps.recordAnalyserHeartbeat = async () => {}; const app = require('../server'); const server = await new Promise((resolve) => { @@ -11,16 +20,17 @@ test('analyser endpoints accept bearer auth and reject missing auth', async () = try { const base = `http://127.0.0.1:${server.address().port}`; - const headerResponse = await fetch(`${base}/ping`, { + const headerResponse = await fetch(`${base}/queue`, { headers: { authorization: 'Bearer test-secret' } }); - const queryResponse = await fetch(`${base}/ping?password=test-secret`); - const missingResponse = await fetch(`${base}/ping`); + const queryResponse = await fetch(`${base}/queue?password=test-secret`); + const missingResponse = await fetch(`${base}/queue`); assert.equal(headerResponse.status, 200); assert.equal(queryResponse.status, 400); assert.equal(missingResponse.status, 400); } finally { + Object.assign(Apps, original); await new Promise((resolve, reject) => { server.close((err) => err ? reject(err) : resolve()); });