diff --git a/models/Apps.js b/models/Apps.js index ab11857..3daaafd 100644 --- a/models/Apps.js +++ b/models/Apps.js @@ -462,10 +462,25 @@ const updateAnalysis = async (appId, analysis, analysisVersion, claimToken) => { // apps.details is the queue-time snapshot and is never updated, so the latest // known storefront row travels with each app for callers that display a title // or icon. See buildListingDetails in lib/appMetadata.js. +// +// The site views only read `success` and `trackers` from an analysis. The rest +// (raw trackerscan output, Info.plists, per-match class lists) is most of each +// stored analysis, so it is left in the database rather than parsed into the +// web process for every app. Analyses that are not JSON objects pass through +// unchanged so the builders treat them exactly as before. const getAllApps = async () => { const result = await pool.query(` SELECT - apps.*, + apps.appid, + apps.analysed, + apps.details, + CASE WHEN json_typeof(apps.analysis) = 'object' + THEN json_build_object( + 'success', apps.analysis->'success', + 'trackers', apps.analysis->'trackers' + ) + ELSE apps.analysis + END AS analysis, cache.details AS current_storefront_details FROM apps LEFT JOIN app_store_cache cache diff --git a/routes/index.js b/routes/index.js index 8192329..2f3d6d6 100644 --- a/routes/index.js +++ b/routes/index.js @@ -248,24 +248,41 @@ async function getSiteData() { // kept in its own cache entry and only touched by the lookup pages and the // sitemap. The in-process copy avoids re-parsing the cache file per request. let reverseIndexMemo = null; // { meta, index } -let allAppsMemo = null; // { meta, apps } -async function getAllAppsForSignature(signature) { - if (allAppsMemo && signatureMatches(allAppsMemo.meta, signature)) - return allAppsMemo.apps; +// The rows are only needed while a cached view is being rebuilt, so they are +// not kept afterwards: holding every app for the life of the process was most +// of the server's resident memory. Concurrent rebuilds for the same signature +// share one load instead of each fetching its own copy. +let allAppsLoad = null; // { meta, promise } +function getAllAppsForSignature(signature) { + if (allAppsLoad && signatureMatches(allAppsLoad.meta, signature)) + return allAppsLoad.promise; + + const load = { meta: signature, promise: loadAllApps() }; + allAppsLoad = load; + const release = () => { + if (allAppsLoad === load) allAppsLoad = null; + }; + load.promise.then(release, release); + return load.promise; +} + +async function loadAllApps() { // Resolve the display metadata once here so that buildSiteData and // buildReverseIndex both see the refreshed title and icon under `details`, - // rather than each reaching into the storefront columns themselves. - const apps = (await Apps.getAllApps()).map((row) => ({ - ...row, + // rather than each reaching into the storefront columns themselves. Only + // the fields the builders read are kept, so the full storefront records can + // be collected straight away. + return (await Apps.getAllApps()).map((row) => ({ + appid: row.appid, + analysed: row.analysed, + analysis: row.analysis, details: buildListingDetails({ queueSnapshot: row.details, storefront: { details: row.current_storefront_details } }) })); - allAppsMemo = { meta: signature, apps }; - return apps; } /** @@ -306,7 +323,7 @@ function invalidateSiteCaches() { cache.invalidate('sitedata'); cache.invalidate('reverseindex'); reverseIndexMemo = null; - allAppsMemo = null; + allAppsLoad = null; signatureMemo = null; }