Stop holding every analysed app in memory between cache rebuilds

models/Apps.js

@@ -462,10 +462,25 @@ const updateAnalysis = async (appId, analysis, analysisVersion, claimToken) => {
 // apps.details is the queue-time snapshot and is never updated, so the latest
 // known storefront row travels with each app for callers that display a title
 // or icon. See buildListingDetails in lib/appMetadata.js.
+//
+// The site views only read `success` and `trackers` from an analysis. The rest
+// (raw trackerscan output, Info.plists, per-match class lists) is most of each
+// stored analysis, so it is left in the database rather than parsed into the
+// web process for every app. Analyses that are not JSON objects pass through
+// unchanged so the builders treat them exactly as before.
 const getAllApps = async () => {
     const result = await pool.query(`
         SELECT
-            apps.*,
+            apps.appid,
+            apps.analysed,
+            apps.details,
+            CASE WHEN json_typeof(apps.analysis) = 'object'
+                THEN json_build_object(
+                    'success', apps.analysis->'success',
+                    'trackers', apps.analysis->'trackers'
+                )
+                ELSE apps.analysis
+            END AS analysis,
             cache.details AS current_storefront_details
         FROM apps
         LEFT JOIN app_store_cache cache

routes/index.js

@@ -248,24 +248,41 @@ async function getSiteData() {
 // kept in its own cache entry and only touched by the lookup pages and the
 // sitemap. The in-process copy avoids re-parsing the cache file per request.
 let reverseIndexMemo = null; // { meta, index }
-let allAppsMemo = null; // { meta, apps }
 
-async function getAllAppsForSignature(signature) {
-  if (allAppsMemo && signatureMatches(allAppsMemo.meta, signature))
-    return allAppsMemo.apps;
+// The rows are only needed while a cached view is being rebuilt, so they are
+// not kept afterwards: holding every app for the life of the process was most
+// of the server's resident memory. Concurrent rebuilds for the same signature
+// share one load instead of each fetching its own copy.
+let allAppsLoad = null; // { meta, promise }
 
+function getAllAppsForSignature(signature) {
+  if (allAppsLoad && signatureMatches(allAppsLoad.meta, signature))
+    return allAppsLoad.promise;
+
+  const load = { meta: signature, promise: loadAllApps() };
+  allAppsLoad = load;
+  const release = () => {
+    if (allAppsLoad === load) allAppsLoad = null;
+  };
+  load.promise.then(release, release);
+  return load.promise;
+}
+
+async function loadAllApps() {
   // Resolve the display metadata once here so that buildSiteData and
   // buildReverseIndex both see the refreshed title and icon under `details`,
-  // rather than each reaching into the storefront columns themselves.
-  const apps = (await Apps.getAllApps()).map((row) => ({
-    ...row,
+  // rather than each reaching into the storefront columns themselves. Only
+  // the fields the builders read are kept, so the full storefront records can
+  // be collected straight away.
+  return (await Apps.getAllApps()).map((row) => ({
+    appid: row.appid,
+    analysed: row.analysed,
+    analysis: row.analysis,
     details: buildListingDetails({
       queueSnapshot: row.details,
       storefront: { details: row.current_storefront_details }
     })
   }));
-  allAppsMemo = { meta: signature, apps };
-  return apps;
 }
 
 /**
@@ -306,7 +323,7 @@ function invalidateSiteCaches() {
   cache.invalidate('sitedata');
   cache.invalidate('reverseindex');
   reverseIndexMemo = null;
-  allAppsMemo = null;
+  allAppsLoad = null;
   signatureMemo = null;
 }