Pod quota

config/customise-me.json

@@ -4,6 +4,9 @@
       "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^7.0.0/components/context.jsonld",
       "https://linkedsoftwaredependencies.org/bundles/npm/@solid/pivot/^1.0.0/components/context.jsonld"
     ],
+    "import": [
+      "pivot:config/storage/backend/quota-fast-file.json"
+    ],
     "@graph": [
       {
         "comment": "The settings of your email server.",
@@ -31,18 +34,6 @@
           "templateFolder": "templates/pod"
         }
       },
-      {
-        "comment": "Sets the maximum size of a single pod to 70MB.",
-        "@type": "Override",
-        "overrideInstance": {
-            "@id": "urn:solid-server:default:QuotaStrategy"
-        },
-        "overrideParameters": {            
-            "@type": "PodQuotaStrategy",
-            "limit_amount": 70000000,
-            "limit_unit": "bytes"
-        }
-      },
       {
         "comment": "Serve Databrowser as default representation",
         "@id": "urn:solid-server:default:DefaultUiConverter",

config/storage/backend/quota-fast-file.json

@@ -0,0 +1,50 @@
+{
+  "comment": "Fast per-pod quota: DuSizeReporter (du + TTL cache) and FastQuotaStrategy (no per-chunk pod walk).",
+  "@context": [
+    "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^7.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/@solid/pivot/^1.0.0/components/context.jsonld"
+  ],
+  "@graph": [
+    {
+      "comment": "SizeReporter backed by du with a per-path TTL cache, measuring apparent bytes (portable, user-manageable). Falls back to the Node walk when du is unavailable.",
+      "@type": "Override",
+      "overrideInstance": {
+        "@id": "urn:solid-server:default:SizeReporter"
+      },
+      "overrideParameters": {
+        "@type": "DuSizeReporter",
+        "fileIdentifierMapper": {
+          "@id": "urn:solid-server:default:FileIdentifierMapper"
+        },
+        "rootFilePath": {
+          "@id": "urn:solid-server:default:variable:rootFilePath"
+        },
+        "ignoreFolders": [
+          "^/\\.internal$"
+        ],
+        "ttl": 5000
+      }
+    },
+    {
+      "comment": "Pod quota strategy that computes the available space once per write instead of per stream chunk.",
+      "@type": "Override",
+      "overrideInstance": {
+        "@id": "urn:solid-server:default:QuotaStrategy"
+      },
+      "overrideParameters": {
+        "@type": "FastQuotaStrategy",
+        "limit_amount": 70000000,
+        "limit_unit": "bytes",
+        "reporter": {
+          "@id": "urn:solid-server:default:SizeReporter"
+        },
+        "identifierStrategy": {
+          "@id": "urn:solid-server:default:IdentifierStrategy"
+        },
+        "accessor": {
+          "@id": "urn:solid-server:default:AtomicFileDataAccessor"
+        }
+      }
+    }
+  ]
+}

scripts/benchmark-quota.js

@@ -0,0 +1,145 @@
+/**
+ * Benchmark: CSS pod-quota chain — old (FileSizeReporter + QuotaStrategy)
+ * vs pivot (DuSizeReporter + FastQuotaStrategy).
+ *
+ * Measures exactly what was optimized:
+ *   1. Full pod walk cost (old recursive Node walk vs du walk)
+ *   2. The per-write quota guard (old: full walk PER CHUNK; new: walk once)
+ *   3. The TTL cache effect on repeated size queries
+ *
+ * Run: node scripts/benchmark-quota.js [fileCount] [fileBytes]
+ *   e.g. node scripts/benchmark-quota.js 10000 1024
+ *
+ * NOTE: on bare Windows there is no `du`, so the walk times will be similar
+ * between the two (the guard + cache wins still show). On Linux/WSL the du
+ * walk is 10-100x faster.
+ */
+const { performance } = require('node:perf_hooks');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { PassThrough } = require('node:stream');
+const {
+  FileSizeReporter,
+  QuotaStrategy,
+} = require('@solid/community-server');
+const { DuSizeReporter } = require('../dist/storage/size-reporter/DuSizeReporter.js');
+const { FastQuotaStrategy } = require('../dist/storage/quota/FastQuotaStrategy.js');
+
+const FILE_COUNT = Number(process.argv[2]) || 5000;
+const FILE_BYTES = Number(process.argv[3]) || 1024;
+const WRITE_BYTES = 4 * 1024 * 1024; // simulated write body
+const CHUNK_BYTES = 64 * 1024;
+
+function makeMapper(root) {
+  return {
+    async mapUrlToFilePath(identifier) {
+      const url = new URL(identifier.path);
+      return { identifier, filePath: path.join(root, url.pathname), contentType: undefined, isMetadata: false };
+    },
+    async mapFilePathToUrl() { throw new Error('n/a'); },
+  };
+}
+
+// Old strategy: base QuotaStrategy whose pod size = reporter.getSize(podRoot)
+class OldBenchStrategy extends QuotaStrategy {
+  constructor(reporter, limit, podRoot) {
+    super(reporter, limit);
+    this.podRoot = podRoot;
+  }
+  async getTotalSpaceUsed() {
+    return this.reporter.getSize(this.podRoot);
+  }
+}
+
+// New strategy: FastQuotaStrategy with the same pod-size hook
+class NewBenchStrategy extends FastQuotaStrategy {
+  constructor(reporter, limit, podRoot) {
+    super(limit, reporter, {}, {});
+    this.podRoot = podRoot;
+  }
+  async getTotalSpaceUsed() {
+    return this.reporter.getSize(this.podRoot);
+  }
+}
+
+function seedPod(root, count, bytes) {
+  const inbox = path.join(root, 'inbox');
+  fs.mkdirSync(inbox, { recursive: true });
+  const buf = Buffer.alloc(bytes, 7);
+  for (let i = 0; i < count; i++) {
+    fs.writeFileSync(path.join(inbox, `file-${i}.bin`), buf);
+  }
+}
+
+function writeThroughGuard(guard, totalBytes, chunkBytes) {
+  return new Promise((resolve, reject) => {
+    const chunks = [];
+    let remaining = totalBytes;
+    guard.on('data', () => {});
+    guard.on('end', () => resolve(chunks));
+    guard.on('error', reject);
+    while (remaining > 0) {
+      const size = Math.min(chunkBytes, remaining);
+      guard.write(Buffer.alloc(size, 1));
+      remaining -= size;
+    }
+    guard.end();
+  });
+}
+
+async function timed(fn) {
+  const start = performance.now();
+  await fn();
+  return performance.now() - start;
+}
+
+async function main() {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'quota-bench-'));
+  const podRoot = { path: 'http://example.com/' };
+  const limit = { unit: 'bytes', amount: 10 * 1024 * 1024 * 1024 };
+
+  console.log(`Seeding ${FILE_COUNT} files × ${FILE_BYTES} B in ${root} …`);
+  const seedStart = performance.now();
+  seedPod(root, FILE_COUNT, FILE_BYTES);
+  console.log(`  seeded in ${(performance.now() - seedStart).toFixed(0)} ms\n`);
+
+  const mapper = makeMapper(root);
+  const reporterOld = new FileSizeReporter(mapper, root);
+  const reporterNew = new DuSizeReporter(mapper, root, [ '^/\\.internal$' ]);
+
+  // --- 1. Full pod walk (single getSize) ---
+  const walkOld = await timed(() => reporterOld.getSize(podRoot));
+  const walkNew = await timed(() => reporterNew.getSize(podRoot));
+  const walkCached = await timed(() => reporterNew.getSize(podRoot));
+
+  console.log('1. FULL POD WALK (getSize of pod root)');
+  console.log(`   old (Node recursive walk): ${walkOld.toFixed(1)} ms`);
+  console.log(`   new (du, first call):       ${walkNew.toFixed(1)} ms`);
+  console.log(`   new (du, cached, TTL hit):  ${walkCached.toFixed(3)} ms`);
+  if (walkNew > 0) {
+    console.log(`   walk speedup (first call):  ${(walkOld / walkNew).toFixed(1)}×`);
+  }
+
+  // --- 2. Per-write guard (write body through the quota guard) ---
+  const strategyOld = new OldBenchStrategy(reporterOld, limit, podRoot);
+  const strategyNew = new NewBenchStrategy(reporterNew, limit, podRoot);
+
+  const guardOld = await strategyOld.createQuotaGuard({ path: 'http://example.com/inbox/file-new.bin' });
+  const guardNew = await strategyNew.createQuotaGuard({ path: 'http://example.com/inbox/file-new.bin' });
+
+  const guardOldMs = await timed(() => writeThroughGuard(guardOld, WRITE_BYTES, CHUNK_BYTES));
+  const guardNewMs = await timed(() => writeThroughGuard(guardNew, WRITE_BYTES, CHUNK_BYTES));
+
+  console.log(`\n2. PER-WRITE QUOTA GUARD (${(WRITE_BYTES / 1024 / 1024).toFixed(0)} MB body, ${CHUNK_BYTES / 1024} KB chunks)`);
+  console.log(`   old (walk per chunk, ${Math.ceil(WRITE_BYTES / CHUNK_BYTES)} chunks): ${guardOldMs.toFixed(1)} ms`);
+  console.log(`   new (walk once + cache):           ${guardNewMs.toFixed(1)} ms`);
+  if (guardNewMs > 0) {
+    console.log(`   guard speedup:                     ${(guardOldMs / guardNewMs).toFixed(1)}×`);
+  }
+
+  fs.rmSync(root, { recursive: true, force: true });
+  console.log('\nDone.');
+}
+
+main().catch((err) => { console.error(err); process.exit(1); });

scripts/verify-size-equivalence.js

@@ -0,0 +1,120 @@
+/**
+ * Equivalence proof: DuSizeReporter vs CSS FileSizeReporter.
+ *
+ * Generates random pod trees and asserts both reporters return the EXACT same
+ * apparent-byte total (same ignoreFolders), for the du path and the Node-walk
+ * fallback. Exits non-zero on any mismatch.
+ *
+ * Run with GNU du on PATH to exercise the real du path, e.g. on Windows:
+ *   $env:PATH = "C:\Program Files\Git\usr\bin;$env:PATH"
+ *   node scripts/verify-size-equivalence.js [iterations] [maxFiles]
+ */
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { FileSizeReporter } = require('@solid/community-server');
+const { DuSizeReporter } = require('../dist/storage/size-reporter/DuSizeReporter.js');
+
+const ITERATIONS = Number(process.argv[2]) || 10;
+const MAX_FILES = Number(process.argv[3]) || 60;
+const IGNORE = [ '^/\\.internal$' ];
+
+class ForcedDu extends DuSizeReporter {
+  async detectDu() { return 'gnu'; }
+}
+class ForcedNode extends DuSizeReporter {
+  async detectDu() { return 'none'; }
+}
+
+function makeMapper(root) {
+  return {
+    async mapUrlToFilePath(identifier) {
+      const url = new URL(identifier.path);
+      return { identifier, filePath: path.join(root, url.pathname), contentType: undefined, isMetadata: false };
+    },
+    async mapFilePathToUrl() { throw new Error('n/a'); },
+  };
+}
+
+// Random names for nested content. .internal is handled separately below:
+// CSS only ever places .internal at the pod ROOT (temp files), where the
+// anchored regex ^/\.internal$ and du's basename --exclude=.internal agree.
+const SEGMENTS = [ 'a', 'b', 'c', 'd', 'e', 'inbox', 'public', 'private', 'settings' ];
+
+function randomTree(root, maxFiles) {
+  const files = [];
+  const count = 1 + Math.floor(Math.random() * maxFiles);
+  for (let i = 0; i < count; i++) {
+    // 1-4 nested segments, each a subdirectory (mkdirp on write).
+    const depth = 1 + Math.floor(Math.random() * 3);
+    const segs = [];
+    for (let d = 0; d < depth; d++) {
+      segs.push(SEGMENTS[Math.floor(Math.random() * SEGMENTS.length)]);
+    }
+    const dir = path.join(root, ...segs);
+    const file = path.join(dir, `f${i}.bin`);
+    const size = Math.floor(Math.random() * 50_000);
+    files.push({ dir, file, size });
+  }
+  return files;
+}
+
+function writeTree(root, files) {
+  for (const { dir, file, size } of files) {
+    fs.mkdirSync(dir, { recursive: true });
+    fs.writeFileSync(file, Buffer.alloc(size, 42));
+  }
+  // Root-level .internal (excluded by both paths identically).
+  fs.mkdirSync(path.join(root, '.internal', 'tempFiles'), { recursive: true });
+  fs.writeFileSync(path.join(root, '.internal', 'tempFiles', 'tmp.bin'), Buffer.alloc(777, 9));
+  // Some empty dirs too.
+  fs.mkdirSync(path.join(root, 'empty-dir-1'), { recursive: true });
+  fs.mkdirSync(path.join(root, 'a', 'empty-dir-2'), { recursive: true });
+}
+
+function assertEqual(label, a, b) {
+  const same = a.amount === b.amount;
+  console.log(`  ${same ? 'OK ' : 'FAIL'} ${label}: FileSizeReporter=${a.amount}  DuSizeReporter=${b.amount}${same ? '' : '  <-- MISMATCH'}`);
+  return same;
+}
+
+let allOk = true;
+
+async function main() {
+  for (let iter = 1; iter <= ITERATIONS; iter++) {
+    const root = fs.mkdtempSync(path.join(os.tmpdir(), 'equiv-'));
+    const files = randomTree(root, MAX_FILES);
+    writeTree(root, files);
+
+    const mapper = makeMapper(root);
+    const oldReporter = new FileSizeReporter(mapper, root, IGNORE);
+    const duReporter = new ForcedDu(mapper, root, IGNORE);
+    const nodeReporter = new ForcedNode(mapper, root, IGNORE);
+    const podId = { path: 'http://example.com/' };
+
+    const oldSize = await oldReporter.getSize(podId);
+    const duSize = await duReporter.getSize(podId);
+    const nodeSize = await nodeReporter.getSize(podId);
+
+    console.log(`Iteration ${iter} (${files.length} files):`);
+    const okDu = assertEqual('du path    ', oldSize, duSize);
+    const okNode = assertEqual('node fallback', oldSize, nodeSize);
+    if (!okDu || !okNode) allOk = false;
+    console.log('');
+
+    fs.rmSync(root, { recursive: true, force: true });
+  }
+}
+
+main().then(() => {
+  if (allOk) {
+    console.log(`EQUIVALENT: DuSizeReporter matches FileSizeReporter across ${ITERATIONS} random trees.`);
+    process.exit(0);
+  } else {
+    console.error('MISMATCH FOUND — sizes are NOT equivalent.');
+    process.exit(1);
+  }
+}).catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

src/index.ts

@@ -1,5 +1,7 @@
 export * from "./storage/RdfPatchingStore";
 export * from "./storage/patch/ThrowingN3Patcher";
+export * from "./storage/quota/FastQuotaStrategy";
+export * from "./storage/size-reporter/DuSizeReporter";
 export * from './FedcmHttpHandler';
 export * from './http/output/PivotResponseWriter';
 export * from './identity/interaction/password/MigratedPasswordLoginHandler';