From 82ec4325b854fa8a43cad98847cfe329eb71d216 Mon Sep 17 00:00:00 2001 From: bourgeoa Date: Sat, 15 Aug 2026 19:11:33 +0200 Subject: [PATCH 1/2] =?UTF-8?q?feat(storage):=20fast=20pod=20quota=20?= =?UTF-8?q?=E2=80=94=20DuSizeReporter=20+=20FastQuotaStrategy=20(A+B)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-write pod quota on the CSS file backend recursively walks the whole pod per stream chunk (chunks × O(N)), causing extreme slowness and memory exhaustion on large inboxes. This adds two pivot components wired via config Overrides (no CSS fork): - DuSizeReporter: SizeReporter measuring apparent bytes (du -sb / BSD -s -A -B 1) with a per-path TTL cache, Node-walk fallback when du is unavailable (Windows), and invalidate() to drop resource + ancestor (pod root) cache entries after writes. - FastQuotaStrategy: extends PodQuotaStrategy; computes available space ONCE per write and only tracks the write's own byte delta per chunk (was: full pod walk per chunk). Config: config/storage/backend/quota-fast-file.json overrides urn:solid-server:default:SizeReporter + QuotaStrategy (70 MB, apparent bytes, ignoreFolders ^/\.internal$, ttl 5000); imported from customise-me.json. Benchmark (5000 files/1KB pod, 4MB write in 64KB chunks): guard 36 279 ms -> 3.3 ms (~11 000x); cached getSize 0.13 ms. Equivalence proof: 12/12 random trees byte-identical to FileSizeReporter (du path + Node fallback). Includes unit tests and scripts/benchmark-quota.js + scripts/verify-size-equivalence.js. --- config/customise-me.json | 15 +- config/storage/backend/quota-fast-file.json | 50 +++++ scripts/benchmark-quota.js | 145 ++++++++++++++ scripts/verify-size-equivalence.js | 120 ++++++++++++ src/index.ts | 2 + src/storage/quota/FastQuotaStrategy.ts | 72 +++++++ src/storage/size-reporter/DuSizeReporter.ts | 206 ++++++++++++++++++++ test/unit/storage/DuSizeReporter.test.ts | 135 +++++++++++++ test/unit/storage/FastQuotaStrategy.test.ts | 111 +++++++++++ 9 files changed, 844 insertions(+), 12 deletions(-) create mode 100644 config/storage/backend/quota-fast-file.json create mode 100644 scripts/benchmark-quota.js create mode 100644 scripts/verify-size-equivalence.js create mode 100644 src/storage/quota/FastQuotaStrategy.ts create mode 100644 src/storage/size-reporter/DuSizeReporter.ts create mode 100644 test/unit/storage/DuSizeReporter.test.ts create mode 100644 test/unit/storage/FastQuotaStrategy.test.ts diff --git a/config/customise-me.json b/config/customise-me.json index 02ab0ac..af26c13 100644 --- a/config/customise-me.json +++ b/config/customise-me.json @@ -4,6 +4,9 @@ "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^7.0.0/components/context.jsonld", "https://linkedsoftwaredependencies.org/bundles/npm/@solid/pivot/^1.0.0/components/context.jsonld" ], + "import": [ + "pivot:config/storage/backend/quota-fast-file.json" + ], "@graph": [ { "comment": "The settings of your email server.", @@ -31,18 +34,6 @@ "templateFolder": "templates/pod" } }, - { - "comment": "Sets the maximum size of a single pod to 70MB.", - "@type": "Override", - "overrideInstance": { - "@id": "urn:solid-server:default:QuotaStrategy" - }, - "overrideParameters": { - "@type": "PodQuotaStrategy", - "limit_amount": 70000000, - "limit_unit": "bytes" - } - }, { "comment": "Serve Databrowser as default representation", "@id": "urn:solid-server:default:DefaultUiConverter", diff --git a/config/storage/backend/quota-fast-file.json b/config/storage/backend/quota-fast-file.json new file mode 100644 index 0000000..405a546 --- /dev/null +++ b/config/storage/backend/quota-fast-file.json @@ -0,0 +1,50 @@ +{ + "comment": "Fast per-pod quota: DuSizeReporter (du + TTL cache) and FastQuotaStrategy (no per-chunk pod walk).", + "@context": [ + "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^7.0.0/components/context.jsonld", + "https://linkedsoftwaredependencies.org/bundles/npm/@solid/pivot/^1.0.0/components/context.jsonld" + ], + "@graph": [ + { + "comment": "SizeReporter backed by du with a per-path TTL cache, measuring apparent bytes (portable, user-manageable). Falls back to the Node walk when du is unavailable.", + "@type": "Override", + "overrideInstance": { + "@id": "urn:solid-server:default:SizeReporter" + }, + "overrideParameters": { + "@type": "DuSizeReporter", + "fileIdentifierMapper": { + "@id": "urn:solid-server:default:FileIdentifierMapper" + }, + "rootFilePath": { + "@id": "urn:solid-server:default:variable:rootFilePath" + }, + "ignoreFolders": [ + "^/\\.internal$" + ], + "ttl": 5000 + } + }, + { + "comment": "Pod quota strategy that computes the available space once per write instead of per stream chunk.", + "@type": "Override", + "overrideInstance": { + "@id": "urn:solid-server:default:QuotaStrategy" + }, + "overrideParameters": { + "@type": "FastQuotaStrategy", + "limit_amount": 70000000, + "limit_unit": "bytes", + "reporter": { + "@id": "urn:solid-server:default:SizeReporter" + }, + "identifierStrategy": { + "@id": "urn:solid-server:default:IdentifierStrategy" + }, + "accessor": { + "@id": "urn:solid-server:default:AtomicFileDataAccessor" + } + } + } + ] +} diff --git a/scripts/benchmark-quota.js b/scripts/benchmark-quota.js new file mode 100644 index 0000000..aebd87d --- /dev/null +++ b/scripts/benchmark-quota.js @@ -0,0 +1,145 @@ +/** + * Benchmark: CSS pod-quota chain — old (FileSizeReporter + QuotaStrategy) + * vs pivot (DuSizeReporter + FastQuotaStrategy). + * + * Measures exactly what was optimized: + * 1. Full pod walk cost (old recursive Node walk vs du walk) + * 2. The per-write quota guard (old: full walk PER CHUNK; new: walk once) + * 3. The TTL cache effect on repeated size queries + * + * Run: node scripts/benchmark-quota.js [fileCount] [fileBytes] + * e.g. node scripts/benchmark-quota.js 10000 1024 + * + * NOTE: on bare Windows there is no `du`, so the walk times will be similar + * between the two (the guard + cache wins still show). On Linux/WSL the du + * walk is 10-100x faster. + */ +const { performance } = require('node:perf_hooks'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { PassThrough } = require('node:stream'); +const { + FileSizeReporter, + QuotaStrategy, +} = require('@solid/community-server'); +const { DuSizeReporter } = require('../dist/storage/size-reporter/DuSizeReporter.js'); +const { FastQuotaStrategy } = require('../dist/storage/quota/FastQuotaStrategy.js'); + +const FILE_COUNT = Number(process.argv[2]) || 5000; +const FILE_BYTES = Number(process.argv[3]) || 1024; +const WRITE_BYTES = 4 * 1024 * 1024; // simulated write body +const CHUNK_BYTES = 64 * 1024; + +function makeMapper(root) { + return { + async mapUrlToFilePath(identifier) { + const url = new URL(identifier.path); + return { identifier, filePath: path.join(root, url.pathname), contentType: undefined, isMetadata: false }; + }, + async mapFilePathToUrl() { throw new Error('n/a'); }, + }; +} + +// Old strategy: base QuotaStrategy whose pod size = reporter.getSize(podRoot) +class OldBenchStrategy extends QuotaStrategy { + constructor(reporter, limit, podRoot) { + super(reporter, limit); + this.podRoot = podRoot; + } + async getTotalSpaceUsed() { + return this.reporter.getSize(this.podRoot); + } +} + +// New strategy: FastQuotaStrategy with the same pod-size hook +class NewBenchStrategy extends FastQuotaStrategy { + constructor(reporter, limit, podRoot) { + super(limit, reporter, {}, {}); + this.podRoot = podRoot; + } + async getTotalSpaceUsed() { + return this.reporter.getSize(this.podRoot); + } +} + +function seedPod(root, count, bytes) { + const inbox = path.join(root, 'inbox'); + fs.mkdirSync(inbox, { recursive: true }); + const buf = Buffer.alloc(bytes, 7); + for (let i = 0; i < count; i++) { + fs.writeFileSync(path.join(inbox, `file-${i}.bin`), buf); + } +} + +function writeThroughGuard(guard, totalBytes, chunkBytes) { + return new Promise((resolve, reject) => { + const chunks = []; + let remaining = totalBytes; + guard.on('data', () => {}); + guard.on('end', () => resolve(chunks)); + guard.on('error', reject); + while (remaining > 0) { + const size = Math.min(chunkBytes, remaining); + guard.write(Buffer.alloc(size, 1)); + remaining -= size; + } + guard.end(); + }); +} + +async function timed(fn) { + const start = performance.now(); + await fn(); + return performance.now() - start; +} + +async function main() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'quota-bench-')); + const podRoot = { path: 'http://example.com/' }; + const limit = { unit: 'bytes', amount: 10 * 1024 * 1024 * 1024 }; + + console.log(`Seeding ${FILE_COUNT} files × ${FILE_BYTES} B in ${root} …`); + const seedStart = performance.now(); + seedPod(root, FILE_COUNT, FILE_BYTES); + console.log(` seeded in ${(performance.now() - seedStart).toFixed(0)} ms\n`); + + const mapper = makeMapper(root); + const reporterOld = new FileSizeReporter(mapper, root); + const reporterNew = new DuSizeReporter(mapper, root, [ '^/\\.internal$' ]); + + // --- 1. Full pod walk (single getSize) --- + const walkOld = await timed(() => reporterOld.getSize(podRoot)); + const walkNew = await timed(() => reporterNew.getSize(podRoot)); + const walkCached = await timed(() => reporterNew.getSize(podRoot)); + + console.log('1. FULL POD WALK (getSize of pod root)'); + console.log(` old (Node recursive walk): ${walkOld.toFixed(1)} ms`); + console.log(` new (du, first call): ${walkNew.toFixed(1)} ms`); + console.log(` new (du, cached, TTL hit): ${walkCached.toFixed(3)} ms`); + if (walkNew > 0) { + console.log(` walk speedup (first call): ${(walkOld / walkNew).toFixed(1)}×`); + } + + // --- 2. Per-write guard (write body through the quota guard) --- + const strategyOld = new OldBenchStrategy(reporterOld, limit, podRoot); + const strategyNew = new NewBenchStrategy(reporterNew, limit, podRoot); + + const guardOld = await strategyOld.createQuotaGuard({ path: 'http://example.com/inbox/file-new.bin' }); + const guardNew = await strategyNew.createQuotaGuard({ path: 'http://example.com/inbox/file-new.bin' }); + + const guardOldMs = await timed(() => writeThroughGuard(guardOld, WRITE_BYTES, CHUNK_BYTES)); + const guardNewMs = await timed(() => writeThroughGuard(guardNew, WRITE_BYTES, CHUNK_BYTES)); + + console.log(`\n2. PER-WRITE QUOTA GUARD (${(WRITE_BYTES / 1024 / 1024).toFixed(0)} MB body, ${CHUNK_BYTES / 1024} KB chunks)`); + console.log(` old (walk per chunk, ${Math.ceil(WRITE_BYTES / CHUNK_BYTES)} chunks): ${guardOldMs.toFixed(1)} ms`); + console.log(` new (walk once + cache): ${guardNewMs.toFixed(1)} ms`); + if (guardNewMs > 0) { + console.log(` guard speedup: ${(guardOldMs / guardNewMs).toFixed(1)}×`); + } + + fs.rmSync(root, { recursive: true, force: true }); + console.log('\nDone.'); +} + +main().catch((err) => { console.error(err); process.exit(1); }); diff --git a/scripts/verify-size-equivalence.js b/scripts/verify-size-equivalence.js new file mode 100644 index 0000000..04407e5 --- /dev/null +++ b/scripts/verify-size-equivalence.js @@ -0,0 +1,120 @@ +/** + * Equivalence proof: DuSizeReporter vs CSS FileSizeReporter. + * + * Generates random pod trees and asserts both reporters return the EXACT same + * apparent-byte total (same ignoreFolders), for the du path and the Node-walk + * fallback. Exits non-zero on any mismatch. + * + * Run with GNU du on PATH to exercise the real du path, e.g. on Windows: + * $env:PATH = "C:\Program Files\Git\usr\bin;$env:PATH" + * node scripts/verify-size-equivalence.js [iterations] [maxFiles] + */ +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { FileSizeReporter } = require('@solid/community-server'); +const { DuSizeReporter } = require('../dist/storage/size-reporter/DuSizeReporter.js'); + +const ITERATIONS = Number(process.argv[2]) || 10; +const MAX_FILES = Number(process.argv[3]) || 60; +const IGNORE = [ '^/\\.internal$' ]; + +class ForcedDu extends DuSizeReporter { + async detectDu() { return 'gnu'; } +} +class ForcedNode extends DuSizeReporter { + async detectDu() { return 'none'; } +} + +function makeMapper(root) { + return { + async mapUrlToFilePath(identifier) { + const url = new URL(identifier.path); + return { identifier, filePath: path.join(root, url.pathname), contentType: undefined, isMetadata: false }; + }, + async mapFilePathToUrl() { throw new Error('n/a'); }, + }; +} + +// Random names for nested content. .internal is handled separately below: +// CSS only ever places .internal at the pod ROOT (temp files), where the +// anchored regex ^/\.internal$ and du's basename --exclude=.internal agree. +const SEGMENTS = [ 'a', 'b', 'c', 'd', 'e', 'inbox', 'public', 'private', 'settings' ]; + +function randomTree(root, maxFiles) { + const files = []; + const count = 1 + Math.floor(Math.random() * maxFiles); + for (let i = 0; i < count; i++) { + // 1-4 nested segments, each a subdirectory (mkdirp on write). + const depth = 1 + Math.floor(Math.random() * 3); + const segs = []; + for (let d = 0; d < depth; d++) { + segs.push(SEGMENTS[Math.floor(Math.random() * SEGMENTS.length)]); + } + const dir = path.join(root, ...segs); + const file = path.join(dir, `f${i}.bin`); + const size = Math.floor(Math.random() * 50_000); + files.push({ dir, file, size }); + } + return files; +} + +function writeTree(root, files) { + for (const { dir, file, size } of files) { + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(file, Buffer.alloc(size, 42)); + } + // Root-level .internal (excluded by both paths identically). + fs.mkdirSync(path.join(root, '.internal', 'tempFiles'), { recursive: true }); + fs.writeFileSync(path.join(root, '.internal', 'tempFiles', 'tmp.bin'), Buffer.alloc(777, 9)); + // Some empty dirs too. + fs.mkdirSync(path.join(root, 'empty-dir-1'), { recursive: true }); + fs.mkdirSync(path.join(root, 'a', 'empty-dir-2'), { recursive: true }); +} + +function assertEqual(label, a, b) { + const same = a.amount === b.amount; + console.log(` ${same ? 'OK ' : 'FAIL'} ${label}: FileSizeReporter=${a.amount} DuSizeReporter=${b.amount}${same ? '' : ' <-- MISMATCH'}`); + return same; +} + +let allOk = true; + +async function main() { + for (let iter = 1; iter <= ITERATIONS; iter++) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'equiv-')); + const files = randomTree(root, MAX_FILES); + writeTree(root, files); + + const mapper = makeMapper(root); + const oldReporter = new FileSizeReporter(mapper, root, IGNORE); + const duReporter = new ForcedDu(mapper, root, IGNORE); + const nodeReporter = new ForcedNode(mapper, root, IGNORE); + const podId = { path: 'http://example.com/' }; + + const oldSize = await oldReporter.getSize(podId); + const duSize = await duReporter.getSize(podId); + const nodeSize = await nodeReporter.getSize(podId); + + console.log(`Iteration ${iter} (${files.length} files):`); + const okDu = assertEqual('du path ', oldSize, duSize); + const okNode = assertEqual('node fallback', oldSize, nodeSize); + if (!okDu || !okNode) allOk = false; + console.log(''); + + fs.rmSync(root, { recursive: true, force: true }); + } +} + +main().then(() => { + if (allOk) { + console.log(`EQUIVALENT: DuSizeReporter matches FileSizeReporter across ${ITERATIONS} random trees.`); + process.exit(0); + } else { + console.error('MISMATCH FOUND — sizes are NOT equivalent.'); + process.exit(1); + } +}).catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/src/index.ts b/src/index.ts index 2d9b1a8..3d77c48 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,7 @@ export * from "./storage/RdfPatchingStore"; export * from "./storage/patch/ThrowingN3Patcher"; +export * from "./storage/quota/FastQuotaStrategy"; +export * from "./storage/size-reporter/DuSizeReporter"; export * from './FedcmHttpHandler'; export * from './http/output/PivotResponseWriter'; export * from './identity/interaction/password/MigratedPasswordLoginHandler'; diff --git a/src/storage/quota/FastQuotaStrategy.ts b/src/storage/quota/FastQuotaStrategy.ts new file mode 100644 index 0000000..93e5466 --- /dev/null +++ b/src/storage/quota/FastQuotaStrategy.ts @@ -0,0 +1,72 @@ +import { PassThrough } from 'node:stream'; +import { + PodQuotaStrategy, + guardStream, +} from '@solid/community-server'; +// PayloadHttpError is not exported from the package root; deep import is +// needed to surface a 413 (Payload Too Large) on quota breaches. +import { PayloadHttpError } from '@solid/community-server/dist/util/errors/PayloadHttpError'; +import type { Guarded } from '@solid/community-server'; +import type { DataAccessor } from '@solid/community-server'; +import type { IdentifierStrategy } from '@solid/community-server'; +import type { ResourceIdentifier } from '@solid/community-server'; +import type { Size } from '@solid/community-server'; +import type { SizeReporter } from '@solid/community-server'; +import type { DuSizeReporter } from '../size-reporter/DuSizeReporter'; + +/** + * Pod quota strategy that avoids the per-chunk full pod walk. + * + * CSS's default `QuotaStrategy.createQuotaGuard` calls `getAvailableSpace` + * (which performs a full pod walk) for EVERY stream chunk — `chunks × O(N)`. + * This override computes the available space ONCE before streaming, then only + * tracks the current write's own byte delta per chunk. When the write + * completes, the reporter's size cache is invalidated so the QuotaValidator's + * post-write check (and the next write's pre-check) re-walk fresh. + */ +export class FastQuotaStrategy extends PodQuotaStrategy { + public constructor( + limit: Size, + reporter: SizeReporter, + identifierStrategy: IdentifierStrategy, + accessor: DataAccessor, + ) { + super(limit, reporter, identifierStrategy, accessor); + } + + public async createQuotaGuard(identifier: ResourceIdentifier): Promise> { + // Compute the available space ONCE. getAvailableSpace already subtracts + // the overwritten resource's own size, and nothing else about the pod + // changes mid-write (atomic writes go to /.internal/, excluded by the + // reporter), so this single value is safe for the whole stream. + const availableSpace = await this.getAvailableSpace(identifier); + const reporter = this.reporter as DuSizeReporter & SizeReporter; + let total = 0; + + return guardStream(new PassThrough({ + async transform(chunk: any, _encoding: string, done: () => void): Promise { + total += await reporter.calculateChunkSize(chunk); + if (availableSpace.amount < total) { + this.destroy(new PayloadHttpError( + `Quota exceeded by ${total - availableSpace.amount} ${availableSpace.unit} during write`, + )); + } + this.push(chunk); + done(); + }, + async flush(done: (error?: Error) => void): Promise { + // Drop the cached sizes (resource + its ancestors incl. the pod root) + // so the QuotaValidator's after-write check re-walks and sees the new + // state. Best-effort: a failure to invalidate must not fail the write. + if (typeof reporter.invalidate === 'function') { + try { + await reporter.invalidate(identifier); + } catch { + // Ignore cache invalidation errors. + } + } + done(); + }, + })); + } +} diff --git a/src/storage/size-reporter/DuSizeReporter.ts b/src/storage/size-reporter/DuSizeReporter.ts new file mode 100644 index 0000000..eb1f80f --- /dev/null +++ b/src/storage/size-reporter/DuSizeReporter.ts @@ -0,0 +1,206 @@ +import { execFile } from 'node:child_process'; +import { promises as fsPromises } from 'node:fs'; +import { promisify } from 'node:util'; +import { + UNIT_BYTES, + joinFilePath, + normalizeFilePath, + trimTrailingSlashes, +} from '@solid/community-server'; +import type { + FileIdentifierMapper, + RepresentationMetadata, + ResourceIdentifier, + Size, + SizeReporter, +} from '@solid/community-server'; + +const execFileAsync = promisify(execFile); + +// Cache entry: computed size + expiry timestamp +interface CacheEntry { + size: number; + expiresAt: number; +} + +/** + * A {@link SizeReporter} that measures a resource (and its children) in + * apparent bytes, using GNU/BSD `du` as a fast C-level walk with a per-path + * TTL cache, falling back to a plain Node walk when no compatible `du` + * exists (e.g. bare Windows). + * + * The unit is apparent bytes (sum of `st_size`) — identical to CSS's + * `FileSizeReporter`, portable across servers/filesystems and + * user-manageable. `stat.blocks` (disk usage) is deliberately NOT used: + * the result would depend on the server's filesystem cluster size. + */ +export class DuSizeReporter implements SizeReporter { + private readonly fileIdentifierMapper: FileIdentifierMapper; + private readonly rootFilePath: string; + private readonly ignoreFolders: RegExp[]; + private readonly ttlMs: number; + private readonly cache: Map = new Map(); + private duFlavor: 'gnu' | 'bsd' | 'none' | null = null; + + public constructor( + fileIdentifierMapper: FileIdentifierMapper, + rootFilePath: string, + ignoreFolders: string[] = [], + ttl: number = 5000, + ) { + this.fileIdentifierMapper = fileIdentifierMapper; + this.rootFilePath = normalizeFilePath(rootFilePath); + this.ignoreFolders = ignoreFolders.map((folder): RegExp => new RegExp(folder, 'u')); + this.ttlMs = ttl; + } + + /** The DuSizeReporter always returns data in the form of bytes. */ + public getUnit(): string { + return UNIT_BYTES; + } + + /** + * Returns the size of the given resource (and its children) in apparent + * bytes, using the per-path TTL cache when possible. + */ + public async getSize(identifier: ResourceIdentifier): Promise { + const { filePath } = await this.fileIdentifierMapper.mapUrlToFilePath(identifier, false); + const normalized = normalizeFilePath(filePath); + const cached = this.cache.get(normalized); + if (cached && cached.expiresAt > Date.now()) { + return { unit: UNIT_BYTES, amount: cached.size }; + } + const amount = await this.computeTotalSize(normalized); + this.cache.set(normalized, { size: amount, expiresAt: Date.now() + this.ttlMs }); + return { unit: UNIT_BYTES, amount }; + } + + /** + * Drop the cached size for the given resource and all of its ancestors + * (e.g. the pod root). Called when a write to the resource completes, so + * the next size query re-walks and reflects the new content. + */ + public async invalidate(identifier: ResourceIdentifier): Promise { + try { + const { filePath } = await this.fileIdentifierMapper.mapUrlToFilePath(identifier, false); + const normalized = normalizeFilePath(filePath); + for (const key of this.cache.keys()) { + if (key === normalized || normalized.startsWith(key)) { + this.cache.delete(key); + } + } + } catch { + // Best-effort: if the resource cannot be mapped, leave the cache as-is. + } + } + + /** The size of a chunk is simply its length in bytes. */ + public async calculateChunkSize(chunk: unknown): Promise { + return Buffer.isBuffer(chunk) ? chunk.length : Number((chunk as any)?.length) || 0; + } + + /** The estimated size of a resource is simply the content-length header. */ + public async estimateSize(metadata: RepresentationMetadata): Promise { + return metadata.contentLength; + } + + // --- Walk implementations --- + + private async computeTotalSize(fileLocation: string): Promise { + const flavor = await this.detectDu(); + if (flavor !== 'none') { + try { + return await this.computeTotalSizeWithDu(fileLocation, flavor); + } catch { + // du failed (e.g. no du after all, permission error) — fall back to the Node walk. + } + } + return this.computeTotalSizeWithNode(fileLocation); + } + + private async computeTotalSizeWithDu(fileLocation: string, flavor: 'gnu' | 'bsd'): Promise { + const args: string[] = flavor === 'gnu' ? [ '-sb' ] : [ '-s', '-A', '-B', '1' ]; + for (const pattern of this.duExcludePatterns()) { + args.push(flavor === 'gnu' ? '--exclude' : '-I', pattern); + } + args.push(fileLocation); + const { stdout } = await execFileAsync('du', args, { maxBuffer: 64 * 1024 * 1024 }); + const amount = Number(stdout.trim().split(/\s+/)[0]); + if (!Number.isFinite(amount)) { + throw new Error(`Could not parse du output: ${stdout.trim()}`); + } + return amount; + } + + /** + * Plain Node recursive walk — the same semantics as CSS's + * `FileSizeReporter.getTotalSize`. Used when no compatible `du` exists. + */ + private async computeTotalSizeWithNode(fileLocation: string): Promise { + let stat; + try { + stat = await fsPromises.stat(fileLocation); + } catch { + return 0; + } + // If the file's location points to a file, simply return the file's size. + if (stat.isFile()) { + return stat.size; + } + // Recursively add all sizes of children to the total. + const childFiles = await fsPromises.readdir(fileLocation); + const rootFilePathLength = trimTrailingSlashes(this.rootFilePath).length; + let totalSize = stat.size; + for (const current of childFiles) { + const childFileLocation = normalizeFilePath(joinFilePath(fileLocation, current)); + // Exclude internal files, matching FileSizeReporter's behavior. + if (!this.ignoreFolders.some((folder): boolean => folder.test(childFileLocation.slice(rootFilePathLength)))) { + totalSize += await this.computeTotalSizeWithNode(childFileLocation); + } + } + return totalSize; + } + + protected async detectDu(): Promise<'gnu' | 'bsd' | 'none'> { + if (this.duFlavor) { + return this.duFlavor; + } + try { + await execFileAsync('du', [ '--version' ], { timeout: 1000 }); + this.duFlavor = 'gnu'; + } catch (error: any) { + // ENOENT: no `du` at all (e.g. bare Windows). Anything else means the + // GNU long option was rejected → assume BSD; if BSD flags fail at use + // time, the caller falls back to the Node walk. + this.duFlavor = error?.code === 'ENOENT' ? 'none' : 'bsd'; + } + return this.duFlavor; + } + + /** + * Convert the configured ignore-folder regexes into `du` exclude patterns. + * GNU/BSD `du` matches exclude patterns against path components/basenames + * (not against the full leading-slash path), so a regex like `^/\.internal$` + * becomes the exclude `.internal`. + * + * Only simple anchored folder patterns are convertible + * (`^/name$`); complex regexes that cannot be expressed as a du exclude are + * skipped here — the Node-walk fallback still applies them verbatim. + */ + private duExcludePatterns(): string[] { + const patterns: string[] = []; + for (const regex of this.ignoreFolders) { + // RegExp.source always escapes '/' as '\/' (e.g. `^/\.internal$` → + // `^\/\.internal$`). Strip the leading '^' + '/' (possibly '\/'), drop + // the trailing '$', then unescape the remaining escapes. + let src = regex.source.replace(/^\^?\\?\//, ''); + src = src.replace(/\$?$/, ''); + src = src.replace(/\\./g, '.'); + // Keep only patterns with no remaining regex metacharacters. + if (/^[A-Za-z0-9._-]+$/.test(src)) { + patterns.push(src); + } + } + return patterns; + } +} diff --git a/test/unit/storage/DuSizeReporter.test.ts b/test/unit/storage/DuSizeReporter.test.ts new file mode 100644 index 0000000..ae013e5 --- /dev/null +++ b/test/unit/storage/DuSizeReporter.test.ts @@ -0,0 +1,135 @@ +import { promises as fs } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { FileIdentifierMapper, ResourceIdentifier, Size } from '@solid/community-server'; +import { DuSizeReporter } from '../../../src/storage/size-reporter/DuSizeReporter'; + +// Force the du-based path (works even where du is absent — the Node walk +// produces the same apparent-byte sum for a simple tree). +class ForceDuReporter extends DuSizeReporter { + protected override async detectDu(): Promise<'gnu' | 'bsd' | 'none'> { + return 'gnu'; + } +} + +// Force the Node-walk fallback path. +class ForceNodeReporter extends DuSizeReporter { + protected override async detectDu(): Promise<'gnu' | 'bsd' | 'none'> { + return 'none'; + } +} + +function createMapper(root: string): FileIdentifierMapper { + return { + async mapUrlToFilePath(identifier: ResourceIdentifier): Promise { + const url = new URL(identifier.path); + return { identifier, filePath: join(root, url.pathname), contentType: undefined, isMetadata: false }; + }, + async mapFilePathToUrl(): Promise { + throw new Error('Not implemented'); + }, + }; +} + +describe('A DuSizeReporter', (): void => { + let root: string; + let mapper: FileIdentifierMapper; + + beforeEach(async(): Promise => { + root = await fs.mkdtemp(join(tmpdir(), 'du-size-reporter-')); + mapper = createMapper(root); + }); + + afterEach(async(): Promise => { + await fs.rm(root, { recursive: true, force: true }); + }); + + it('returns the apparent size of a file.', async(): Promise => { + const reporter = new DuSizeReporter(mapper, root); + await fs.writeFile(join(root, 'a.txt'), Buffer.alloc(100)); + const size = await reporter.getSize({ path: 'http://example.com/a.txt' }); + expect(size).toEqual({ unit: 'bytes', amount: 100 }); + }); + + it('reports the same size whether du or the Node fallback is used.', async(): Promise => { + await fs.mkdir(join(root, 'dir')); + await fs.writeFile(join(root, 'dir', 'a.txt'), Buffer.alloc(100)); + await fs.writeFile(join(root, 'dir', 'b.txt'), Buffer.alloc(50)); + const viaDu = await new ForceDuReporter(mapper, root).getSize({ path: 'http://example.com/dir/a.txt' }); + const viaNode = await new ForceNodeReporter(mapper, root).getSize({ path: 'http://example.com/dir/a.txt' }); + expect(viaDu.amount).toBe(100); + expect(viaNode.amount).toBe(100); + }); + + it('serves a cached result within the TTL window without re-walking.', async(): Promise => { + const reporter = new ForceDuReporter(mapper, root, [], 60_000); + await fs.writeFile(join(root, 'a.txt'), Buffer.alloc(100)); + const first = await reporter.getSize({ path: 'http://example.com/a.txt' }); + // Change the file without invalidating — the cache must still serve the old size. + await fs.writeFile(join(root, 'a.txt'), Buffer.alloc(200)); + const cached = await reporter.getSize({ path: 'http://example.com/a.txt' }); + expect(first.amount).toBe(100); + expect(cached.amount).toBe(100); + }); + + it('recomputes after invalidation.', async(): Promise => { + const reporter = new DuSizeReporter(mapper, root, [], 60_000); + await fs.writeFile(join(root, 'a.txt'), Buffer.alloc(100)); + await reporter.getSize({ path: 'http://example.com/a.txt' }); + await fs.writeFile(join(root, 'a.txt'), Buffer.alloc(200)); + await reporter.invalidate({ path: 'http://example.com/a.txt' }); + const after = await reporter.getSize({ path: 'http://example.com/a.txt' }); + expect(after.amount).toBe(200); + }); + + it('invalidates ancestor entries (e.g. the pod root) as well.', async(): Promise => { + const reporter = new DuSizeReporter(mapper, root, [], 60_000); + await fs.mkdir(join(root, 'dir')); + await fs.writeFile(join(root, 'dir', 'a.txt'), Buffer.alloc(100)); + const rootSize = await reporter.getSize({ path: 'http://example.com/' }); + expect(rootSize.amount).toBeGreaterThanOrEqual(100); + await fs.writeFile(join(root, 'dir', 'a.txt'), Buffer.alloc(300)); + await reporter.invalidate({ path: 'http://example.com/dir/a.txt' }); + const newRootSize = await reporter.getSize({ path: 'http://example.com/' }); + expect(newRootSize.amount).toBe(rootSize.amount + 200); + }); + + it('excludes the ignoreFolders from the total.', async(): Promise => { + const reporter = new DuSizeReporter(mapper, root, [ '^/\\.internal$' ]); + await fs.mkdir(join(root, '.internal')); + await fs.writeFile(join(root, '.internal', 'x.txt'), Buffer.alloc(1000)); + const without = await reporter.getSize({ path: 'http://example.com/' }); + // Adding a file inside .internal must not change the reported size. + await fs.writeFile(join(root, '.internal', 'y.txt'), Buffer.alloc(1000)); + await reporter.invalidate({ path: 'http://example.com/' }); + const still = await reporter.getSize({ path: 'http://example.com/' }); + expect(still.amount).toBe(without.amount); + // Adding a normal file must increase it. + await fs.writeFile(join(root, 'a.txt'), Buffer.alloc(50)); + await reporter.invalidate({ path: 'http://example.com/' }); + const increased = await reporter.getSize({ path: 'http://example.com/' }); + expect(increased.amount).toBe(without.amount + 50); + }); + + it('returns the content-length as the estimated size.', async(): Promise => { + const reporter = new DuSizeReporter(mapper, root); + await expect(reporter.estimateSize({ contentLength: 42 } as any)).resolves.toBe(42); + await expect(reporter.estimateSize({} as any)).resolves.toBeUndefined(); + }); + + it('calculates the chunk size as the buffer length.', async(): Promise => { + const reporter = new DuSizeReporter(mapper, root); + await expect(reporter.calculateChunkSize(Buffer.alloc(17))).resolves.toBe(17); + }); + + it('returns the byte unit.', async(): Promise => { + const reporter = new DuSizeReporter(mapper, root); + expect(reporter.getUnit()).toBe('bytes'); + }); + + it('reports a size of 0 for a missing resource.', async(): Promise => { + const reporter = new DuSizeReporter(mapper, root); + const size: Size = await reporter.getSize({ path: 'http://example.com/nope' }); + expect(size.amount).toBe(0); + }); +}); diff --git a/test/unit/storage/FastQuotaStrategy.test.ts b/test/unit/storage/FastQuotaStrategy.test.ts new file mode 100644 index 0000000..c7d185b --- /dev/null +++ b/test/unit/storage/FastQuotaStrategy.test.ts @@ -0,0 +1,111 @@ +import { PassThrough } from 'node:stream'; +import type { + DataAccessor, + IdentifierStrategy, + ResourceIdentifier, + Size, + SizeReporter, +} from '@solid/community-server'; +import { FastQuotaStrategy } from '../../../src/storage/quota/FastQuotaStrategy'; + +// A strategy with a fixed pod total so createQuotaGuard can be exercised +// without a real pod/accessor setup. +class FixedTotalStrategy extends FastQuotaStrategy { + private readonly used: number; + public constructor(used: number, limit: Size, reporter: SizeReporter) { + super(limit, reporter, {} as IdentifierStrategy, {} as DataAccessor); + this.used = used; + } + + protected async getTotalSpaceUsed(): Promise { + return { unit: 'bytes', amount: this.used }; + } +} + +function mockReporter(oldResourceSize: number): jest.Mocked> & { invalidate: jest.Mock } { + const reporter: any = { + getUnit: jest.fn((): string => 'bytes'), + getSize: jest.fn(async(): Promise => ({ unit: 'bytes', amount: oldResourceSize })), + calculateChunkSize: jest.fn(async(chunk: Buffer): Promise => chunk.length), + estimateSize: jest.fn(async(): Promise => undefined), + invalidate: jest.fn(async(): Promise => undefined), + }; + return reporter; +} + +// Writes all chunks into the guard and waits for it to end (or error). +function writeChunks(guard: PassThrough, chunks: Buffer[]): Promise { + return new Promise((resolve, reject): void => { + guard.on('data', (): void => { + // consume + }); + guard.on('end', resolve); + guard.on('error', reject); + for (const chunk of chunks) { + guard.write(chunk); + } + guard.end(); + }); +} + +describe('A FastQuotaStrategy', (): void => { + const identifier: ResourceIdentifier = { path: 'http://example.com/foo' }; + // available space = limit - pod total + overwritten resource size + // = 100 - 90 + 10 = 20 bytes + const limit: Size = { unit: 'bytes', amount: 100 }; + + it('calls getAvailableSpace (the pod walk) only once per write, not per chunk.', async(): Promise => { + const reporter = mockReporter(10); + const strategy = new FixedTotalStrategy(90, limit, reporter); + const guard = await strategy.createQuotaGuard(identifier); + // Two 5-byte chunks → 10 bytes total, under the 20-byte budget. + await writeChunks(guard, [ Buffer.alloc(5), Buffer.alloc(5) ]); + // getSize is called once by getAvailableSpace (for the overwritten + // resource) and must NOT be called again per chunk. + expect(reporter.getSize).toHaveBeenCalledTimes(1); + }); + + it('passes chunks through while the write stays under the available space.', async(): Promise => { + const reporter = mockReporter(10); + const strategy = new FixedTotalStrategy(90, limit, reporter); + const guard = await strategy.createQuotaGuard(identifier); + const received: Buffer[] = []; + guard.on('data', (chunk: Buffer): void => { + received.push(chunk); + }); + await writeChunks(guard, [ Buffer.alloc(5), Buffer.alloc(10) ]); + expect(received.reduce((sum, chunk): number => sum + chunk.length, 0)).toBe(15); + }); + + it('errors when the write exceeds the available space.', async(): Promise => { + const reporter = mockReporter(10); + const strategy = new FixedTotalStrategy(90, limit, reporter); + const guard = await strategy.createQuotaGuard(identifier); + // 25 bytes > 20 available. + await expect(writeChunks(guard, [ Buffer.alloc(25) ])).rejects.toThrow(/Quota exceeded/); + }); + + it('invalidates the reporter cache when the write completes.', async(): Promise => { + const reporter = mockReporter(10); + const strategy = new FixedTotalStrategy(90, limit, reporter); + const guard = await strategy.createQuotaGuard(identifier); + await writeChunks(guard, [ Buffer.alloc(5) ]); + expect(reporter.invalidate).toHaveBeenCalledTimes(1); + expect(reporter.invalidate).toHaveBeenLastCalledWith(identifier); + }); + + it('does not fail the write when cache invalidation fails.', async(): Promise => { + const reporter = mockReporter(10); + reporter.invalidate.mockRejectedValue(new Error('mapping failed')); + const strategy = new FixedTotalStrategy(90, limit, reporter); + const guard = await strategy.createQuotaGuard(identifier); + await expect(writeChunks(guard, [ Buffer.alloc(5) ])).resolves.toBeUndefined(); + }); + + it('never errors when the quota does not apply (infinite space).', async(): Promise => { + const reporter = mockReporter(0); + const strategy = new FixedTotalStrategy(Number.MAX_SAFE_INTEGER, limit, reporter); + const guard = await strategy.createQuotaGuard(identifier); + await expect(writeChunks(guard, [ Buffer.alloc(10_000) ])).resolves.toBeUndefined(); + }); +}); From 3eefb4b47ed0901cd53bef62fc67eda2e4294dc4 Mon Sep 17 00:00:00 2001 From: bourgeoa Date: Sat, 15 Aug 2026 19:14:28 +0200 Subject: [PATCH 2/2] docs: add pod storage quota design & verification (POD-STORAGE-QUOTA.md) Moved from docs/ (which is gitignored) to the repo root so it is tracked. Documents the problem (per-chunk pod walks), the A+B design (DuSizeReporter + FastQuotaStrategy), the decisions (apparent bytes, persist-per-write, A+B first), staleness/recovery (SS4.8), and the verification section SS7 (benchmark ~11 000x guard speedup + 12/12 byte-identical equivalence). --- POD-STORAGE-QUOTA.md | 380 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 380 insertions(+) create mode 100644 POD-STORAGE-QUOTA.md diff --git a/POD-STORAGE-QUOTA.md b/POD-STORAGE-QUOTA.md new file mode 100644 index 0000000..cbceae4 --- /dev/null +++ b/POD-STORAGE-QUOTA.md @@ -0,0 +1,380 @@ +# Pod Storage Quota — Performance Analysis & Design (2026-08-11) + +Pivot runs on the Community Solid Server (CSS) file backend with a per-pod quota +(`css:config/storage/backend/pod-quota-file.json`, enabled from `config/prod.json`). + +**Problem:** the quota is enforced by recursively walking the whole pod tree on +every write. With tens of thousands of files (e.g. a large inbox) this becomes +extremely slow and, under concurrent writes, fills memory and takes the server +down. Disabling the quota (config-only) removes the cost but loses the feature. + +--- + +## 1. How quota works today (CSS v7.x) + +Three moving parts: + +### 1.1 `QuotaValidator` (`dist/storage/validators/QuotaValidator.js`) +Runs on every write/PATCH pipeline and: +1. `getAvailableSpace()` **before** the write → full pod walk. +2. `createQuotaGuard()` — a streaming guard wrapped around the write body. +3. `getAvailableSpace()` **again** after the write (`afterWrite` flush). + +### 1.2 `QuotaStrategy.createQuotaGuard()` — the real killer +```js +async transform(chunk, enc, done) { + total += await reporter.calculateChunkSize(chunk); + const availableSpace = await that.getAvailableSpace(identifier); // ← FULL POD WALK, PER CHUNK + ... +} +``` +`getAvailableSpace()` → `getTotalSpaceUsed()` → `reporter.getSize(podRoot)` → +full recursive `FileSizeReporter.getTotalSize()` walk. So a single multi-chunk +upload walks the **entire pod once per stream chunk**. This is `chunks × O(N)`. + +### 1.3 `FileSizeReporter.getTotalSize()` (`dist/storage/size-reporter/FileSizeReporter.js`) +```js +if (stat.isFile()) return stat.size; +const childFiles = await fs.readdir(fileLocation); // one big dirent array +let totalSize = stat.size; +for (const current of childFiles) { + // skip ignoreFolders (e.g. /.internal/) + totalSize += await this.getTotalSize(childFileLocation); // recursion +} +``` +Sequential `stat` + `readdir` per entry — O(N) syscalls per walk, one large +dirent array per directory in the Node heap. + +### Cost profile +| | Per write | +|---|---| +| Pre-check | 1 full walk | +| Streaming guard | 1 full walk **per chunk** | +| Post-check | 1 full walk | + +Concurrent writes multiply the walks → memory exhaustion (dirent arrays + stat +results + in-flight async walks pile up in the Node heap). + +--- + +## 2. Backend impact (memory / database) + +CSS has **only one size reporter: `FileSizeReporter`** (filesystem-specific). +There is no memory or database reporter. + +| Backend | Quota wired? | Notes | +|---|---|---| +| **File** (`file.json`, `pod-quota-file.json`) | ✅ | The O(N)-per-write problem above. | +| **Memory** (`memory.json`, `MemoryDataAccessor`) | ❌ none | No quota at all. 10k-file inbox lives in RAM (inherent to backend). If quota wanted, size calc is trivial (sum contentLength in-memory) and an incremental counter needs no persistence. | +| **Database** (`sparql.json`) | ❌ none | No quota. "Size" is ambiguous (bytes vs quads); counter would be a table + delta queries. | + +The recursive-walk problem is specific to the **file backend** (what pivot uses). +Design C targets the file backend first, keeping the reporter/counter pluggable. + +--- + +## 3. Fix directions + +### A. Stop the per-chunk full walk (mandatory, lowest risk) +Compute `availableSpace` **once** before the stream; during streaming only track +the current write's byte delta and compare: + +```js +availableSpace = await strategy.getAvailableSpace(identifier); // once +transform(chunk) { + total += chunk.length; + if (availableSpace.amount < total) throw Quota exceeded; + push(chunk); +} +``` +Correct: `getAvailableSpace` already subtracts the overwritten resource's size, +and nothing else about the pod changes mid-write. Atomic writes go to +`/.internal/` (ignored by the walk) anyway. +Cost: `chunks × O(N)` → **~2 × O(N) per write** (pre + post). Kills the memory +blowup from concurrent chunk-streams. + +### B. Cache / memoize the pod size +Wrap `getSize(pod)` with a per-pod cache: +- **TTL variant:** cache `{ size, expiresAt }` (e.g. 1–5 s); one walk per expiry. +- **Invalidation variant:** drop the pod's cache entry when a write completes. +Combined with A, roughly **1 walk per pod per TTL window / per invalidation**. +Caveat: TTL window staleness (concurrent writes may exceed the limit undetected +within the window; CSS already documents tolerance of races). + +### C. Incremental per-pod counter (durable end-state) +Per-write O(1); one full walk only for bootstrap/recovery. See §4. + +--- + +## 4. Design C — incremental per-pod byte counter + +### 4.1 Counter store +- **In-memory:** `Map` — O(1) reads. +- **Persistence:** per-pod sidecar, e.g. `/.internal/pivot-quota.json` + (`.internal/` is already ignored by size accounting). Written **atomically** + (write temp + rename) on each write → crash-safe. + +### 4.2 Recount (bootstrap / recovery) +- First access to a pod with no valid counter → **one** full walk to seed it. +- Startup: load sidecar if present (no walk); else lazy recount on first access. +- Crash between write and persist: mark dirty / recount on next access. + +### 4.3 Delta hooks +A pivot store wrapper (like the existing `RdfPatchingStore`) around the backend: +- **Before:** `oldSize = accessor.getSize(identifier)` (O(1) single `stat`). +- Perform write/delete. +- **After:** `newSize = accessor.getSize(identifier)`. +- `Δ = newSize − oldSize` → `counter[pod] += Δ`. + +Cases: +- Create: `Δ = file size`. Overwrite: `Δ = new − old` (can be negative). +- Delete: `Δ = −oldSize`. Auxiliary writes (`.acl`/`.meta`): included (matches today). +- Atomic temp files: no special handling (rename exposes only the final file). + +### 4.4 Read path +`getSize(pod)` → `counter[pod].total` if valid, else recount. Plugs into the +**existing** `PodQuotaStrategy`/`QuotaValidator` unchanged — only +`urn:solid-server:default:SizeReporter` is replaced (plus the delta store +wrapper). No validator/strategy changes. + +### 4.5 Concurrency +- Per-pod mutex serializes `counter += Δ`. +- Sidecar atomic rename per write (cheap). Optional mode: persist periodically + + always recount on startup (simpler, costs a startup walk per pod). + +### 4.6 Edge cases / staleness +- Out-of-band file changes → stale counter. Mitigations: on-demand recount + endpoint, or periodic background recount. Documented limitation. +- Pod deletion → remove counter entry. + +### 4.7 Config wiring (pivot) +- New pivot components: `IncrementalSizeReporter` (counter + recount) and a + quota-delta store wrapper. +- Override `urn:solid-server:default:SizeReporter` and insert the wrapper in the + store chain via `pivot:config/storage/backend/...`, keeping + `pod-quota-file.json`'s validator/strategy. No CSS fork. + +### 4.8 Staleness detection & recovery (added 2026-08-15) + +The counter is **only an optimization — the filesystem is the source of +truth**, and `du` is a cheap way to re-derive truth. So a de-synchronized +counter is never permanent or catastrophic: it self-heals on next access. + +**Causes of de-sync** + +| Cause | How it happens | +|---|---| +| Out-of-band changes | Files added/removed directly on disk (admin, scripts, restore, sync tool) — bypasses the delta hook | +| Crash between write & persist | Data write lands, but the process dies before the sidecar rename — counter short by that one delta | +| Migration/bootstrap | Pods created before the feature → no sidecar (handled lazily at first access) | +| Auxiliary writes bypassing the hook | A code path (`.acl`/`.meta`, temp files, future store change) that forgets to report its delta | +| Manual tampering | Sidecar edited/deleted/restored from a backup without the pod | + +**Detection — cheap validity checks on read** (no walk required): + +1. `valid` flag / generation marker — sidecar records `{ total, valid, + version }`. Any path that can't guarantee a delta flips `valid: false`. +2. **Pod-root mtime comparison** — sidecar stores the pod root's + `lastRecordedMtime`; on read, one `stat` of the pod root — if newer than + `lastRecordedMtime`, the counter may be behind → invalidate → recount. + Catches out-of-band writes for the cost of a single `stat`. +3. **Sanity bound** — if `total` is wildly inconsistent with expectation + (e.g. 0 but the pod has files), treat as dirty. + +**Recovery — reuse the lazy path**: invalidation just means "delete/flag the +sidecar"; the existing lazy-bootstrap path does the rest — next access to +that pod runs one `du` and re-seeds. + +- On suspicion (mtime/flag/`valid:false`) → mark dirty → recount on next + access. One `du`, non-blocking. +- Crash window — atomic rename keeps the sidecar internally consistent (old + or new, never torn); add `fsync` before the rename completes the response, + shrinking the window. If a crash still lands in it, the pod-root mtime + check catches it on next access. +- Admin/on-demand recount — small CLI/HTTP/admin hook to force recount of one + pod or all pods (delete sidecars → next accesses recount, or an explicit + sweep). +- Optional background reconciliation — low-priority idle job walks pods with + `du` every few hours to bound drift over time. Never at startup, never + blocking writes. + +**Bottom line:** de-sync is detected cheaply (mtime/valid flag, one `stat`) +and fixed cheaply (`du` recount on next access). Worst persistent state is +"counter slightly behind until next access of that pod", which then +self-heals. + +--- + +## 5. Using `du` as the fast walk / recount primitive + +Instead of the Node recursive walk, shell out to GNU `du` — C-level `fts` +traversal: + +- **Speed:** 10–100× faster than the per-entry Node `stat`/`readdir` loop. +- **Memory:** the walk runs in a child process — dirent arrays / stat buffers + never touch the Node heap (directly fixes the memory blowup). +- **Unit:** `du -sb` = apparent bytes (sum of `st_size`) — identical semantics + to today's `FileSizeReporter` and the **chosen unit (2026-08-15)**: portable + across servers/filesystems and user-manageable. `du -s --block-size=1` = + disk usage — **rejected**: the result is server/filesystem-dependent + (cluster size, compression, COW), not user-manageable, and Solid pods are + small-file-heavy so cluster rounding would dominate. + +### Critical caveat +**Never call `du` per stream chunk** — spawning a process per chunk is a spawn +storm. `du` must be combined with fix **A** (walk only at pre/post check) and +ideally with **B**/**C** (walk only on recount). In design C, `du` is the +recount/seed engine only; steady-state writes are O(1) with zero spawns. + +### Practical concerns +| Concern | Handling | +|---|---| +| `ignoreFolders` (`.internal/`) | GNU `du --exclude=PATTERN` (or `--exclude-from`) — must match current behavior. | +| Path safety | `execFile('du', ['-sb', '--exclude', ..., podPath])` — never shell interpolation; CSS already containment-checks mapped paths. | +| Portability | `du` is coreutils/BSD — fine on Linux/WSL test servers, **not native on Windows**. Fall back to the Node walk when `du` is unavailable. | +| Symlinks/hardlinks | `du` doesn't follow symlinks by default (FileSizeReporter's `stat` does). Minor edge case — document. | +| Concurrency | With A+B/C, `du` spawns are rare → no process storm. | + +### Recommendation +- **Short/medium term:** `du` as the walk engine behind `getTotalSize`, **plus** + fix A (no per-chunk walk), **plus** a small TTL cache (B) → ~1 `du` spawn per + write. Small change, keeps the architecture, kills time + memory problems. +- **Long term:** same `du` as the recount engine inside **C** — incremental + counter, deltas per write, `du -sb` only to seed/repair. O(1) writes with + native-speed recovery. + +### Platform / flags + +**Performance: apparent-size vs disk-blocks — no meaningful difference.** +Both variants traverse the identical tree (same `readdir`/`stat` syscalls); +apparent size sums `st_size`, disk usage sums `st_blocks` (already in the +`stat` result). No extra syscalls either way → pick the unit purely on quota +semantics, not performance. + +`du` availability: + +| Platform | `du` | Flags | +|---|---|---| +| **Linux** (incl. WSL) | ✅ GNU coreutils | `-sb` (apparent bytes), `--block-size=1` / `-B 1` (disk bytes), `--exclude=PATTERN` | +| **macOS** | ✅ BSD `du` (always present) | **No GNU long options.** Apparent size: `-A`; bytes: `-B 1`; exclude: `-I pattern`; default block unit 512 B. So `du -s -A -B 1` ≈ GNU `du -sb`, `du -s -B 1` ≈ GNU `du -s --block-size=1` | +| **Windows** | ❌ **no native `du`** | None in cmd/PowerShell. Sysinternals `du.exe` (different CLI, not GNU-compatible), or GNU `du` via Git Bash / MSYS2 / Cygwin / WSL. PowerShell `Get-ChildItem -Recurse \| Measure-Object Length -Sum` is the slow JS-style walk | + +**Implementation note:** GNU-first, with probe & fallback: +1. Detect GNU vs BSD (`du --version` succeeds on GNU, fails on BSD → use + `-A`/`-B 1`/`-I`). +2. Fall back to the existing Node recursive walk when no compatible `du` is + found (e.g. bare Windows) — correct, just slower. + +### Unit consistency: apparent bytes everywhere (decided 2026-08-15) + +The chosen unit is **apparent bytes** (sum of `st_size` — same as today's +`FileSizeReporter`). This makes the unit fully portable: the same content +measures the same on any server/filesystem, and users can reason about it +("I used 500 MB of 1 GB"). Disk blocks (`st_blocks`) were considered and +rejected because the result is server-dependent (cluster size, compression, +COW) and small-file-heavy Solid pods would be dominated by cluster rounding. + +With apparent bytes there is **no platform inconsistency**: + +| Path | Unit | How | +|---|---|---| +| `du` (Linux/macOS) | apparent bytes | `du -sb` / BSD `du -s -A -B 1` | +| Node walk (Windows fallback) | apparent bytes | sum `stat.size` | + +Both paths sum `st_size` — identical quantity everywhere. (Verified: Node +`fs.stat` also exposes `blocks` on Windows, e.g. a 1000-byte file → +`blocks*512: 4096`, but we deliberately do NOT use it — that would introduce +server-dependent cluster rounding.) + +Edge cases: +- Sparse files: apparent bytes counts them at their logical size (matches + today; a disk-space quota would count less). +- Symlinks: `stat` follows symlinks (matches today's `FileSizeReporter`); + `du` doesn't by default — already noted as a minor edge case. + +--- + +## 6. Decision points before implementing + +### Decisions (2026-08-15) +1. **Unit: apparent bytes** (`du -sb` / sum of `st_size`) — portable across + servers/filesystems and **user-manageable**; identical semantics to + today's `FileSizeReporter`, so the existing 70 MB limit in + `customise-me.json` keeps its meaning. Disk blocks considered and + rejected (server/filesystem-dependent result; small-file-heavy pods). +2. **Persistence mode: persist-per-write** (atomic sidecar + `/.internal/pivot-quota.json`, exact across restarts). **Restart does + NOT trigger a bulk recount of all pods** — counters are loaded from disk; + the only full walks are lazy per-pod (first access of a pre-existing pod / + crash-dirty pod / de-synced pod — i.e. effectively *at login* for that + pod, never a startup sweep). +3. **Scope: A+B first** (quick win: no per-chunk walk + du-based walk with + TTL cache), then build C on top later. +4. **Staleness/recovery:** mtime/valid-flag check on read + lazy recount + + optional background reconciliation (see §4.8). + +### Remaining to verify before implementing +- Where before/after sizes are cleanest in the `AtomicFileDataAccessor` / + store stack (determines how much of C is new plumbing vs reusing existing + hooks). +- Exact `du` platform handling (GNU `--block-size=1` / BSD `-B 1 -s`, probe + & fallback to the Node walk; never per-chunk spawns). +- TTL window semantics with concurrent writes (CSS already tolerates race; + the `QuotaValidator` after-write flush check still catches breaches). + +--- + +## 7. Verification: benchmark + equivalence proof (2026-08-15) + +Two scripts in `scripts/` demonstrate the improvement and prove the size +calculation is unchanged. + +### 7.1 Benchmark — `scripts/benchmark-quota.js` + +Measures exactly what A+B optimizes: the pod walk and the per-write quota +guard. Run `node scripts/benchmark-quota.js [fileCount] [fileBytes]`. + +Results on a pod with **5 000 files × 1 KB** (body write = 4 MB in 64 KB +chunks): + +| Measure | Old (FileSizeReporter + QuotaStrategy) | New (DuSizeReporter + FastQuotaStrategy) | +|---|---|---| +| Full pod walk (`getSize`) | 669.6 ms | 657.8 ms first call; **0.13 ms cached** | +| Per-write quota guard | **36 279 ms** (≈36 s; full walk **per chunk**) | **3.3 ms** (walk once + cache) | +| Guard speedup | — | **~11 000×** | + +Notes: +- On bare Windows both paths use the Node walk (no `du`), so the walk line + shows ~1×; on Linux/WSL the `du` walk is 10–100× faster than the Node walk. +- The 36 s guard is the exact production bug: a single 4 MB upload into a + 5 000-file pod triggered 64 full pod walks. + +### 7.2 Equivalence — `scripts/verify-size-equivalence.js` + +Proves the new reporter returns **byte-for-byte identical** apparent-byte +totals to CSS's `FileSizeReporter`. Generates random pod trees (nested dirs, +random-size files, empty dirs, root-level `.internal` temp files) and asserts +equality for **both** the `du` path and the Node-walk fallback. + +Run `node scripts/verify-size-equivalence.js [iterations] [maxFiles]` +(with GNU `du` on PATH to exercise the real `du` path — e.g. Git Bash's +`C:\Program Files\Git\usr\bin` on Windows). + +Result: **12/12 random trees match exactly** on both paths. + +Findings / documented differences: +- **Node-walk fallback is mathematically identical** to `FileSizeReporter` + (same recursive `stat.size` sum) — always equivalent. +- **`du` path** sums the same `st_size` values (directory sizes included by + both) — equivalent. +- **Exclusion semantics caveat** (found by the proof): `du --exclude=.internal` + matches a `.internal` component at *any* depth, while CSS's anchored regex + `^/\.internal$` only excludes it at the *pod root*. Identical for real pods + (`.internal` is only ever created at the root — CSS temp files). A nested + `.internal` would be excluded by `du` but counted by the Node walk — a + documented, non-issue-in-practice difference. +- **Symlinks** (`du` doesn't follow by default; Node `stat` does) and + **hardlinks** (`du` counts once; Node per directory entry) differ — neither + occurs in normal Solid pod storage. + +---