diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0e90d7c6..4585577b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,8 +63,9 @@ jobs: retention-days: 3 if-no-files-found: ignore - # Storage targets (docs/storage-targets.md) against real servers: S3 - # (moto), WebDAV (wsgidav) and SMB (Samba with smbclient, as in the image). + # Storage targets (docs/storage-targets.md) and the off-site backup + # transports (services/offsite) against real servers: S3 (moto), WebDAV + # (wsgidav) and SMB (Samba with smbclient, as in the image). storage-targets: name: storage targets (S3, WebDAV, SMB) runs-on: ubuntu-latest @@ -108,8 +109,8 @@ jobs: done curl -sf -X PUT http://127.0.0.1:5055/gc-test-bucket - - name: Run the live storage target tests - run: node --test --test-force-exit tests/storage_targets_live.test.js + - name: Run the live storage target and off-site tests + run: node --test --test-force-exit tests/storage_targets_live.test.js tests/offsite_live.test.js env: NODE_ENV: test GC_TEST_S3_ENDPOINT: http://127.0.0.1:5055 diff --git a/docs/feature-release-b.md b/docs/feature-release-b.md index 7a8b2b31..b5771b44 100644 --- a/docs/feature-release-b.md +++ b/docs/feature-release-b.md @@ -177,6 +177,16 @@ Einstellungen `GET/PUT /api/v1/settings/waf`: password}`. Antworten enthalten nie Geheimnisse (`has_password: true`). - Upload nach jedem geplanten Backup an alle aktiven Ziele, Aufbewahrung `keep` (nur eigene Dateien `gatecontrol-*.gcbk` löschen), Ergebnis in `last_*`, SSE `backup`. +- Das Archiv ist nie als Ganzes im Speicher: `gcbk.writeBackupArchive` schreibt es als + Stream in eine Datei im privaten Ordner `/.incoming` (Name vom Server, + zufällig; vorher Prüfung des freien Platzes, sonst `NO_SPACE`/507, Hinweis `no_space`; + Löschen im `finally`, Reste eines Absturzes beim ersten Gebrauch). Die Transporte senden + aus der Datei (`services/offsite/source.js`): WebDAV ein PUT als Stream mit + Content-Length; S3 bis 64 MiB ein PUT, darüber Multipart in 64-MiB-Teilen (jeder Teil mit + eigenem SHA-256 signiert, beim Schreiben des Archivs berechnet; Abbruch per + AbortMultipartUpload bei einem Fehler); SMB/SFTP `put` direkt aus der Datei (Zeitlimit + wächst mit der Größe). Der Wiederherstellungstest lädt per `downloadTo` in eine Datei. + Messung: `node scripts/offsite-memory-check.js --mb 400`. ## 8. Navigation und Schnellsuche (UI B4) diff --git a/docs/guide/02-server-setup.md b/docs/guide/02-server-setup.md index 53f394a9..3e1b0ea8 100644 --- a/docs/guide/02-server-setup.md +++ b/docs/guide/02-server-setup.md @@ -244,6 +244,10 @@ backup. For a NAS at home behind a gateway, choose “Through a VPN device” or target; “Take over a storage target from Drive/plugins” fills in its settings. See [FAQ: Off-site backup to a NAS behind the VPN](07-faq.md#off-site-backup-to-a-nas-behind-the-vpn). +The encrypted copy is stored briefly on the server's disk next to the +backups while it is uploaded (it is never held in memory, so a small VPS is +fine) – plan free space for about one more backup. If it is missing, +GateControl uploads nothing and says so at the target. The connection is always encrypted, without any certificate file: diff --git a/docs/handbuch/02-server-einrichten.md b/docs/handbuch/02-server-einrichten.md index efe0b0e2..e6021ce3 100644 --- a/docs/handbuch/02-server-einrichten.md +++ b/docs/handbuch/02-server-einrichten.md @@ -245,6 +245,10 @@ S3 oder WebDAV hoch. Steht die NAS zu Hause hinter einem Gateway, wähle bei beim Speicherziel für Drive; „Speicherziel von Drive/Plugins übernehmen“ trägt dessen Einstellungen ein. Siehe [FAQ: Offsite-Backup auf eine NAS hinter dem VPN](07-faq.md#offsite-backup-auf-eine-nas-hinter-dem-vpn). +Die verschlüsselte Kopie liegt während des Hochladens kurz auf der Platte +des Servers neben den Backups (nie als Ganzes im Arbeitsspeicher, ein kleiner +VPS reicht also) – plane Platz für etwa ein Backup mehr ein. Fehlt er, lädt +GateControl nichts hoch und sagt es beim Ziel. Die Verbindung ist immer verschlüsselt, ohne dass du eine Zertifikatsdatei brauchst: diff --git a/public/js/ops-ui.js b/public/js/ops-ui.js index dca229c5..e1432813 100644 --- a/public/js/ops-ui.js +++ b/public/js/ops-ui.js @@ -123,6 +123,7 @@ INVALID_FINGERPRINT: ['offsite.err.generic', 'That did not work.'], NOT_GCBK: ['offsite.err.corrupt', 'The file is damaged or not a GateControl backup.'], UNSUPPORTED: ['offsite.err.unsupported', 'This version cannot read that archive format.'], + NO_SPACE: ['offsite.err.no_space', 'There is not enough free disk space on the GateControl server to store the encrypted archive temporarily.'], INVALID_WINDOW: ['autoupdate.err.invalid_window', 'Invalid maintenance window: times as HH:MM, start ≠ end, a valid time zone.'], INVALID_MODE: ['autoupdate.err.generic', 'Could not save.'], INVALID_NOTIFY_EMAIL: ['autoupdate.err.generic', 'Could not save.'], @@ -151,6 +152,8 @@ cert_not_yet_valid: ['offsite.hint.cert_not_yet_valid', 'The NAS certificate is not valid yet. Check the clock of the NAS and of GateControl.'], cert_decrypts: ['offsite.hint.cert_decrypts', 'The route decrypts the connection itself. Set the TLS mode of the L4 route to pass-through.'], smb_no_encryption: ['offsite.hint.smb_no_encryption', 'The share cannot encrypt (SMB3). On Synology: Control Panel › File Services › SMB › Advanced → SMB3 encryption. Or switch off “Encrypt connection”.'], + // the server's own disk: the archive is stored there briefly before the upload + no_space: ['offsite.hint.no_space', 'There is too little free disk space on the GateControl server. The encrypted archive is stored there briefly before the upload. Delete old local backups (Settings › Backup) or free up space.'], }; /** The hint of an API answer / a target row, '' when none. */ function hintText(code) { diff --git a/scripts/offsite-memory-check.js b/scripts/offsite-memory-check.js new file mode 100644 index 00000000..89c1cf80 --- /dev/null +++ b/scripts/offsite-memory-check.js @@ -0,0 +1,149 @@ +#!/usr/bin/env node +'use strict'; + +// Peak memory of an off-site backup upload (services/offsite) — the archive +// written as a file and sent from read streams, against holding it as one +// Buffer as up to 1.258.1 ("--legacy"). +// +// node scripts/offsite-memory-check.js [--mb 400] [--legacy] [--part-mb 16] +// +// Writes (in a child process) a backup file with MB of incompressible plugin data (as +// base64, as in a real backup) into a temp folder, then builds the archive +// and uploads it to a WebDAV and an S3 server running in this process (both +// only count and hash what arrives — the S3 one with a multipart upload +// above --part-mb). RSS is sampled every 10 ms from the start of the build +// to the end of the uploads. Prints one JSON line: +// { mode, plugin_mb, archive_mb, baseline_rss_mb, peak_rss_mb, extra_mb, max_rss_mb, ok } +// extra_mb = peak − baseline: what the archive transfer added. +// +// tests/offsite_memory.test.js runs it with a smaller size and checks the +// bound; the numbers for the PR come from a run with --mb 400. + +process.env.NODE_ENV = process.env.NODE_ENV || 'test'; +process.env.GC_LOG_LEVEL = process.env.GC_LOG_LEVEL || 'silent'; + +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const http = require('node:http'); +const crypto = require('node:crypto'); + +const args = process.argv.slice(2); +const opt = (name, dflt) => { const i = args.indexOf(`--${name}`); return i === -1 ? dflt : Number(args[i + 1]); }; +const MB = 1024 * 1024; +const PLUGIN_MB = opt('mb', 400); +const PART_MB = opt('part-mb', 16); +const LEGACY = args.includes('--legacy'); + +const gcbk = require('../src/services/offsite/gcbk'); +const webdav = require('../src/services/offsite/webdav'); +const s3 = require('../src/services/offsite/s3'); + +const rssMb = () => process.memoryUsage().rss / MB; + +/** The backup file, written piece by piece (its own RSS stays small). */ +async function writeBackup(file) { + const out = fs.createWriteStream(file); + const write = (s) => new Promise((resolve) => { if (out.write(s)) resolve(); else out.once('drain', resolve); }); + await write('{\n "version": 1,\n "data": {\n "settings": [],\n "plugins": [\n {\n "id": "big",\n "db": "'); + // base64 of random bytes in pieces of 3 MiB (a multiple of 3: the pieces join to one valid base64 text) + for (let left = PLUGIN_MB * MB; left > 0; left -= 3 * MB) await write(crypto.randomBytes(Math.min(left, 3 * MB)).toString('base64')); + await write('"\n }\n ]\n }\n}\n'); + await new Promise((resolve, reject) => out.end((err) => (err ? reject(err) : resolve()))); +} + +/** A server that takes WebDAV and S3 requests and only counts and hashes the bodies. */ +function sink() { + const got = { bytes: 0, objects: new Map() }; // name → { size, sha } + const uploads = new Map(); + const server = http.createServer((req, res) => { + const h = crypto.createHash('sha256'); + let n = 0; + req.on('data', (c) => { n += c.length; h.update(c); }); + req.on('end', () => { + got.bytes += n; + const u = new URL(req.url, 'http://x'); + const name = decodeURIComponent(u.pathname.split('/').pop()); + if (req.method === 'PROPFIND') { res.writeHead(207); res.end(''); return; } + if (req.method === 'POST' && u.searchParams.has('uploads')) { + const id = crypto.randomBytes(6).toString('hex'); + uploads.set(id, 0); + res.writeHead(200); res.end(`${id}`); return; + } + if (req.method === 'PUT' && u.searchParams.has('uploadId')) { + uploads.set(u.searchParams.get('uploadId'), uploads.get(u.searchParams.get('uploadId')) + n); + res.writeHead(200, { etag: `"${h.digest('hex').slice(0, 32)}"` }); res.end(); return; + } + if (req.method === 'POST' && u.searchParams.has('uploadId')) { + got.objects.set(name, { size: uploads.get(u.searchParams.get('uploadId')), sha: null }); + res.writeHead(200); res.end(''); return; + } + if (req.method === 'PUT') { got.objects.set(name, { size: n, sha: h.digest('hex') }); res.writeHead(201); res.end(); return; } + res.writeHead(204); res.end(); + }); + }); + return new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve({ server, got, base: `http://127.0.0.1:${server.address().port}` }))); +} + +async function main() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gc-offsite-mem-')); + const { server, got, base } = await sink(); + try { + const json = path.join(dir, 'gatecontrol-20261109-030000.json'); + // the input in a child process: its memory never shows in this one's RSS + require('node:child_process').execFileSync(process.execPath, [__filename, '--mb', String(PLUGIN_MB), '--write-input', json], { stdio: 'inherit' }); + const davCfg = { url: `${base}/dav/`, username: 'u', password: 'p' }; + const s3Cfg = { endpoint: base, region: 'us-east-1', bucket: 'bk', prefix: 'gc', access_key_id: 'AK', secret_access_key: 'SK', path_style: true }; + s3._limits.part = PART_MB * MB; + const opts = { passphrase: 'memory check passphrase', kdf: { N: 1024, r: 8, p: 1 }, gcVersion: 'mem' }; + + if (global.gc) global.gc(); + await new Promise((r) => setTimeout(r, 200)); + const baseline = rssMb(); + let peak = baseline; + const timer = setInterval(() => { peak = Math.max(peak, rssMb()); }, 10); + let archiveSize; + let sha; + try { + if (LEGACY) { + // up to 1.258.1: the archive as one Buffer, handed to every transport + const buf = await gcbk.encryptBackupFile(json, opts); + archiveSize = buf.length; + sha = crypto.createHash('sha256').update(buf).digest('hex'); + await webdav.upload(davCfg, 'dav.gcbk', buf); + await s3.upload(s3Cfg, 's3.gcbk', buf); + } else { + const file = path.join(dir, 'archive.gcbk'); + const r = await gcbk.writeBackupArchive(json, file, { ...opts, partSize: s3._limits.part }); + archiveSize = r.size; + const archive = { path: file, ...r }; + await webdav.upload(davCfg, 'dav.gcbk', archive); + await s3.upload(s3Cfg, 's3.gcbk', archive); + sha = crypto.createHash('sha256'); + for await (const c of fs.createReadStream(file)) sha.update(c); + sha = sha.digest('hex'); + } + } finally { clearInterval(timer); } + peak = Math.max(peak, rssMb()); + const dav = got.objects.get('dav.gcbk'); + const up = got.objects.get('s3.gcbk'); + const ok = !!dav && !!up && dav.size === archiveSize && up.size === archiveSize && dav.sha === sha; + const r1 = (n) => Math.round(n * 10) / 10; + process.stdout.write(JSON.stringify({ + mode: LEGACY ? 'legacy (Buffer)' : 'stream (file)', + plugin_mb: PLUGIN_MB, archive_mb: r1(archiveSize / MB), part_mb: PART_MB, + baseline_rss_mb: r1(baseline), peak_rss_mb: r1(peak), extra_mb: r1(peak - baseline), + max_rss_mb: r1(process.resourceUsage().maxRSS / 1024), ok, + }) + '\n'); + if (!ok) process.exitCode = 1; + } finally { + server.close(); + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +if (args.includes('--write-input')) { + writeBackup(args[args.indexOf('--write-input') + 1]).catch((err) => { process.stderr.write(`${err.stack || err}\n`); process.exit(1); }); +} else { + main().catch((err) => { process.stderr.write(`${err.stack || err}\n`); process.exit(1); }); +} diff --git a/src/i18n/de.json b/src/i18n/de.json index d9a6e754..8864f562 100644 --- a/src/i18n/de.json +++ b/src/i18n/de.json @@ -1830,6 +1830,7 @@ "offsite.err.cert_mismatch": "Die NAS zeigt inzwischen ein anderes Zertifikat. Bitte noch einmal testen.", "offsite.err.not_trustable": "Diesem Zertifikat kann hier nicht vertraut werden. „Testen“ zeigt, warum.", "offsite.err.unsupported": "Dieses Archivformat kann diese Version nicht lesen.", + "offsite.err.no_space": "Auf dem GateControl-Server ist nicht genug Speicherplatz frei, um das verschlüsselte Archiv zwischenzuspeichern.", "offsite.err.generic": "Das hat nicht geklappt.", "offsite.f_via": "Verbindung", "offsite.via_url": "Direkt zur Adresse", @@ -1902,6 +1903,7 @@ "offsite.hint.cert_not_yet_valid": "Das Zertifikat der NAS gilt noch nicht. Prüfe die Uhrzeit von NAS und GateControl.", "offsite.hint.cert_decrypts": "Die Weiterleitung entschlüsselt die Verbindung selbst. Stelle bei der L4-Weiterleitung den TLS-Modus auf „Durchleitung“ oder „Keiner“.", "offsite.hint.smb_no_encryption": "Die Freigabe kann keine Verschlüsselung (SMB3). Bei Synology: Systemsteuerung › Dateidienste › SMB › Erweitert → SMB3-Verschlüsselung. Oder unter „Bearbeiten“ „Verbindung verschlüsseln (SMB3)“ ausschalten.", + "offsite.hint.no_space": "Auf dem GateControl-Server ist zu wenig Speicherplatz frei. Das verschlüsselte Archiv wird vor dem Hochladen kurz dort abgelegt. Lösche alte lokale Backups (Einstellungen › Backup) oder gib Speicher frei.", "premig.title": "Sicherungen vor Datenbank-Updates", "premig.empty": "Noch keine – die erste entsteht beim nächsten Update mit Datenbank-Änderung.", "premig.versions": "Schema {from} → {to}", diff --git a/src/i18n/en.json b/src/i18n/en.json index 39e21d7d..f50280a5 100644 --- a/src/i18n/en.json +++ b/src/i18n/en.json @@ -1830,6 +1830,7 @@ "offsite.err.cert_mismatch": "The NAS now shows a different certificate. Please test again.", "offsite.err.not_trustable": "This certificate cannot be trusted here. “Test” shows why.", "offsite.err.unsupported": "This version cannot read that archive format.", + "offsite.err.no_space": "There is not enough free disk space on the GateControl server to store the encrypted archive temporarily.", "offsite.err.generic": "That did not work.", "offsite.f_via": "Connection", "offsite.via_url": "Directly to the address", @@ -1902,6 +1903,7 @@ "offsite.hint.cert_not_yet_valid": "The NAS certificate is not valid yet. Check the clock of the NAS and of GateControl.", "offsite.hint.cert_decrypts": "The route decrypts the connection itself. Set the TLS mode of the L4 route to “Passthrough” or “None”.", "offsite.hint.smb_no_encryption": "The share cannot encrypt (SMB3). On Synology: Control Panel › File Services › SMB › Advanced → SMB3 encryption. Or switch off “Encrypt connection (SMB3)” under “Edit”.", + "offsite.hint.no_space": "There is too little free disk space on the GateControl server. The encrypted archive is stored there briefly before the upload. Delete old local backups (Settings › Backup) or free up space.", "premig.title": "Snapshots before database updates", "premig.empty": "None yet — the first one is taken on the next update that changes the database.", "premig.versions": "Schema {from} → {to}", diff --git a/src/services/backupUpload.js b/src/services/backupUpload.js index 59383d7d..30973f6c 100644 --- a/src/services/backupUpload.js +++ b/src/services/backupUpload.js @@ -7,6 +7,9 @@ // An off-site archive (.gcbk) is decrypted into a second file next to it // (gcbk.decryptBackupFileTo). Both files stay until the request is done: // the large plugin parts of the parsed backup are still read from them. +// The off-site backups use the folder too (services/offsite/index.js): the +// archive of an upload (".offsite") and the one a restore test +// downloads (".verify"). const fs = require('node:fs'); const path = require('node:path'); @@ -49,7 +52,7 @@ function storage() { } // Names of the files in the folder: always made here (random hex), never taken from a request. -const NAME_RE = /^[0-9a-f]{24}\.(upload|verify)$/; +const NAME_RE = /^[0-9a-f]{24}\.(upload|verify|offsite)$/; /** A file of the incoming folder by its NAME (checked, below the folder) → absolute path; throws otherwise. */ function incomingPath(name) { diff --git a/src/services/offsite/gcbk.js b/src/services/offsite/gcbk.js index 96aad9e9..879f0e20 100644 --- a/src/services/offsite/gcbk.js +++ b/src/services/offsite/gcbk.js @@ -34,7 +34,7 @@ const { Transform } = require('node:stream'); // gzip in the thread pool. encryptBackup()/decryptBackup() work on one buffer // (small backups, tests); backup FILES go through the streaming -// encryptBackupFile() / decryptBackupFileTo() — a backup with plugin data has +// writeBackupArchive() / decryptBackupFileTo() — a backup with plugin data has // up to several hundred MB. const gzip = promisify(zlib.gzip); const gunzip = promisify(zlib.gunzip); @@ -111,18 +111,25 @@ async function encryptBackup(backup, { passphrase, encryptionKey, gcVersion = nu /** * encryptBackup() for a backup FILE (services/autobackup writes them), as a - * stream: the file is read in chunks and spliced into the payload text as it - * is — never parsed, never stringified again — then gzip (thread pool) and - * AES-GCM run chunk by chunk. JSON.parse + JSON.stringify + one cipher pass - * over a few hundred MB blocked the event loop for seconds before. The - * payload is the same as encryptBackup() makes (the backup part keeps the - * whitespace of the file — gzip makes that free, JSON.parse ignores it). + * stream into the archive FILE `outFile` (new, 0600): the backup file is + * read in chunks and spliced into the payload text as it is — never parsed, + * never stringified again — then gzip (thread pool) and AES-GCM run chunk by + * chunk and the archive is written piece by piece. Neither the backup nor + * the archive is ever in memory as a whole (an archive with plugin data has + * a few hundred MB; services/offsite uploads it from the file). The payload + * is the same as encryptBackup() makes (the backup part keeps the whitespace + * of the file — gzip makes that free, JSON.parse ignores it). On any error + * `outFile` is removed. + * + * `partSize`: also the SHA-256 of every `partSize` bytes of the archive + * (the S3 transport signs its parts with them — no second read of the file). * * @param {string} file path of a JSON backup file - * @param {object} opts as encryptBackup() - * @returns {Promise} + * @param {string} outFile where the archive goes (must not exist) + * @param {object} opts as encryptBackup(), plus partSize + * @returns {Promise<{size: number, partSize: number|null, partHashes: string[]|null}>} */ -async function encryptBackupFile(file, { passphrase, encryptionKey, gcVersion = null, kdf = DEFAULT_KDF } = {}) { +async function writeBackupArchive(file, outFile, { passphrase, encryptionKey, gcVersion = null, kdf = DEFAULT_KDF, partSize = null } = {}) { if (typeof passphrase !== 'string' || passphrase.length < MIN_PASSPHRASE) { throw new GcbkError('PASSPHRASE_TOO_SHORT', `passphrase must have at least ${MIN_PASSPHRASE} characters`); } @@ -152,14 +159,62 @@ async function encryptBackupFile(file, { passphrase, encryptionKey, gcVersion = for await (const chunk of fs.createReadStream(file, { highWaterMark: 256 * 1024 })) yield chunk; yield tail; } - // into a temp file next to the backup, then read in one piece: the archive - // is in memory once (chunks + Buffer.concat held it twice) + const parts = partSize ? new PartHasher(partSize) : null; + let size = 0; + // everything written to the archive passes here: counted (and hashed) + const tap = (chunk) => { size += chunk.length; if (parts) parts.update(chunk); return chunk; }; + async function* archive(source) { + yield tap(aad); + for await (const chunk of source) yield tap(chunk); + yield tap(cipher.getAuthTag()); // after the last ciphertext: cipher.final() has run + } + try { + await pipeline(payload, zlib.createGzip(), cipher, archive, fs.createWriteStream(outFile, { flags: 'wx', mode: 0o600 })); + } catch (err) { + // EEXIST: someone else's file — never remove that + if (!(err && err.code === 'EEXIST')) await fs.promises.rm(outFile, { force: true }).catch(() => {}); + throw err; + } + return { size, partSize: parts ? partSize : null, partHashes: parts ? parts.digest() : null }; +} + +/** SHA-256 of every `partSize` bytes of a stream (the last part may be shorter). */ +class PartHasher { + constructor(partSize) { + if (!Number.isSafeInteger(partSize) || partSize < 1) throw new TypeError('partSize must be a positive integer'); + this.partSize = partSize; + this.hashes = []; + this.cur = null; + this.left = 0; + } + update(buf) { + for (let at = 0; at < buf.length;) { + if (!this.cur) { this.cur = crypto.createHash('sha256'); this.left = this.partSize; } + const n = Math.min(this.left, buf.length - at); + this.cur.update(buf.subarray(at, at + n)); + at += n; + this.left -= n; + if (this.left === 0) { this.hashes.push(this.cur.digest('hex')); this.cur = null; } + } + } + digest() { + if (this.cur) { this.hashes.push(this.cur.digest('hex')); this.cur = null; } + return this.hashes; + } +} + +/** + * writeBackupArchive() into a Buffer (tests, small backups): through a temp + * file next to the backup, read in one piece afterwards. + * + * @param {string} file path of a JSON backup file + * @param {object} opts as encryptBackup() + * @returns {Promise} + */ +async function encryptBackupFile(file, opts = {}) { const tmp = `${file}.${crypto.randomBytes(6).toString('hex')}.gcbk.tmp`; try { - const out = fs.createWriteStream(tmp, { flags: 'wx', mode: 0o600 }); - out.write(aad); - await pipeline(payload, zlib.createGzip(), cipher, out); - await fs.promises.appendFile(tmp, cipher.getAuthTag()); + await writeBackupArchive(file, tmp, { ...opts, partSize: null }); return await fs.promises.readFile(tmp); } finally { await fs.promises.rm(tmp, { force: true }).catch(() => {}); @@ -322,6 +377,8 @@ module.exports = { readHeader, encryptBackup, encryptBackupFile, + writeBackupArchive, + PartHasher, decryptBackup, decryptBackupFileTo, checkPayload, diff --git a/src/services/offsite/index.js b/src/services/offsite/index.js index e6e8d16a..25c0fa0e 100644 --- a/src/services/offsite/index.js +++ b/src/services/offsite/index.js @@ -371,6 +371,7 @@ function errorHint(text) { if (text.includes(trust.UNTRUSTED_TEXT)) return 'cert_untrusted'; if (text.includes(trust.NAME_TEXT)) return 'cert_name'; if (text.includes(require('./smb').NO_ENCRYPTION_TEXT)) return 'smb_no_encryption'; + if (text.includes(NO_SPACE_TEXT)) return 'no_space'; return null; } @@ -415,20 +416,50 @@ function withIncomplete(t) { return { ...t, last_incomplete: e && Array.isArray(e.plugins) && e.plugins.length ? e : null }; } -/** An archive (Buffer) → { data, archive, cleanup } via a temp file: decrypted and read in pieces. */ -async function readArchiveBuffer(buf, passphrase) { - if (!gcbk.isGcbk(buf)) throw new gcbk.GcbkError('NOT_GCBK', 'not a GCBK1 file'); +// ── Archive files ────────────────────────────────────────────────────────── +// An archive is never held in memory: it is written to a file in the backup +// disk's private incoming folder (backupUpload.incomingDir, 0700; a random +// name made here, checked to stay inside the folder) and the transports send +// it from there (source.js); the restore test downloads into such a file. +// The file is removed when the transfer is done (finally); one left over by a +// crash is removed on the first use in the next process. + +// the text of the error (last_error, logs); the page shows its own words (hint 'no_space') +const NO_SPACE_TEXT = 'not enough free disk space on this server for the off-site archive'; +let archivesSwept = false; + +async function archiveDir() { const up = require('../backupUpload'); const dir = up.incomingDir(); await fs.promises.mkdir(dir, { recursive: true, mode: 0o700 }); - const name = up.newName('verify'); - const file = up.incomingPath(name); - const cleanup = () => up.removeUpload(name); - try { - await fs.promises.writeFile(file, buf, { mode: 0o600, flag: 'wx' }); - const r = await up.readUpload(name, passphrase); - return { ...r, cleanup }; - } catch (err) { await cleanup(); throw err; } + try { await fs.promises.chmod(dir, 0o700); } catch { /* best effort */ } + if (!archivesSwept) { + // nothing of this process is in flight yet: an archive there is a leftover + archivesSwept = true; + try { + for (const n of await fs.promises.readdir(dir)) { + if (/^[0-9a-f]{24}\.(offsite|verify)(\.plain\.json)?$/.test(n)) await fs.promises.rm(path.join(dir, n), { force: true }); + } + } catch { /* best effort */ } + } + return dir; +} + +/** The disk is full (a check before, or ENOSPC while writing) → OffsiteError NO_SPACE (507, hint no_space). */ +function noSpace(err) { + const mb = (n) => Math.ceil(n / 1048576); + const detail = err && Number.isFinite(err.need) && Number.isFinite(err.free) ? `: about ${mb(err.need)} MB needed, ${mb(err.free)} MB free` : ''; + const e = new OffsiteError('NO_SPACE', NO_SPACE_TEXT + detail, 507); + e.hint = 'no_space'; + return e; +} +function isNoSpace(err) { return !!(err && (err.code === 'NO_SPACE' || err.code === 'ENOSPC' || err.code === 'EDQUOT')); } + +/** Free space for `bytes` (+ the backup's disk reserve) in the archive folder, else NO_SPACE. */ +async function checkArchiveSpace(dir, bytes) { + try { await require('../backup').checkFreeSpace(dir, bytes); } catch (err) { + throw isNoSpace(err) ? noSpace(err) : err; + } } function getTarget(id) { @@ -718,7 +749,7 @@ function remoteNameTime(name) { * POST /targets/:id/verify — restore test. * @returns {Promise<{file,size,created_at,gc_version,include_key,counts,warnings}>} * @throws {OffsiteError} NO_REMOTE_BACKUP | PASSPHRASE_NOT_SET | DECRYPT_FAILED - * | CORRUPT | TRANSPORT_FAILED | NOT_FOUND + * | CORRUPT | TRANSPORT_FAILED | NOT_FOUND | NO_SPACE */ async function verifyTarget(id) { const row = getRow(id); @@ -727,7 +758,7 @@ async function verifyTarget(id) { const passphrase = getPassphrase(); if (!passphrase) throw new OffsiteError('PASSPHRASE_NOT_SET', 'set the off-site passphrase first', 409); const { cfg, t, ctx } = transportFor(row); - if (typeof t.download !== 'function') throw new OffsiteError('TRANSPORT_FAILED', 'this target type cannot read archives back', 502); + if (typeof t.downloadTo !== 'function') throw new OffsiteError('TRANSPORT_FAILED', 'this target type cannot read archives back', 502); const listed = await withTrust(row, cfg, t, (c) => t.list(c, ctx)); const files = listed.value @@ -739,24 +770,36 @@ async function verifyTarget(id) { throw new OffsiteError('CORRUPT', `archive is larger than ${Math.round(MAX_VERIFY_BYTES / 1048576)} MB — not read back`, 400); } - const buf = await t.download(listed.cfg, newest.name, ctx); - if (!Buffer.isBuffer(buf) || buf.length === 0) throw new OffsiteError('CORRUPT', 'the archive came back empty', 400); - if (buf.length > MAX_VERIFY_BYTES) throw new OffsiteError('CORRUPT', 'archive too large', 400); - + // into a file, piece by piece (the archive is never in memory as a whole) + const up = require('../backupUpload'); + const dir = await archiveDir(); + await checkArchiveSpace(dir, Number(newest.size) || 0); + const fileName = up.newName('verify'); + const file = up.incomingPath(fileName); + let size; let opened; - try { - opened = await readArchiveBuffer(buf, passphrase); - } catch (err) { - if (err instanceof gcbk.GcbkError) { - const code = err.code === 'DECRYPT_FAILED' ? 'DECRYPT_FAILED' : 'CORRUPT'; - throw new OffsiteError(code, err.message, 400); - } - throw err; - } - const { archive } = opened; let counts; let incomplete; try { + try { + size = await t.downloadTo(listed.cfg, newest.name, file, ctx, { maxBytes: MAX_VERIFY_BYTES, size: Number(newest.size) || 0 }); + } catch (err) { + if (err && err.code === 'TOO_LARGE') throw new OffsiteError('CORRUPT', 'archive too large', 400); + if (isNoSpace(err)) throw noSpace(err); + throw err; + } + if (!size) throw new OffsiteError('CORRUPT', 'the archive came back empty', 400); + try { + if (!(await gcbk.isGcbkFile(file))) throw new gcbk.GcbkError('NOT_GCBK', 'not a GCBK1 file'); + opened = await up.readUpload(fileName, passphrase); + } catch (err) { + if (err instanceof gcbk.GcbkError) { + const code = err.code === 'DECRYPT_FAILED' ? 'DECRYPT_FAILED' : 'CORRUPT'; + throw new OffsiteError(code, err.message, 400); + } + if (isNoSpace(err)) throw noSpace(err); + throw err; + } const data = opened.data; const errors = require('../backup').validateBackup(data); if (errors.length > 0) { @@ -767,7 +810,8 @@ async function verifyTarget(id) { routes: summary.routes, peers: summary.peers, users: summary.users, settings: summary.settings, }; incomplete = summary.plugins_incomplete || []; - } finally { await opened.cleanup(); } + } finally { await up.removeUpload(fileName); } + const { archive } = opened; const includeKey = !!archive.payload.encryption_key; const gcVersion = archive.header.gc_version || archive.payload.gc_version || null; @@ -784,7 +828,7 @@ async function verifyTarget(id) { const result = { file: newest.name, - size: buf.length, + size, created_at: createdAt, gc_version: gcVersion, include_key: includeKey, @@ -793,7 +837,7 @@ async function verifyTarget(id) { plugins_incomplete: incomplete, }; recordVerify(row.id, warnings.length ? 'warning' : 'ok', result); - logger.info({ target: row.id, type: row.type, file: newest.name, size: buf.length, warnings: warnings.length }, + logger.info({ target: row.id, type: row.type, file: newest.name, size, warnings: warnings.length }, 'Off-site restore test passed'); publish(row.id, 'verified', { file: newest.name, verify_status: warnings.length ? 'warning' : 'ok' }); return result; @@ -833,8 +877,8 @@ async function applyRetention(row, cfg, t, ctx) { return doomed.length; } -/** Upload one archive buffer to one target (serialised per target). */ -async function uploadTo(id, name, buf) { +/** Upload one archive (buildArchive's file, source.js) to one target (serialised per target). */ +async function uploadTo(id, name, archive) { const prevRun = running.get(id) || Promise.resolve(); const job = prevRun.catch(() => { /* previous upload's failure was already reported to its own caller; only serialisation matters here */ }).then(async () => { const row = getRow(id); @@ -842,14 +886,15 @@ async function uploadTo(id, name, buf) { publish(row.id, 'running', { file: name }); try { const { cfg: stored, t, ctx } = transportFor(row); - const up = await withTrust(row, stored, t, (c) => t.upload(c, name, buf, ctx)); + // a fresh read of the file per attempt (withTrust may run it twice) + const up = await withTrust(row, stored, t, (c) => t.upload(c, name, archive, ctx)); const cfg = up.cfg; let deleted = 0; try { deleted = await applyRetention(row, cfg, t, ctx); } catch (err) { logger.warn({ target: row.id, err: shortError(err) }, 'Off-site retention failed (upload itself succeeded)'); } recordResult(row.id, 'ok', null); publish(row.id, 'ok', { file: name }); - logger.info({ target: row.id, type: row.type, file: name, size: buf.length, deleted }, 'Off-site backup uploaded'); + logger.info({ target: row.id, type: row.type, file: name, size: archive.size, deleted }, 'Off-site backup uploaded'); return { status: 'ok', file: name, deleted, ...(up.trusted ? { trusted: up.trusted } : {}) }; } catch (err) { const msg = shortError(err); @@ -874,18 +919,38 @@ function remoteNameFor(localFile) { return m ? `gatecontrol-${m[1]}.gcbk` : null; } -/** Encrypt a local JSON backup file into a GCBK1 buffer. */ -async function buildArchive(localFile) { +/** + * Encrypt a local JSON backup file into a GCBK1 archive FILE (see "Archive + * files" above), streamed from the backup file into the archive file — no + * JSON.parse/stringify, never the archive in memory. `s3`: also the SHA-256 + * of its S3 parts (an S3 target signs them; computed while writing). + * → { path, size, partSize, partHashes, cleanup () } — call cleanup() when done. + * Throws OffsiteError PASSPHRASE_NOT_SET, NO_SPACE (hint no_space). + */ +async function buildArchive(localFile, { s3 = false } = {}) { const passphrase = getPassphrase(); if (!passphrase) throw new OffsiteError('PASSPHRASE_NOT_SET', 'set the off-site passphrase first', 409); const { include_key: includeKey } = getOffsiteSettings(); const config = require('../../../config/default'); - // streamed from the file (no JSON.parse/stringify of a few hundred MB on the main thread) - return gcbk.encryptBackupFile(localFile, { - passphrase, - encryptionKey: includeKey ? config.encryption.key : undefined, - gcVersion: require('../../../package.json').version, - }); + const up = require('../backupUpload'); + const dir = await archiveDir(); + // gzip'd JSON: at most about as large as the backup file (base64 plugin data shrinks to ~3/4) + const { size: plain } = await fs.promises.stat(localFile); + await checkArchiveSpace(dir, Math.ceil(plain * 1.01) + 64 * 1024); + const file = up.incomingPath(up.newName('offsite')); + const cleanup = () => fs.promises.rm(file, { force: true }).catch((err) => logger.warn({ err: shortError(err) }, 'Off-site: archive file not removed')); + try { + const r = await gcbk.writeBackupArchive(localFile, file, { + passphrase, + encryptionKey: includeKey ? config.encryption.key : undefined, + gcVersion: require('../../../package.json').version, + partSize: s3 ? require('./s3')._limits.part : null, + }); + return { path: file, size: r.size, partSize: r.partSize, partHashes: r.partHashes, cleanup }; + } catch (err) { + await cleanup(); + throw isNoSpace(err) ? noSpace(err) : err; + } } function hasLicense() { @@ -897,25 +962,27 @@ function hasLicense() { * targets. Never throws (logged + recorded per target). */ async function uploadAfterBackup(localFile) { - const targets = getDb().prepare('SELECT id FROM backup_targets WHERE enabled = 1 ORDER BY id').all(); + const targets = getDb().prepare('SELECT id, type FROM backup_targets WHERE enabled = 1 ORDER BY id').all(); if (targets.length === 0) return { uploaded: 0, failed: 0, skipped: 'no_targets' }; if (!hasLicense()) return { uploaded: 0, failed: 0, skipped: 'unlicensed' }; const name = remoteNameFor(localFile); if (!name) return { uploaded: 0, failed: 0, skipped: 'bad_name' }; - let buf; + let archive; try { - buf = await buildArchive(localFile); + archive = await buildArchive(localFile, { s3: targets.some((t) => t.type === 's3') }); } catch (err) { const msg = shortError(err); - for (const t of targets) { recordResult(t.id, 'failed', msg); publish(t.id, 'failed', { error: msg }); } + for (const t of targets) { recordResult(t.id, 'failed', msg); publish(t.id, 'failed', { error: msg, ...(err.hint ? { hint: err.hint } : {}) }); } logger.warn({ err: msg }, 'Off-site backup skipped'); return { uploaded: 0, failed: targets.length, skipped: null }; } let uploaded = 0; let failed = 0; - for (const t of targets) { - try { await uploadTo(t.id, name, buf); uploaded++; await noteIncomplete(t.id, localFile); } catch { failed++; } - } + try { + for (const t of targets) { + try { await uploadTo(t.id, name, archive); uploaded++; await noteIncomplete(t.id, localFile); } catch { failed++; } + } + } finally { await archive.cleanup(); } return { uploaded, failed, skipped: null }; } @@ -938,8 +1005,9 @@ async function runTargetNow(id) { } if (files.length === 0) throw new OffsiteError('NO_LOCAL_BACKUP', 'no local backup available', 409); const localFile = path.join(autobackup.BACKUP_DIR, files[0].filename); - const buf = await buildArchive(localFile); - const r = await uploadTo(row.id, remoteNameFor(localFile), buf); + const archive = await buildArchive(localFile, { s3: row.type === 's3' }); + let r; + try { r = await uploadTo(row.id, remoteNameFor(localFile), archive); } finally { await archive.cleanup(); } return { ...r, plugins_incomplete: await noteIncomplete(row.id, localFile) }; } diff --git a/src/services/offsite/s3.js b/src/services/offsite/s3.js index 0506206c..df7614e7 100644 --- a/src/services/offsite/s3.js +++ b/src/services/offsite/s3.js @@ -20,13 +20,26 @@ // net.request (one checked, pinned address per request; TLS always // verified); a timeout is an idle timeout (no byte for that long), so a // large upload over a slow line is fine. +// +// Uploads come from the archive file (source.js) as streams with +// Content-Length, never as one buffer: up to PART_SIZE one PUT, above that a +// multipart upload of PART_SIZE parts read part by part from the file (each +// signed with its own SHA-256 — gcbk.writeBackupArchive computed them while +// writing the archive, otherwise the part is hashed from the file first), +// aborted when anything fails, so no half upload stays on the bucket. const crypto = require('node:crypto'); const net = require('../storageTargets/net'); const via = require('../storageTargets/via'); +const { sourceOf, hashFile, streamToFile, isLocalError } = require('./source'); const SERVICE = 's3'; const TIMEOUT_MS = 120 * 1000; +// one PUT up to this size, multipart above (S3: parts 5 MiB … 5 GiB, at most 10 000) +const PART_SIZE = 64 * 1024 * 1024; +const MAX_PARTS = 10000; +// the tests lower `part` (a multipart upload of a few MB) +const limits = { part: PART_SIZE }; function sha256hex(data) { return crypto.createHash('sha256').update(data).digest('hex'); @@ -153,7 +166,7 @@ async function sha256hexAsync(data) { } const MAX_ANSWER = 16 * 1024 * 1024; -const MAX_DOWNLOAD = 1024 * 1024 * 1024; // the restore test refuses more than 128 MB anyway +const MAX_DOWNLOAD = 1024 * 1024 * 1024; // a GateControl archive (index.js MAX_VERIFY_BYTES) /** A failed connection → Error code TRANSPORT; `sys`: the system's code (index.js reads the certificate after one). */ function transportError(method, err) { @@ -163,19 +176,25 @@ function transportError(method, err) { return e; } -/** Sign and send one request → { status, headers, stream } (the caller reads the body). */ -async function send(cfg, method, key, { query, body, headers = {}, timeoutMs = TIMEOUT_MS } = {}) { +/** + * Sign and send one request → { status, headers, stream } (the caller reads + * the body). `body`: a Buffer, or a Readable (`stream`) of `length` bytes + * whose SHA-256 is `payloadHash` — sent with Content-Length, never chunked. + */ +async function send(cfg, method, key, { query, body, stream = null, length = 0, payloadHash = null, headers = {}, timeoutMs = TIMEOUT_MS } = {}) { const url = objectUrl(cfg, key, query); - const payload = body || Buffer.alloc(0); + const payload = stream ? null : (body || Buffer.alloc(0)); const signed = signRequest({ - method, url, headers, payloadHash: await sha256hexAsync(payload), + method, url, headers, payloadHash: stream ? payloadHash : await sha256hexAsync(payload), region: cfg.region || 'us-east-1', accessKeyId: cfg.access_key_id, secretAccessKey: cfg.secret_access_key, }); + // the length of a stream: not signed (as net.request's for a Buffer), only sent + if (stream) signed['content-length'] = String(length); try { const dialer = via.webdavDialer(via.webdavViaOf(cfg), url); // Host (signed) is url.host, the same net.request sends return await net.request({ - method, url, headers: signed, body: method === 'PUT' ? payload : null, idleMs: timeoutMs, + method, url, headers: signed, body: stream || (method === 'PUT' || method === 'POST' ? payload : null), idleMs: timeoutMs, tlsCa: cfg.tls_ca || null, dial: dialer ? dialer() : null, }); } catch (err) { @@ -202,11 +221,81 @@ async function request(cfg, method, key, opts = {}) { return { status: res.status, text }; } -async function upload(cfg, name, buf) { - await request(cfg, 'PUT', prefixOf(cfg) + name, { - body: buf, - headers: { 'content-type': 'application/octet-stream' }, - }); +/** The SHA-256 of bytes start…end of an archive file: computed while it was written, else read from the file. */ +async function partHash(src, n, start, end, partSize) { + if (src.partSize === partSize && src.partHashes && src.partHashes.length === Math.ceil(src.size / partSize) && src.partHashes[n]) return src.partHashes[n]; + return hashFile(src.path, start, end); +} + +/** PUT bytes start…end of the archive file as one request (a whole object or one part). */ +async function putRange(cfg, key, src, { query, headers = {}, hash, start, end }) { + const stream = src.open(start, end); + try { + const res = await send(cfg, 'PUT', key, { query, headers, stream, length: end - start + 1, payloadHash: hash }); + let text = ''; + try { text = (await net.readBody(res.stream, MAX_ANSWER)).toString('utf8'); } catch (err) { + if (res.status < 300) throw transportError('PUT', err); + } + if (res.status >= 300) throw statusError('PUT', res.status, text); + return res; + } finally { stream.destroy(); } +} + +function xmlEscape(s) { return String(s).replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); } + +async function uploadMultipart(cfg, key, src) { + const partSize = Math.max(limits.part, Math.ceil(src.size / MAX_PARTS)); + const init = await request(cfg, 'POST', key, { query: { uploads: '' }, headers: { 'content-type': 'application/octet-stream' } }); + const uploadId = xmlValue(init.text, 'UploadId'); + if (!uploadId) { + const e = new Error('S3 POST: no UploadId in the answer'); + e.code = 'TRANSPORT'; + throw e; + } + try { + const parts = []; + for (let n = 1, start = 0; start < src.size; n++, start += partSize) { + const end = Math.min(src.size, start + partSize) - 1; + const hash = await partHash(src, n - 1, start, end, partSize); + const res = await putRange(cfg, key, src, { query: { partNumber: String(n), uploadId }, hash, start, end }); + const etag = res.headers.etag; + if (!etag) { + const e = new Error(`S3 PUT: no ETag for part ${n}`); + e.code = 'TRANSPORT'; + throw e; + } + parts.push({ n, etag }); + } + const xml = `${parts.map((p) => `${p.n}${xmlEscape(p.etag)}`).join('')}`; + // assembling the parts takes the server a while for a large object + const done = await request(cfg, 'POST', key, { query: { uploadId }, body: Buffer.from(xml, 'utf8'), headers: { 'content-type': 'application/xml' }, timeoutMs: 10 * 60 * 1000 }); + // S3 may answer 200 with an body when completing fails late + if (//.test(done.text)) throw statusError('POST', done.status, done.text); + } catch (err) { + try { await request(cfg, 'DELETE', key, { query: { uploadId }, timeoutMs: 30000 }); } catch { /* best effort: the bucket's lifecycle rule cleans up the rest */ } + throw err; + } +} + +/** + * Upload the archive (source.js: the archive file, or a Buffer): up to + * limits.part one PUT, above that a multipart upload — both streamed from + * the file with Content-Length. + */ +async function upload(cfg, name, body) { + const src = sourceOf(body); + const key = prefixOf(cfg) + name; + const headers = { 'content-type': 'application/octet-stream' }; + if (src.buffer) { + await request(cfg, 'PUT', key, { body: src.buffer, headers }); + return; + } + if (src.size > limits.part) { + await uploadMultipart(cfg, key, src); + return; + } + const hash = await partHash(src, 0, 0, src.size - 1, limits.part); + await putRange(cfg, key, src, { headers, hash, start: 0, end: src.size - 1 }); } async function list(cfg) { @@ -231,8 +320,12 @@ async function list(cfg) { return files; } -/** Read one object back (restore test) as a Buffer. Nothing is written on the remote side. */ -async function download(cfg, name) { +/** + * Read one object back (restore test) into the local `file` (new, 0600), + * piece by piece. Nothing is written on the remote side. → bytes. On an + * error the caller removes `file`. + */ +async function downloadTo(cfg, name, file, ctx, { maxBytes = MAX_DOWNLOAD } = {}) { const res = await send(cfg, 'GET', prefixOf(cfg) + name); if (res.status >= 300) { let text = ''; @@ -241,7 +334,10 @@ async function download(cfg, name) { e.message = e.message.replace(/: .*$/, ''); throw e; } - try { return await net.readBody(res.stream, MAX_DOWNLOAD); } catch (err) { throw transportError('GET', err); } + try { return await streamToFile(res.stream, file, maxBytes); } catch (err) { + if (err && (err.code === 'TOO_LARGE' || isLocalError(err))) throw err; + throw transportError('GET', err); + } } async function remove(cfg, name) { @@ -276,4 +372,4 @@ function route(cfg) { return { kind: v.kind, dial: dialer || (() => ({ host: url.hostname, port: Number(url.port) || 443 })) }; } -module.exports = { signRequest, uriEncode, objectUrl, upload, download, list, remove, test, inspectTls, route, _sha256hex: sha256hex }; +module.exports = { signRequest, uriEncode, objectUrl, upload, downloadTo, list, remove, test, inspectTls, route, PART_SIZE, _limits: limits, _sha256hex: sha256hex }; diff --git a/src/services/offsite/sftp.js b/src/services/offsite/sftp.js index f2949031..61fbbfdc 100644 --- a/src/services/offsite/sftp.js +++ b/src/services/offsite/sftp.js @@ -12,11 +12,15 @@ // /offsite/known_hosts; a changed key fails with a clear message and // is reset when the target's host/port changes or the target is deleted. // Uploads go to ".part" and are renamed afterwards, so an interrupted -// upload never counts as a backup for the retention. +// upload never counts as a backup for the retention. The archive goes up from +// its file (source.js): `put` streams it — nothing of it passes through +// GateControl's memory; the time limit grows with its size. +const fs = require('node:fs'); const { run, writeTemp, writeTempAsync, rmQuiet, workDir } = require('./proc'); const path = require('node:path'); const via = require('../storageTargets/via'); +const { sourceOf, localPath, transferTimeout, TooLarge } = require('./source'); const TIMEOUT_MS = 180 * 1000; @@ -124,17 +128,19 @@ function parseLs(out) { return files; } -async function upload(cfg, name, buf, ctx) { - const local = await writeTempAsync('upload', buf); +/** Upload the archive (source.js: its file as it is, a Buffer through a temp file) as ".part", then rename. */ +async function upload(cfg, name, body, ctx) { + const src = sourceOf(body); + const local = src.path ? localPath(src.path, 'SFTP') : await writeTempAsync('upload', src.buffer); try { await batch(cfg, ctx, [ ...cdScript(cfg.path), `-rm ${q(name + '.part')}`, `put ${q(local)} ${q(name + '.part')}`, `rename ${q(name + '.part')} ${q(name)}`, - ]); + ], transferTimeout(TIMEOUT_MS, src.size)); } finally { - rmQuiet(local); + if (!src.path) rmQuiet(local); } } @@ -143,15 +149,19 @@ async function list(cfg, ctx) { return parseLs(out); } -/** Read one file back (restore test) into a Buffer; the remote side is read-only here. */ -async function download(cfg, name, ctx) { - const local = writeTemp('download', ''); - try { - await batch(cfg, ctx, [...cdScript(cfg.path), `get ${q(name)} ${q(local)}`]); - return await require('node:fs').promises.readFile(local); - } finally { - rmQuiet(local); - } +/** + * Read one file back (restore test) into the local `file` (created here, + * 0600; sftp writes it in chunks); the remote side is read-only here. + * `size`: what the listing said (the time limit). → bytes. On an error the + * caller removes `file`. + */ +async function downloadTo(cfg, name, file, ctx, { maxBytes = Infinity, size = 0 } = {}) { + const local = localPath(file, 'SFTP'); + await fs.promises.writeFile(local, '', { flag: 'wx', mode: 0o600 }); + await batch(cfg, ctx, [...cdScript(cfg.path), `get ${q(name)} ${q(local)}`], transferTimeout(TIMEOUT_MS, size)); + const bytes = (await fs.promises.stat(local)).size; + if (bytes > maxBytes) throw new TooLarge(maxBytes); + return bytes; } async function remove(cfg, name, ctx) { @@ -191,4 +201,4 @@ async function forgetHost(cfg) { } catch { /* no ssh-keygen / no file — nothing to forget */ } } -module.exports = { upload, download, list, remove, test, forgetHost, parseLs, cdScript, sshOptions }; +module.exports = { upload, downloadTo, list, remove, test, forgetHost, parseLs, cdScript, sshOptions }; diff --git a/src/services/offsite/smb.js b/src/services/offsite/smb.js index 60e80fbf..c499e075 100644 --- a/src/services/offsite/smb.js +++ b/src/services/offsite/smb.js @@ -12,12 +12,17 @@ // a VPN peer (its VPN address, `via.port`) or an L4 route (what Caddy dials // for it) — smbclient then connects to that address (-I, -p) while the // share is still named after `host` (or the address, when no host is set). +// The archive goes up from its file (source.js): smbclient `put` reads and +// writes it in chunks itself — nothing of it passes through GateControl's +// memory; the time limit grows with its size. // Credentials go through a temporary authentication file (-A, mode 0600, // deleted afterwards) — never on the command line, where every process on // the host could read them from /proc. +const fs = require('node:fs'); const { run, writeTemp, writeTempAsync, rmQuiet } = require('./proc'); const via = require('../storageTargets/via'); +const { sourceOf, localPath, transferTimeout, TooLarge } = require('./source'); const { encryptArgs, noEncryption } = require('../storageTargets/drivers/smb'); // English, for logs, last_error and API callers; the page has its own text (hint 'smb_no_encryption') @@ -131,13 +136,16 @@ function parseLs(out) { return files; } -async function upload(cfg, name, buf) { - const local = await writeTempAsync('upload', buf); +/** Upload the archive (source.js: its file as it is, a Buffer through a temp file) as ".part", then rename. */ +async function upload(cfg, name, body) { + const src = sourceOf(body); + const local = src.path ? localPath(src.path, 'SMB') : await writeTempAsync('upload', src.buffer); try { await ensurePath(cfg); - await smb(cfg, [...cd(cfg), `put ${q(local)} ${q(name + '.part')}`, `rename ${q(name + '.part')} ${q(name)}`]); + await smb(cfg, [...cd(cfg), `put ${q(local)} ${q(name + '.part')}`, `rename ${q(name + '.part')} ${q(name)}`], + { timeoutMs: transferTimeout(TIMEOUT_MS, src.size) }); } finally { - rmQuiet(local); + if (!src.path) rmQuiet(local); } } @@ -146,15 +154,19 @@ async function list(cfg) { return parseLs(res.stdout); } -/** Read one file back (restore test) into a Buffer; nothing on the share changes. */ -async function download(cfg, name) { - const local = writeTemp('download', ''); - try { - await smb(cfg, [...cd(cfg), `get ${q(name)} ${q(local)}`]); - return await require('node:fs').promises.readFile(local); - } finally { - rmQuiet(local); - } +/** + * Read one file back (restore test) into the local `file` (created here, + * 0600; smbclient writes it in chunks); nothing on the share changes. + * `size`: what the listing said (the time limit). → bytes. On an error the + * caller removes `file`. + */ +async function downloadTo(cfg, name, file, ctx, { maxBytes = Infinity, size = 0 } = {}) { + const local = localPath(file, 'SMB'); + await fs.promises.writeFile(local, '', { flag: 'wx', mode: 0o600 }); + await smb(cfg, [...cd(cfg), `get ${q(name)} ${q(local)}`], { timeoutMs: transferTimeout(TIMEOUT_MS, size) }); + const bytes = (await fs.promises.stat(local)).size; + if (bytes > maxBytes) throw new TooLarge(maxBytes); + return bytes; } async function remove(cfg, name) { @@ -174,4 +186,4 @@ async function test(cfg) { } } -module.exports = { upload, download, list, remove, test, parseLs, baseArgs, explain, NO_ENCRYPTION_TEXT }; +module.exports = { upload, downloadTo, list, remove, test, parseLs, baseArgs, explain, NO_ENCRYPTION_TEXT }; diff --git a/src/services/offsite/source.js b/src/services/offsite/source.js new file mode 100644 index 00000000..4eff123f --- /dev/null +++ b/src/services/offsite/source.js @@ -0,0 +1,96 @@ +'use strict'; + +// What an off-site transport uploads (upload(cfg, name, body)): the archive +// FILE that services/offsite/index.js builds — { path, size, partSize?, +// partHashes? } (gcbk.writeBackupArchive) — or a Buffer (the write probes of +// test(), small uploads in tests). A file is never read into memory: the +// transports send it from read streams (one per request, so a retry after a +// certificate was trusted starts from the first byte again) or hand its path +// to smbclient / sftp, which read it in chunks themselves. + +const fs = require('node:fs'); +const crypto = require('node:crypto'); +const { Transform } = require('node:stream'); +const { pipeline } = require('node:stream/promises'); + +const READ_CHUNK = 256 * 1024; + +/** + * → { size, buffer (Buffer | null), path (string | null), partSize, partHashes, + * open(start, end) → Readable of bytes start…end (inclusive) } + */ +function sourceOf(body) { + if (Buffer.isBuffer(body)) { + return { size: body.length, buffer: body, path: null, partSize: null, partHashes: null, open: null }; + } + if (body && typeof body.path === 'string' && Number.isSafeInteger(body.size) && body.size > 0) { + return { + size: body.size, + buffer: null, + path: body.path, + partSize: Number.isSafeInteger(body.partSize) ? body.partSize : null, + partHashes: Array.isArray(body.partHashes) ? body.partHashes : null, + open: (start = 0, end = body.size - 1) => fs.createReadStream(body.path, { start, end, highWaterMark: READ_CHUNK }), + }; + } + throw new TypeError('upload: a Buffer or an archive file { path, size }'); +} + +/** SHA-256 (hex) of bytes start…end (inclusive) of a file, read in chunks. */ +async function hashFile(file, start, end) { + const h = crypto.createHash('sha256'); + for await (const c of fs.createReadStream(file, { start, end, highWaterMark: READ_CHUNK })) h.update(c); + return h.digest('hex'); +} + +// a local path handed to smbclient (-c "…; put "" …") or to an sftp +// batch line ("put "" …"): no quote, backslash, ";" or control character +const LOCAL_PATH_RE = /^\/[^"\\;\u0000-\u001f\u007f]*$/; + +/** A local file path the smbclient / sftp command lines can carry; throws code TRANSPORT otherwise. */ +function localPath(p, what) { + if (typeof p !== 'string' || !LOCAL_PATH_RE.test(p)) { + const e = new Error(`${what}: the local path of the archive contains characters the transfer cannot pass on`); + e.code = 'TRANSPORT'; + throw e; + } + return p; +} + +// errors of this server's own disk while writing a download — never the target's fault +const LOCAL_CODES = new Set(['ENOSPC', 'EDQUOT', 'EEXIST', 'EACCES', 'EPERM', 'EROFS']); +function isLocalError(err) { return !!(err && LOCAL_CODES.has(err.code)); } + +class TooLarge extends Error { + constructor(max) { super(`the archive is larger than ${Math.round(max / 1048576)} MB`); this.code = 'TOO_LARGE'; } +} + +/** + * Write a response body to `file` (new, 0600), at most `maxBytes` — piece by + * piece, never the whole body in memory. → bytes written. On an error the + * file stays for the caller to remove (it owns the name). + */ +async function streamToFile(stream, file, maxBytes) { + let n = 0; + const limit = new Transform({ + transform(chunk, enc, cb) { + n += chunk.length; + if (n > maxBytes) cb(new TooLarge(maxBytes)); + else cb(null, chunk); + }, + }); + await pipeline(stream, limit, fs.createWriteStream(file, { flags: 'wx', mode: 0o600 })); + return n; +} + +/** + * Hard time limit of an smbclient / sftp transfer of `bytes`: the base limit + * plus what the slowest line still worth waiting for (1 Mbit/s) needs — a + * few hundred MB over a home upload take longer than the old fixed 3 min. + */ +function transferTimeout(baseMs, bytes) { + const MIN_RATE = 128 * 1024; // bytes per second + return baseMs + Math.ceil((Number(bytes) || 0) / MIN_RATE) * 1000; +} + +module.exports = { sourceOf, hashFile, localPath, streamToFile, transferTimeout, isLocalError, TooLarge, READ_CHUNK }; diff --git a/src/services/offsite/webdav.js b/src/services/offsite/webdav.js index 0810dde0..2d1e45a3 100644 --- a/src/services/offsite/webdav.js +++ b/src/services/offsite/webdav.js @@ -18,9 +18,10 @@ const crypto = require('node:crypto'); const net = require('../storageTargets/net'); const via = require('../storageTargets/via'); +const { sourceOf, streamToFile, isLocalError } = require('./source'); const TIMEOUT_MS = 120 * 1000; -const MAX_DOWNLOAD = 1024 * 1024 * 1024; // the restore test refuses more than 128 MB anyway +const MAX_DOWNLOAD = 1024 * 1024 * 1024; // a GateControl archive (index.js MAX_VERIFY_BYTES) function collectionUrl(cfg) { const u = new URL(cfg.url); @@ -126,14 +127,24 @@ async function ensureCollection(cfg) { return false; } -async function upload(cfg, name, buf) { +/** + * PUT the archive (source.js: the archive file, streamed with + * Content-Length — never chunked, never in memory — or a Buffer). + */ +async function upload(cfg, name, body) { + const src = sourceOf(body); await ensureCollection(cfg); const url = new URL(encodeURIComponent(name), collectionUrl(cfg)); - await dav(cfg, 'PUT', url, { - body: buf, - headers: { 'content-type': 'application/octet-stream', 'content-length': String(buf.length) }, - timeoutMs: TIMEOUT_MS, - }); + const stream = src.buffer ? null : src.open(); + try { + await dav(cfg, 'PUT', url, { + body: src.buffer || stream, + headers: { 'content-type': 'application/octet-stream', 'content-length': String(src.size) }, + timeoutMs: TIMEOUT_MS, + }); + } finally { + if (stream) stream.destroy(); + } } async function list(cfg) { @@ -159,15 +170,22 @@ async function list(cfg) { return files; } -/** Read one file back (restore test). Returns a Buffer; never writes anything. */ -async function download(cfg, name) { +/** + * Read one file back (restore test) into the local `file` (new, 0600), piece + * by piece. Never writes anything on the target. → bytes. On an error the + * caller removes `file`. + */ +async function downloadTo(cfg, name, file, ctx, { maxBytes = MAX_DOWNLOAD } = {}) { const url = new URL(encodeURIComponent(name), collectionUrl(cfg)); const res = await send(cfg, 'GET', url, { timeoutMs: TIMEOUT_MS }); if (res.status < 200 || res.status >= 300) { await net.drain(res.stream); throw statusError('GET', res); } - try { return await net.readBody(res.stream, MAX_DOWNLOAD); } catch (err) { throw await transportError(cfg, 'GET', url, err); } + try { return await streamToFile(res.stream, file, maxBytes); } catch (err) { + if (err && (err.code === 'TOO_LARGE' || isLocalError(err))) throw err; + throw await transportError(cfg, 'GET', url, err); + } } async function remove(cfg, name) { @@ -205,4 +223,4 @@ function route(cfg) { return { kind: v.kind, dial: dialer || (() => ({ host: url.hostname, port: Number(url.port) || 443 })) }; } -module.exports = { upload, download, list, remove, test, inspectTls, route, parseMultistatus, collectionUrl }; +module.exports = { upload, downloadTo, list, remove, test, inspectTls, route, parseMultistatus, collectionUrl }; diff --git a/tests/backup_streaming.test.js b/tests/backup_streaming.test.js index e98c3a1b..d56570e1 100644 --- a/tests/backup_streaming.test.js +++ b/tests/backup_streaming.test.js @@ -204,6 +204,56 @@ describe('backup with several MB of plugin data', () => { fs.rmSync(bad); }); + it('the off-site archive written into a FILE (writeBackupArchive): size and part hashes counted while writing, decryptable piece by piece', async () => { + const gcbk = require('../src/services/offsite/gcbk'); + const kdf = { N: 1024, r: 8, p: 1 }; + const passphrase = 'correct horse battery'; + const dir = path.dirname(file); + const out = path.join(dir, 'archive.gcbk'); + const partSize = 256 * 1024; + const { result: r, maxGap } = await whileTicking(() => gcbk.writeBackupArchive(file, out, { passphrase, kdf, gcVersion: '9.9.9', partSize })); + assert.ok(maxGap < MAX_TICK_GAP_MS, `max timer gap ${maxGap} ms`); + // one open file for every look at it (no check-then-use on the path) + const fd = fs.openSync(out, 'r'); + const content = () => { const b = Buffer.alloc(fs.fstatSync(fd).size); fs.readSync(fd, b, 0, b.length, 0); return b; }; + const bytes = content(); + assert.equal(r.size, bytes.length); + assert.equal(r.partSize, partSize); + assert.equal((fs.fstatSync(fd).mode & 0o777).toString(8), '600'); + assert.equal(r.partHashes.length, Math.ceil(bytes.length / partSize)); + r.partHashes.forEach((h, i) => assert.equal(h, crypto.createHash('sha256').update(bytes.subarray(i * partSize, (i + 1) * partSize)).digest('hex'), `part ${i}`)); + const plain = path.join(dir, 'archive.plain.json'); + await gcbk.decryptBackupFileTo(out, passphrase, plain); + const payload = gcbk.checkPayload(JSON.parse(fs.readFileSync(plain, 'utf8'))); + assert.deepEqual(payload.backup, parsed); + fs.rmSync(plain); + // without partSize: no hashes + const out2 = path.join(dir, 'archive2.gcbk'); + const r2 = await gcbk.writeBackupArchive(file, out2, { passphrase, kdf }); + assert.equal(r2.partHashes, null); + assert.equal(r2.size, fs.readFileSync(out2).length); + fs.rmSync(out2); + // an existing file is never overwritten — nor removed + await assert.rejects(gcbk.writeBackupArchive(file, out, { passphrase, kdf }), { code: 'EEXIST' }); + assert.ok(content().equals(bytes)); + fs.closeSync(fd); + fs.rmSync(out); + // a failure while writing (the disk refuses after the first bytes): nothing half-written stays + const realCreate = fs.createWriteStream; + const out3 = path.join(dir, 'archive3.gcbk'); + fs.createWriteStream = (p, o) => { + const w = realCreate(p, o); + let n = 0; + const write = w._write.bind(w); + w._write = (chunk, enc, cb) => { n += chunk.length; if (n > 64 * 1024) cb(Object.assign(new Error('no space left on device'), { code: 'ENOSPC' })); else write(chunk, enc, cb); }; + return w; + }; + try { + await assert.rejects(gcbk.writeBackupArchive(file, out3, { passphrase, kdf }), { code: 'ENOSPC' }); + } finally { fs.createWriteStream = realCreate; } + assert.equal(fs.existsSync(out3), false); + }); + it('without a usable temp folder the plugin database is still in the backup (one-piece copy)', async () => { const prev = process.env.TMPDIR; process.env.TMPDIR = path.join(path.dirname(file), 'missing', 'tmp'); diff --git a/tests/helpers/offsiteFiles.js b/tests/helpers/offsiteFiles.js new file mode 100644 index 00000000..c84a913e --- /dev/null +++ b/tests/helpers/offsiteFiles.js @@ -0,0 +1,41 @@ +'use strict'; + +// The off-site transports upload archive FILES ({ path, size }, see +// src/services/offsite/source.js) and download into files (downloadTo) — +// helpers for tests that want Buffers on their side. + +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const crypto = require('node:crypto'); + +let dir = null; +function tmpDir() { + if (!dir) { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gc-offsite-files-')); + process.on('exit', () => { try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* gone */ } }); + } + return dir; +} + +function tmpName(kind) { return path.join(tmpDir(), `${kind}-${crypto.randomBytes(6).toString('hex')}`); } + +/** A Buffer as an archive file for upload(): { path, size, ...extra }. */ +function archiveFile(buf, extra = {}) { + const p = tmpName('archive'); + fs.writeFileSync(p, buf, { mode: 0o600 }); + return { path: p, size: buf.length, ...extra }; +} + +/** downloadTo() into a temp file → its content as a Buffer (the file is removed). */ +async function downloadBuffer(transport, cfg, name, ctx, opts) { + const p = tmpName('download'); + try { + const n = await transport.downloadTo(cfg, name, p, ctx, opts); + const buf = fs.readFileSync(p); + if (n !== buf.length) throw new Error(`downloadTo said ${n} bytes, the file has ${buf.length}`); + return buf; + } finally { fs.rmSync(p, { force: true }); } +} + +module.exports = { archiveFile, downloadBuffer, tmpName }; diff --git a/tests/offsite_live.test.js b/tests/offsite_live.test.js new file mode 100644 index 00000000..87f81234 --- /dev/null +++ b/tests/offsite_live.test.js @@ -0,0 +1,115 @@ +'use strict'; + +// The off-site transports (services/offsite) against real servers — the +// same ones as tests/storage_targets_live.test.js, started by the CI job +// "storage targets (S3, WebDAV, SMB)" (.github/workflows/test.yml): S3 +// (moto / MinIO), WebDAV (wsgidav / Nextcloud), SMB (Samba with smbclient, +// SMB3 encryption required). Each part runs only when its server is named +// in the environment (the GC_TEST_* variables listed there). +// +// A real archive (gcbk.writeBackupArchive of a backup file with +// incompressible plugin data) goes up from its FILE — S3 as a multipart +// upload (the part size lowered to S3's minimum of 5 MiB) and as one PUT — +// comes back into a file byte for byte (downloadTo) and still decrypts. + +require('./helpers/test-env'); +const { describe, it, before, after } = require('node:test'); +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); + +const gcbk = require('../src/services/offsite/gcbk'); + +const env = process.env; +const RUN = crypto.randomBytes(3).toString('hex'); +const LIVE = { + s3: env.GC_TEST_S3_ENDPOINT ? { + endpoint: env.GC_TEST_S3_ENDPOINT, region: 'us-east-1', bucket: env.GC_TEST_S3_BUCKET, prefix: `gc-offsite-${RUN}`, + access_key_id: env.GC_TEST_S3_KEY, secret_access_key: env.GC_TEST_S3_SECRET, path_style: true, + } : null, + webdav: env.GC_TEST_WEBDAV_URL ? { + url: `${env.GC_TEST_WEBDAV_URL.replace(/\/+$/, '')}/gc-offsite-${RUN}/`, username: env.GC_TEST_WEBDAV_USER, password: env.GC_TEST_WEBDAV_PASSWORD, + } : null, + smb: env.GC_TEST_SMB_HOST ? { + host: env.GC_TEST_SMB_HOST, port: Number(env.GC_TEST_SMB_PORT || 445), share: env.GC_TEST_SMB_SHARE, path: `gc-offsite-${RUN}/sub`, + username: env.GC_TEST_SMB_USER, password: env.GC_TEST_SMB_PASSWORD, encrypt: true, + } : null, +}; + +const PASS = 'live test passphrase'; +const KDF = { N: 1024, r: 8, p: 1 }; +const S3_MIN_PART = 5 * 1024 * 1024; + +let dir; +let archive; // { path, size, partSize, partHashes, sha } + +function sha(file) { return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex'); } + +before(async () => { + if (!LIVE.s3 && !LIVE.webdav && !LIVE.smb) return; + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gc-offsite-live-')); + // a backup file with ~12 MB of incompressible plugin data → an archive of ~12 MB (three S3 parts) + const json = path.join(dir, 'gatecontrol-20261109-030000.json'); + fs.writeFileSync(json, JSON.stringify({ version: 1, data: { settings: [], plugins: [{ id: 'blob', db: crypto.randomBytes(12 * 1024 * 1024).toString('base64') }] } }, null, 2)); + const file = path.join(dir, 'archive.gcbk'); + const r = await gcbk.writeBackupArchive(json, file, { passphrase: PASS, kdf: KDF, gcVersion: 'live', partSize: S3_MIN_PART }); + archive = { path: file, ...r, sha: sha(file) }; + assert.equal(archive.partHashes.length, 3); +}); +after(() => { if (dir) fs.rmSync(dir, { recursive: true, force: true }); }); + +/** downloadTo into a fresh file → it decrypts to the backup that went in. */ +async function backAndDecrypt(t, cfg, name) { + const back = path.join(dir, `back-${crypto.randomBytes(4).toString('hex')}`); + const n = await t.downloadTo(cfg, name, back, undefined, { maxBytes: 64 * 1024 * 1024, size: archive.size }); + assert.equal(n, archive.size); + assert.equal(sha(back), archive.sha, 'byte for byte'); + const plain = back + '.json'; + await gcbk.decryptBackupFileTo(back, PASS, plain); + const payload = gcbk.checkPayload(JSON.parse(fs.readFileSync(plain, 'utf8'))); + assert.equal(payload.backup.data.plugins[0].id, 'blob'); + fs.rmSync(back); + fs.rmSync(plain); +} + +for (const kind of ['s3', 'webdav', 'smb']) { + const cfg = LIVE[kind]; + describe(`live off-site ${kind}`, { skip: cfg ? false : `GC_TEST_${kind.toUpperCase()}_* not set` }, () => { + const t = () => require(`../src/services/offsite/${kind}`); + + it('the write test passes', async () => { + assert.match(await t().test(cfg), /write \+ delete ok/); + }); + + it('an archive goes up from its file, is listed with its size, comes back byte for byte and decrypts', { timeout: 300000 }, async () => { + const name = 'gatecontrol-20261109-030000.gcbk'; + await t().upload(cfg, name, archive); + const listed = (await t().list(cfg)).filter((f) => f.name === name); + assert.equal(listed.length, 1); + assert.equal(listed[0].size, archive.size); + assert.ok(fs.existsSync(archive.path), 'the archive file stays the caller\'s'); + await backAndDecrypt(t(), cfg, name); + await t().remove(cfg, name); + assert.equal((await t().list(cfg)).some((f) => f.name === name), false); + }); + + it('S3: multipart with the part hashes computed while writing, with hashes read from the file, and one PUT below the part size', { skip: kind !== 's3' ? 'S3 only' : false, timeout: 300000 }, async () => { + const lim = t()._limits; + const saved = lim.part; + try { + lim.part = S3_MIN_PART; // three parts, the last one short — with the writer's hashes + await t().upload(cfg, 'gatecontrol-20261109-040000.gcbk', archive); + await backAndDecrypt(t(), cfg, 'gatecontrol-20261109-040000.gcbk'); + lim.part = S3_MIN_PART + 1024 * 1024; // other parts than the writer hashed: each part hashed from the file + await t().upload(cfg, 'gatecontrol-20261109-050000.gcbk', archive); + await backAndDecrypt(t(), cfg, 'gatecontrol-20261109-050000.gcbk'); + lim.part = 64 * 1024 * 1024; // the archive fits into one PUT + await t().upload(cfg, 'gatecontrol-20261109-060000.gcbk', { path: archive.path, size: archive.size }); + await backAndDecrypt(t(), cfg, 'gatecontrol-20261109-060000.gcbk'); + } finally { lim.part = saved; } + for (const n of ['040000', '050000', '060000']) await t().remove(cfg, `gatecontrol-20261109-${n}.gcbk`); + }); + }); +} diff --git a/tests/offsite_memory.test.js b/tests/offsite_memory.test.js new file mode 100644 index 00000000..aed04e77 --- /dev/null +++ b/tests/offsite_memory.test.js @@ -0,0 +1,33 @@ +'use strict'; + +// Peak memory of an off-site upload stays bounded, whatever the size of the +// archive: it is written to a file and sent from read streams (WebDAV PUT, +// S3 multipart), never held as one Buffer. scripts/offsite-memory-check.js +// does the measuring in its own process (RSS sampled every 10 ms during build +// + uploads); up to 1.258.1 the archive alone added its full size. + +require('./helpers/test-env'); +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { execFile } = require('node:child_process'); + +function check(args) { + return new Promise((resolve, reject) => { + execFile(process.execPath, [path.join(__dirname, '..', 'scripts', 'offsite-memory-check.js'), ...args], + { timeout: 240000, maxBuffer: 1024 * 1024 }, (err, stdout, stderr) => { + if (err) { reject(new Error(`${err.message}\n${stderr}`)); return; } + resolve(JSON.parse(stdout.trim().split('\n').pop())); + }); + }); +} + +describe('off-site upload memory', () => { + it('an archive of ~160 MB goes up (WebDAV + S3 multipart) with far less extra memory than its size', { timeout: 300000 }, async () => { + const r = await check(['--mb', '160', '--part-mb', '16']); + assert.equal(r.ok, true, JSON.stringify(r)); + assert.ok(r.archive_mb > 150, JSON.stringify(r)); + // measured ~45 MB for 50 … 800 MB archives (V8/zlib working memory, not the archive) + assert.ok(r.extra_mb < 100, `extra RSS ${r.extra_mb} MB for a ${r.archive_mb} MB archive: ${JSON.stringify(r)}`); + }); +}); diff --git a/tests/offsite_s3.test.js b/tests/offsite_s3.test.js index e4401b28..c393d3c1 100644 --- a/tests/offsite_s3.test.js +++ b/tests/offsite_s3.test.js @@ -9,6 +9,8 @@ const assert = require('node:assert/strict'); const http = require('node:http'); const crypto = require('node:crypto'); const s3 = require('../src/services/offsite/s3'); +const gcbk = require('../src/services/offsite/gcbk'); +const { archiveFile, downloadBuffer } = require('./helpers/offsiteFiles'); const EMPTY = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'; const sigOf = (h) => /Signature=([0-9a-f]{64})$/.exec(h.authorization)[1]; @@ -84,6 +86,9 @@ describe('fake S3 server', () => { const REGION = 'eu-test-1'; const objects = new Map(); // key → Buffer const seen = []; + const uploads = new Map(); // UploadId → { key, parts: Map(n → { etag, body }) } + const fault = { part: 0, complete: false, aborted: [] }; + const bodies = []; // { method, url, length, chunked, got } of every request with a body let server; let cfg; @@ -111,6 +116,7 @@ describe('fake S3 server', () => { req.on('end', () => { const body = Buffer.concat(chunks); seen.push(`${req.method} ${req.url}`); + if (body.length) bodies.push({ method: req.method, url: req.url, length: req.headers['content-length'], chunked: !!req.headers['transfer-encoding'], got: body.length }); if (!verify(req, body)) { res.writeHead(403, { 'content-type': 'application/xml' }); res.end('SignatureDoesNotMatchbad sig'); @@ -121,8 +127,39 @@ describe('fake S3 server', () => { const bucket = parts.shift(); if (bucket !== 'bk') { res.writeHead(404); res.end('NoSuchBucket'); return; } const key = parts.join('/'); + const uploadId = url.searchParams.get('uploadId'); + if (req.method === 'POST' && url.searchParams.has('uploads')) { + const id = crypto.randomBytes(8).toString('hex'); + uploads.set(id, { key, parts: new Map() }); + res.writeHead(200, { 'content-type': 'application/xml' }); + res.end(`bk${xml(key)}${id}`); + return; + } + if (uploadId) { + const u = uploads.get(uploadId); + if (!u || u.key !== key) { res.writeHead(404); res.end('NoSuchUpload'); return; } + if (req.method === 'PUT') { + const n = Number(url.searchParams.get('partNumber')); + if (fault.part === n) { res.writeHead(500); res.end('InternalErrordisk'); return; } + const etag = `"${crypto.createHash('md5').update(body).digest('hex')}"`; + u.parts.set(n, { etag, body }); + res.writeHead(200, { etag }); res.end(); return; + } + if (req.method === 'POST') { + const listed = [...body.toString('utf8').matchAll(/(\d+)<\/PartNumber>([^<]+)<\/ETag><\/Part>/g)] + .map((m) => ({ n: Number(m[1]), etag: m[2].replace(/"/g, '"') })); + if (fault.complete || !listed.length || listed.some((p) => !u.parts.has(p.n) || u.parts.get(p.n).etag !== p.etag)) { + res.writeHead(200, { 'content-type': 'application/xml' }); res.end('InvalidPartlate'); return; + } + objects.set(key, Buffer.concat(listed.map((p) => u.parts.get(p.n).body))); + uploads.delete(uploadId); + res.writeHead(200, { 'content-type': 'application/xml' }); res.end('"x-2"'); return; + } + if (req.method === 'DELETE') { uploads.delete(uploadId); fault.aborted.push(uploadId); res.writeHead(204); res.end(); return; } + } if (req.method === 'PUT') { objects.set(key, body); res.writeHead(200); res.end(); return; } if (req.method === 'DELETE') { objects.delete(key); res.writeHead(204); res.end(); return; } + if (req.method === 'GET' && key && objects.has(key)) { res.writeHead(200); res.end(objects.get(key)); return; } if (req.method === 'GET' && !key && url.searchParams.get('list-type') === '2') { const prefix = url.searchParams.get('prefix') || ''; const all = [...objects.keys()].filter((k) => k.startsWith(prefix)).sort(); @@ -178,6 +215,91 @@ describe('fake S3 server', () => { (e) => e.code === 'TRANSPORT' && /403 SignatureDoesNotMatch: bad sig/.test(e.message)); }); + it('an archive file up to the part size: one PUT streamed with Content-Length, signed with its SHA-256 (computed while writing, or read from the file)', async () => { + const data = crypto.randomBytes(300 * 1024 + 5); + bodies.length = 0; + await s3.upload(cfg, 'file-1.gcbk', archiveFile(data)); + // the hash gcbk.writeBackupArchive computed (one part: the whole file) + const hash = crypto.createHash('sha256').update(data).digest('hex'); + await s3.upload(cfg, 'file-2.gcbk', archiveFile(data, { partSize: s3._limits.part, partHashes: [hash] })); + assert.ok(objects.get('gc/backups/file-1.gcbk').equals(data)); + assert.ok(objects.get('gc/backups/file-2.gcbk').equals(data)); + assert.equal(bodies.length, 2); + assert.ok(bodies.every((b) => b.method === 'PUT' && !b.chunked && Number(b.length) === data.length)); + // a wrong precomputed hash is what the server refuses: the hash really is the one sent + await assert.rejects(s3.upload(cfg, 'file-3.gcbk', archiveFile(data, { partSize: s3._limits.part, partHashes: ['0'.repeat(64)] })), /403 SignatureDoesNotMatch/); + assert.ok((await downloadBuffer(s3, cfg, 'file-1.gcbk')).equals(data)); + }); + + describe('multipart (the part size lowered)', () => { + const PART = 64 * 1024; + let saved; + before(() => { saved = s3._limits.part; s3._limits.part = PART; }); + after(() => { s3._limits.part = saved; }); + + it('above the part size: parts read from the file one by one, each signed with its own SHA-256, then completed', async () => { + const data = crypto.randomBytes(3 * PART + 17); // four parts, the last one short + bodies.length = 0; + seen.length = 0; + await s3.upload(cfg, 'big.gcbk', archiveFile(data)); + assert.ok(objects.get('gc/backups/big.gcbk').equals(data)); + const partPuts = bodies.filter((b) => b.method === 'PUT' && /partNumber=/.test(b.url)); + assert.deepEqual(partPuts.map((b) => b.got), [PART, PART, PART, 17]); + assert.ok(partPuts.every((b) => !b.chunked && Number(b.length) === b.got)); + assert.ok(seen.some((x) => /^POST .*\?uploads=$/.test(x)), 'initiated'); + assert.equal(uploads.size, 0, 'completed'); + }); + + it('with the part hashes gcbk.writeBackupArchive computed while writing the archive', async () => { + const dir = require('node:fs').mkdtempSync(require('node:path').join(require('node:os').tmpdir(), 'gc-s3-parts-')); + try { + const json = require('node:path').join(dir, 'b.json'); + require('node:fs').writeFileSync(json, JSON.stringify({ version: 1, data: { blob: crypto.randomBytes(200 * 1024).toString('base64') } })); + const out = require('node:path').join(dir, 'a.gcbk'); + const r = await gcbk.writeBackupArchive(json, out, { passphrase: 'a long passphrase', kdf: { N: 1024, r: 8, p: 1 }, partSize: PART }); + assert.ok(r.size > 3 * PART); + assert.equal(r.partHashes.length, Math.ceil(r.size / PART)); + bodies.length = 0; + await s3.upload(cfg, 'from-archive.gcbk', { path: out, ...r }); + assert.ok(objects.get('gc/backups/from-archive.gcbk').equals(require('node:fs').readFileSync(out))); + assert.equal(bodies.filter((b) => /partNumber=/.test(b.url)).length, r.partHashes.length); + } finally { require('node:fs').rmSync(dir, { recursive: true, force: true }); } + }); + + it('a failing part: the multipart upload is aborted, the error says what S3 said, no object', async () => { + fault.part = 2; + fault.aborted.length = 0; + try { + await assert.rejects(s3.upload(cfg, 'broken.gcbk', archiveFile(crypto.randomBytes(2 * PART + 1))), + (e) => e.code === 'TRANSPORT' && /S3 PUT 500 InternalError: disk/.test(e.message)); + } finally { fault.part = 0; } + assert.equal(fault.aborted.length, 1); + assert.equal(uploads.size, 0); + assert.equal(objects.has('gc/backups/broken.gcbk'), false); + }); + + it('completing fails late (200 with an body): aborted too', async () => { + fault.complete = true; + fault.aborted.length = 0; + try { + await assert.rejects(s3.upload(cfg, 'late.gcbk', archiveFile(crypto.randomBytes(PART + 1))), /InvalidPart/); + } finally { fault.complete = false; } + assert.equal(fault.aborted.length, 1); + assert.equal(objects.has('gc/backups/late.gcbk'), false); + }); + + it('a Buffer (the write probe) still goes up as one PUT', async () => { + await s3.upload(cfg, 'probe.bin', crypto.randomBytes(PART * 2)); + assert.equal(objects.get('gc/backups/probe.bin').length, PART * 2); + }); + }); + + it('downloadTo: into a file, limited', async () => { + objects.set('gc/backups/dl.gcbk', crypto.randomBytes(5000)); + assert.ok((await downloadBuffer(s3, cfg, 'dl.gcbk')).equals(objects.get('gc/backups/dl.gcbk'))); + await assert.rejects(downloadBuffer(s3, cfg, 'dl.gcbk', undefined, { maxBytes: 4000 }), { code: 'TOO_LARGE' }); + }); + it('unreachable endpoint → TRANSPORT error', async () => { await assert.rejects(s3.list({ ...cfg, endpoint: 'http://127.0.0.1:1' }), { code: 'TRANSPORT' }); }); diff --git a/tests/offsite_sftp_smb.test.js b/tests/offsite_sftp_smb.test.js index 73bb1d24..f09f2ad7 100644 --- a/tests/offsite_sftp_smb.test.js +++ b/tests/offsite_sftp_smb.test.js @@ -36,6 +36,13 @@ for (const flag of ['-i', '-A']) { if (i !== -1) { const f = args[i + 1]; out.files[flag] = { mode: (fs.statSync(f).mode & 0o777).toString(8), content: fs.readFileSync(f, 'utf8') }; } } const n = fs.readdirSync(rec).length; +// "get ": the tool writes the local file (it exists already, 0600 — record its mode) +const cmdText = stdin + '\\n' + (args.indexOf('-c') !== -1 ? args[args.indexOf('-c') + 1] : ''); +const get = /get "[^"]+" "([^"]+)"/.exec(cmdText); +if (get) { out.files.get = { mode: (fs.statSync(get[1]).mode & 0o777).toString(8) }; fs.writeFileSync(get[1], 'remote archive'); } +// "put ": what the local file held at that moment +const put = /put "([^"]+)" "[^"]+"/.exec(cmdText); +if (put) out.files.put = { content: fs.readFileSync(put[1], 'utf8') }; fs.writeFileSync(path.join(rec, String(n).padStart(3, '0') + '.json'), JSON.stringify(out)); const mode = fs.existsSync(path.join(rec, '..', 'fail')) ? fs.readFileSync(path.join(rec, '..', 'fail'), 'utf8') : ''; if (tool === 'sftp') { @@ -59,6 +66,7 @@ for (const t of ['sftp', 'smbclient']) { const sftp = require('../src/services/offsite/sftp'); const smb = require('../src/services/offsite/smb'); const { generate } = require('../src/services/offsite/sshKey'); +const { archiveFile, tmpName } = require('./helpers/offsiteFiles'); function records() { return fs.readdirSync(rec).sort().map((f) => JSON.parse(fs.readFileSync(path.join(rec, f), 'utf8'))); @@ -104,6 +112,34 @@ describe('SFTP', () => { assert.deepEqual(workFiles(), [], 'key + upload temp files removed'); }); + it('upload of an archive FILE: put straight from it (no copy, no temp file), the file stays the caller\'s', async () => { + reset(); + const a = archiveFile(Buffer.from('the archive')); + await sftp.upload(cfg, 'gatecontrol-20260914-030000.gcbk', a, ctx); + const [r] = records(); + assert.ok(r.stdin.split('\n').includes(`put "${a.path}" "gatecontrol-20260914-030000.gcbk.part"`)); + assert.equal(r.files.put.content, 'the archive'); + assert.ok(fs.existsSync(a.path)); + assert.deepEqual(workFiles(), []); + // a path the batch line cannot carry is refused before anything runs + reset(); + await assert.rejects(sftp.upload(cfg, 'x.gcbk', { path: '/tmp/a"b', size: 3 }, ctx), (e) => e.code === 'TRANSPORT' && /local path/.test(e.message)); + assert.equal(records().length, 0); + }); + + it('downloadTo: get into the given file (created 0600 first)', async () => { + reset(); + const p = tmpName('dl'); + assert.equal(await sftp.downloadTo(cfg, 'gatecontrol-20260914-030000.gcbk', p, ctx), 'remote archive'.length); + const [r] = records(); + assert.ok(r.stdin.split('\n').includes(`get "gatecontrol-20260914-030000.gcbk" "${p}"`)); + assert.equal(r.files.get.mode, '600'); + assert.equal(fs.readFileSync(p, 'utf8'), 'remote archive'); + fs.rmSync(p); + await assert.rejects(sftp.downloadTo(cfg, 'gatecontrol-20260914-030000.gcbk', p, ctx, { maxBytes: 4 }), { code: 'TOO_LARGE' }); + fs.rmSync(p, { force: true }); + }); + it('list parses `ls -ln` (files only)', async () => { reset(); const files = await sftp.list(cfg, ctx); @@ -158,6 +194,38 @@ describe('SMB', () => { assert.deepEqual(workFiles(), []); }); + it('upload of an archive FILE: put straight from it (no copy, no temp file); ";" in its path is refused (smbclient -c splits there)', async () => { + reset(); + const a = archiveFile(Buffer.from('the archive')); + await smb.upload(cfg, 'gatecontrol-20260914-030000.gcbk', a); + const r = records()[2]; + assert.match(r.args[r.args.indexOf('-c') + 1], new RegExp(`; put "${a.path.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}" "gatecontrol-20260914-030000\\.gcbk\\.part"; `)); + assert.equal(r.files.put.content, 'the archive'); + assert.ok(fs.existsSync(a.path)); + assert.deepEqual(workFiles(), []); + reset(); + await assert.rejects(smb.upload(cfg, 'x.gcbk', { path: '/tmp/a;b', size: 3 }), (e) => e.code === 'TRANSPORT' && /local path/.test(e.message)); + assert.equal(records().length, 0); + }); + + it('downloadTo: get into the given file (created 0600 first)', async () => { + reset(); + const p = tmpName('dl'); + assert.equal(await smb.downloadTo(cfg, 'gatecontrol-20260914-030000.gcbk', p), 'remote archive'.length); + const [r] = records(); + assert.equal(r.args[r.args.indexOf('-c') + 1], `cd "gc/host1"; get "gatecontrol-20260914-030000.gcbk" "${p}"`); + assert.equal(r.files.get.mode, '600'); + assert.equal(fs.readFileSync(p, 'utf8'), 'remote archive'); + fs.rmSync(p); + }); + + it('the time limit of a transfer grows with the archive size', () => { + const { transferTimeout } = require('../src/services/offsite/source'); + assert.equal(transferTimeout(180000, 0), 180000); + // 336 MB at the slowest rate worth waiting for (1 Mbit/s): about 45 min more + assert.equal(transferTimeout(180000, 336 * 1048576), 180000 + 2688 * 1000); + }); + it('list parses smbclient `ls` (no dirs, names with spaces, dates)', async () => { reset(); const files = await smb.list(cfg); diff --git a/tests/offsite_targets_api.test.js b/tests/offsite_targets_api.test.js index 7c1ec18a..3cb97616 100644 --- a/tests/offsite_targets_api.test.js +++ b/tests/offsite_targets_api.test.js @@ -45,12 +45,15 @@ function startDav() { res.writeHead(207); res.end(`${items.join('')}`); return; } if (req.method === 'MKCOL') { cols.add(p); res.writeHead(201); res.end(); return; } - if (req.method === 'PUT') { files.set(p, Buffer.concat(chunks)); res.writeHead(201); res.end(); return; } + if (req.method === 'PUT') { + dav.puts.push({ length: req.headers['content-length'], chunked: !!req.headers['transfer-encoding'], got: Buffer.concat(chunks).length }); + files.set(p, Buffer.concat(chunks)); res.writeHead(201); res.end(); return; + } if (req.method === 'DELETE') { files.delete(p); res.writeHead(204); res.end(); return; } res.writeHead(405); res.end(); }); }); - dav = { files, cols, failPut: false }; + dav = { files, cols, failPut: false, puts: [] }; return new Promise((r) => server.listen(0, '127.0.0.1', () => { davBase = `http://127.0.0.1:${server.address().port}`; r(); })); } @@ -271,6 +274,63 @@ describe('targets CRUD + validation', () => { } finally { dav.failPut = false; } }); + // the archive is a file on the backup disk (never in memory), gone after the upload + const archiveFiles = () => { + const d = path.join(backupDir, '.incoming'); + return fs.existsSync(d) ? fs.readdirSync(d).filter((n) => /\.(offsite|verify)/.test(n)) : []; + }; + + it('run: the archive goes up from a file with Content-Length (not chunked); the file is removed afterwards, also after a failure', async () => { + dav.puts.length = 0; + const res = await agentReq('post', `${API}/targets/${davId}/run`, {}); + assert.equal(res.status, 200, JSON.stringify(res.body)); + const put = dav.puts.find((x) => x.got > 100); + assert.ok(put, 'the archive arrived'); + assert.equal(put.chunked, false); + assert.equal(Number(put.length), put.got); + assert.deepEqual(archiveFiles(), []); + dav.failPut = true; + try { assert.equal((await agentReq('post', `${API}/targets/${davId}/run`, {})).status, 502); } finally { dav.failPut = false; } + assert.deepEqual(archiveFiles(), []); + }); + + it('a leftover archive of a crashed run is removed; a foreign file in the folder is not', async () => { + // the sweep runs on the first use in a process — here: a fresh copy of the module state + const d = path.join(backupDir, '.incoming'); + fs.mkdirSync(d, { recursive: true }); + fs.writeFileSync(path.join(d, 'aaaaaaaaaaaaaaaaaaaaaaaa.offsite'), 'old'); + fs.writeFileSync(path.join(d, 'bbbbbbbbbbbbbbbbbbbbbbbb.upload'), 'a restore upload in flight'); + const key = require.resolve('../src/services/offsite'); + const saved = require.cache[key]; + delete require.cache[key]; + try { + await require('../src/services/offsite').buildArchive(path.join(backupDir, require('../src/services/autobackup').listBackupFiles()[0].filename)) + .then((a) => a.cleanup()); + } finally { require.cache[key] = saved; } + assert.deepEqual(fs.readdirSync(d).sort(), ['bbbbbbbbbbbbbbbbbbbbbbbb.upload']); + fs.rmSync(path.join(d, 'bbbbbbbbbbbbbbbbbbbbbbbb.upload')); + }); + + it('not enough free disk space for the archive: 507 NO_SPACE with the hint, recorded per target after a backup', async () => { + const backup = require('../src/services/backup'); + const orig = backup.checkFreeSpace; + backup.checkFreeSpace = async () => { throw Object.assign(new Error('not enough free disk space for the backup'), { code: 'NO_SPACE', need: 300 * 1048576, free: 20 * 1048576 }); }; + try { + const res = await agentReq('post', `${API}/targets/${davId}/run`, {}); + assert.equal(res.status, 507); + assert.equal(res.body.code, 'NO_SPACE'); + assert.equal(res.body.hint, 'no_space'); + assert.match(res.body.error, /about 300 MB needed, 20 MB free/); + const autobackup = require('../src/services/autobackup'); + const r = await require('../src/services/offsite').uploadAfterBackup(path.join(autobackup.BACKUP_DIR, autobackup.listBackupFiles()[0].filename)); + assert.deepEqual(r, { uploaded: 0, failed: 1, skipped: null }); + const t = (await getAgent().get(`${API}/targets`)).body.targets.find((x) => x.id === davId); + assert.equal(t.last_status, 'failed'); + assert.equal(t.last_error_hint, 'no_space'); + } finally { backup.checkFreeSpace = orig; } + assert.deepEqual(archiveFiles(), []); + }); + it('test with wrong credentials → 502 with detail', async () => { await agentReq('put', `${API}/targets/${davId}`, { config: { password: 'wrong' } }); const res = await agentReq('post', `${API}/targets/${davId}/test`, {}); diff --git a/tests/offsite_trust.test.js b/tests/offsite_trust.test.js index b6d434c2..661e6036 100644 --- a/tests/offsite_trust.test.js +++ b/tests/offsite_trust.test.js @@ -38,6 +38,7 @@ const https = require('node:https'); const crypto = require('node:crypto'); const { execFileSync } = require('node:child_process'); const { setup, teardown, getAgent, getCsrf } = require('./helpers/setup'); +const { archiveFile, downloadBuffer } = require('./helpers/offsiteFiles'); const NAS = 'nas.heim.example'; // resolves only where a test says so const API = '/api/v1/settings/backup'; @@ -277,9 +278,10 @@ describe('S3 through a VPN device ("Verbindung") with the trust handling', () => s3srv.seen.length = 0; assert.match(await s3().test(cfg), /write \+ delete ok/); const data = crypto.randomBytes(200 * 1024 + 3); - await s3().upload(cfg, 'gatecontrol-20261109-030000.gcbk', data); + // the archive FILE, streamed through the tunnel (signed with its SHA-256, read from the file) + await s3().upload(cfg, 'gatecontrol-20261109-030000.gcbk', archiveFile(data)); assert.deepEqual((await s3().list(cfg)).map((f) => [f.name, f.size]), [['gatecontrol-20261109-030000.gcbk', data.length]]); - assert.ok((await s3().download(cfg, 'gatecontrol-20261109-030000.gcbk')).equals(data)); + assert.ok((await downloadBuffer(s3(), cfg, 'gatecontrol-20261109-030000.gcbk')).equals(data)); await s3().remove(cfg, 'gatecontrol-20261109-030000.gcbk'); assert.ok(s3srv.seen.length >= 6); assert.ok(s3srv.seen.every((s) => s.host === `${NAS}:9000` && s.sni === NAS), JSON.stringify(s3srv.seen.slice(0, 2))); diff --git a/tests/offsite_via.test.js b/tests/offsite_via.test.js index d8bfaaa7..68743e33 100644 --- a/tests/offsite_via.test.js +++ b/tests/offsite_via.test.js @@ -50,6 +50,7 @@ const http = require('node:http'); const https = require('node:https'); const { execFileSync } = require('node:child_process'); const { setup, teardown, getAgent, getCsrf } = require('./helpers/setup'); +const { archiveFile, downloadBuffer } = require('./helpers/offsiteFiles'); const NAS = 'nas.domaincaster.example'; // never resolves: the connection has to go through the peer / route const API = '/api/v1/settings/backup'; @@ -174,9 +175,10 @@ describe('WebDAV through a VPN peer with an own certificate (the DS218+ of the r assert.deepEqual(cfg.via, { kind: 'peer', peerId: gw, port: secure.port }); assert.match(await webdav().test(cfg), /folder created, write \+ delete ok/); const data = require('node:crypto').randomBytes(300 * 1024 + 7); - await webdav().upload(cfg, 'gatecontrol-20261109-030000.gcbk', data); + // the archive FILE, streamed through the tunnel with Content-Length + await webdav().upload(cfg, 'gatecontrol-20261109-030000.gcbk', archiveFile(data)); assert.deepEqual((await webdav().list(cfg)).map((f) => [f.name, f.size]), [['gatecontrol-20261109-030000.gcbk', data.length]]); - assert.ok((await webdav().download(cfg, 'gatecontrol-20261109-030000.gcbk')).equals(data)); + assert.ok((await downloadBuffer(webdav(), cfg, 'gatecontrol-20261109-030000.gcbk')).equals(data)); await webdav().remove(cfg, 'gatecontrol-20261109-030000.gcbk'); assert.equal(secure.files.size, 0); assert.ok(secure.seen.length >= 8); diff --git a/tests/offsite_webdav.test.js b/tests/offsite_webdav.test.js index 72056308..b2e2ef02 100644 --- a/tests/offsite_webdav.test.js +++ b/tests/offsite_webdav.test.js @@ -6,12 +6,15 @@ const { describe, it, before, after } = require('node:test'); const assert = require('node:assert/strict'); const http = require('node:http'); +const crypto = require('node:crypto'); const webdav = require('../src/services/offsite/webdav'); +const { archiveFile, downloadBuffer } = require('./helpers/offsiteFiles'); describe('WebDAV transport', () => { const files = new Map(); // path → Buffer const cols = new Set(['/dav/']); const log = []; + const puts = []; let server; let base; @@ -52,8 +55,13 @@ describe('WebDAV transport', () => { if (req.method === 'PUT') { const dir = p.replace(/[^/]+$/, ''); if (!cols.has(dir)) { res.writeHead(409); res.end(); return; } + puts.push({ length: req.headers['content-length'], chunked: !!req.headers['transfer-encoding'], got: Buffer.concat(chunks).length }); files.set(p, Buffer.concat(chunks)); res.writeHead(201); res.end(); return; } + if (req.method === 'GET') { + if (!files.has(p)) { res.writeHead(404); res.end(); return; } + res.writeHead(200); res.end(files.get(p)); return; + } if (req.method === 'DELETE') { const had = files.delete(p); res.writeHead(had ? 204 : 404); res.end(); return; } res.writeHead(405); res.end(); }); @@ -83,6 +91,19 @@ describe('WebDAV transport', () => { assert.equal(f.modified, '2026-09-14T03:00:00.000Z'); }); + it('an archive file goes up as a stream with Content-Length (never chunked) and comes back into a file', async () => { + const data = crypto.randomBytes(1024 * 1024 + 3); + puts.length = 0; + const a = archiveFile(data); + await webdav.upload(cfg(), 'gatecontrol-20260915-030000.gcbk', a); + assert.deepEqual(puts, [{ length: String(data.length), chunked: false, got: data.length }]); + assert.ok(require('node:fs').existsSync(a.path), 'the caller owns the archive file'); + assert.ok((await downloadBuffer(webdav, cfg(), 'gatecontrol-20260915-030000.gcbk')).equals(data)); + await assert.rejects(downloadBuffer(webdav, cfg(), 'gatecontrol-20260915-030000.gcbk', undefined, { maxBytes: 1024 }), { code: 'TOO_LARGE' }); + await assert.rejects(downloadBuffer(webdav, cfg(), 'missing.gcbk'), (e) => e.code === 'TRANSPORT' && e.remoteStatus === 404); + await webdav.remove(cfg(), 'gatecontrol-20260915-030000.gcbk'); + }); + it('remove deletes; a missing file is fine', async () => { await webdav.remove(cfg(), 'a b&c.txt'); assert.equal(files.has('/dav/gatecontrol/a b&c.txt'), false);