From 75a9fa65acde92ea61cdb028fa8770de92d59a47 Mon Sep 17 00:00:00 2001 From: Tim Perry <1526883+pimterry@users.noreply.github.com> Date: Thu, 3 Sep 2026 10:30:21 +0100 Subject: [PATCH 001/217] test: fix flaky cleanup in http2 test Signed-off-by: Tim Perry PR-URL: https://github.com/nodejs/node/pull/65701 Reviewed-By: Filip Skokan Reviewed-By: Antoine du Hamel Reviewed-By: Luigi Pinca --- test/parallel/test-http2-bidirectional-write-deadlock.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/parallel/test-http2-bidirectional-write-deadlock.js b/test/parallel/test-http2-bidirectional-write-deadlock.js index 9dd4b91c5234..306dce021cb0 100644 --- a/test/parallel/test-http2-bidirectional-write-deadlock.js +++ b/test/parallel/test-http2-bidirectional-write-deadlock.js @@ -38,6 +38,7 @@ class StalledClientSocket extends Duplex { super(); this.inner = net.connect(port, common.localhostIPv4); this.inner.on('data', (chunk) => this.push(chunk)); + this.inner.on('end', () => this.push(null)); } _read() { // Incoming data is pushed as it arrives. @@ -54,7 +55,7 @@ class StalledClientSocket extends Duplex { callback(); } _final(callback) { - callback(); + this.inner.end(callback); } _destroy(err, callback) { this.inner.destroy(); @@ -101,7 +102,8 @@ server.listen(0, common.mustCall(() => { stallWrites = false; for (const callback of heldCallbacks) callback(); - client.destroy(); + req.end(); + client.close(); server.close(); })); })); From 48158fba8c51533501816d0ea7f66dcab1a78c47 Mon Sep 17 00:00:00 2001 From: Shelley Vohr Date: Mon, 7 Sep 2026 14:34:37 +0200 Subject: [PATCH 002/217] src: stop leaking a CppHeap in CommonEnvironmentSetup `CommonEnvironmentSetup` created a `CppHeap` for its `CreateParams` before deciding how to create the isolate, but `NewIsolate()` ignores `params->cpp_heap` and attaches a heap of its own (or the one from `IsolateSettings`). The first heap was never attached or destroyed, so every non-snapshotting setup leaked one `CppHeap`; cppgc's heap registry keeps it reachable, which is why LSAN stays quiet about it. Only create the heap on the snapshotting path, where the params go to the `SnapshotCreator` directly. Refs: https://github.com/nodejs/node/pull/55337 Signed-off-by: Shelley Vohr PR-URL: https://github.com/nodejs/node/pull/65792 Reviewed-By: James M Snell Reviewed-By: Joyee Cheung Reviewed-By: Jake Yuesong Li --- src/api/embed_helpers.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/embed_helpers.cc b/src/api/embed_helpers.cc index c1bcb3948c4c..051110c9c6e9 100644 --- a/src/api/embed_helpers.cc +++ b/src/api/embed_helpers.cc @@ -117,8 +117,6 @@ CommonEnvironmentSetup::CommonEnvironmentSetup( Isolate::CreateParams params; params.array_buffer_allocator = impl_->allocator.get(); params.external_references = external_references.data(); - params.cpp_heap = - v8::CppHeap::Create(platform, v8::CppHeapCreateParams{{}}).release(); Isolate* isolate; @@ -130,6 +128,8 @@ CommonEnvironmentSetup::CommonEnvironmentSetup( // isolate, so that the memory reducer can be initialized. isolate = impl_->isolate = Isolate::Allocate(GetOrCreateIsolateGroup()); platform->RegisterIsolate(isolate, loop); + params.cpp_heap = + v8::CppHeap::Create(platform, v8::CppHeapCreateParams{{}}).release(); impl_->snapshot_creator.emplace(isolate, params); isolate->SetCaptureStackTraceForUncaughtExceptions( From 7cdf5f014a75bbf6b3109e06b9ae278fd894c2ee Mon Sep 17 00:00:00 2001 From: Philipp Dunkel Date: Mon, 7 Sep 2026 16:48:56 +0200 Subject: [PATCH 003/217] vfs: answer for unowned paths under reserved root The module loader manufactures paths under the reserved VFS root that no layer owns: resolving a mount point as a directory first probes the sibling names `.js`, `.json` and `.node`, and a package.json walk-up passes the parents of the mount point. The lookup declined those because their layer segment is not a plain id, so they fell through to the native loader and the real file system. On POSIX that is harmless (ENOTDIR under /dev/null), but on Windows the root sits under `\\.\nul`, and `\\.\nul\` opens the NUL device: libuv reports it as a character device and a read returns nothing. The loader therefore picked `\\.\nul\vfs\.js` as an existing file, and the native walk-up above it then read the device as an empty package.json and failed with ERR_INVALID_PACKAGE_CONFIG for `\\.\nul\package.json`. Any require() of a mount point hits this on Windows. Distinguish "under the root but unowned" from "outside the root" in the lookup and have every loader override report the former as not found: stat gives ENOENT, reads and realpath throw ENOENT, the package.json lookups return their "no package.json" results, and upward walks stop at the reserved root. Paths outside the root still go to the native loader as before. Refs: https://github.com/nodejs/node/pull/65748 Signed-off-by: Philipp Dunkel PR-URL: https://github.com/nodejs/node/pull/65814 Reviewed-By: Matteo Collina Reviewed-By: Trivikram Kamat --- lib/internal/vfs/setup.js | 110 ++++++++++++------ .../test-vfs-reserved-root-unowned.js | 59 ++++++++++ 2 files changed, 133 insertions(+), 36 deletions(-) create mode 100644 test/parallel/test-vfs-reserved-root-unowned.js diff --git a/lib/internal/vfs/setup.js b/lib/internal/vfs/setup.js index a45cd47a9bf4..3e6f246d794a 100644 --- a/lib/internal/vfs/setup.js +++ b/lib/internal/vfs/setup.js @@ -35,6 +35,7 @@ const { assertEncoding, setVfsHandlers } = require('internal/fs/utils'); const permission = require('internal/process/permission'); const { getOptionValue } = require('internal/options'); const nativeModulesBinding = internalBinding('modules'); +const { UV_ENOENT } = internalBinding('uv'); let debug = require('internal/util/debuglog').debuglog('vfs', (fn) => { debug = fn; }); @@ -115,29 +116,50 @@ function deregisterVFS(vfs) { } /** - * Resolves a path string to the active VFS that owns it, or null. - * Ownership is decidable from the path alone: all mount points live - * under the reserved `${os.devNull}/vfs/` namespace, so a single - * prefix comparison rejects every real-file-system path and a map - * lookup finds the owning layer. The normalized path is returned + * Resolves a path string to the reserved VFS root, or null for a path + * outside it. Ownership is decidable from the path alone: all mount + * points live under the reserved `${os.devNull}/vfs/` namespace, so + * a single prefix comparison rejects every real-file-system path and a + * map lookup finds the owning layer. The normalized path is returned * alongside the layer so downstream helpers can skip renormalization. + * + * A path under the root that no active layer owns comes back with + * `vfs: null` rather than as `null`, because the two cases must not be + * treated alike by the module loader. The loader manufactures such + * paths itself: resolving a mount point as a directory first probes the + * sibling names `.js`, `.json`, ..., and a package.json + * walk-up passes the parents of the mount point. They cannot name + * anything real, but on Windows the root sits under `\\.\nul`, and + * `\\.\nul\` opens the NUL device, which stats as a character + * device and reads as empty. Handed to the native loader, such a probe + * "finds" a file and the walk-up above it rejects the empty device as an + * invalid package.json, so the loader must answer for the whole root. * @param {string} inputPath - * @returns {{ vfs: object, normalized: string }|null} + * @returns {{ vfs: object|null, normalized: string }|null} */ -function findVFS(inputPath) { +function findVFSOrRoot(inputPath) { const normalized = normalizeMountedPath(inputPath); if (!StringPrototypeStartsWith(normalized, normalizedVfsRootPrefix)) { return null; } const layerId = getLayerIdFromPath(normalized); - if (layerId === -1) return null; - const vfs = activeVFSLayers.get(layerId); + const vfs = layerId === -1 ? undefined : activeVFSLayers.get(layerId); if (vfs === undefined || !vfs.shouldHandleNormalized(normalized)) { - return null; + return { vfs: null, normalized }; } return { vfs, normalized }; } +/** + * Resolves a path string to the active VFS that owns it, or null. + * @param {string} inputPath + * @returns {{ vfs: object, normalized: string }|null} + */ +function findVFS(inputPath) { + const r = findVFSOrRoot(inputPath); + return r === null || r.vfs === null ? null : r; +} + /** * Drop the cache entries under `vfs`'s mount point from the * JS-reachable loader caches. Real-fs entries and other-VFS entries @@ -210,16 +232,16 @@ function findVFSForStat(filename) { } /** - * Finds the VFS owning `filename` and reads it. + * Reads `filename` from the VFS that owns it, reporting a missing file + * or a directory the way the native loader's read does. + * @param {object} vfs The VFS owning filename * @param {string} filename The absolute path to read * @param {string|object} options Read options - * @returns {{ vfs: object, content: Buffer|string }|null} + * @returns {Buffer|string} */ -function findVFSForRead(filename, options) { - const r = findVFS(filename); - if (r === null) return null; +function readVFS(vfs, filename, options) { try { - return { vfs: r.vfs, content: r.vfs.readFileSync(filename, options) }; + return vfs.readFileSync(filename, options); } catch (e) { const code = e?.code; if (code === 'ENOENT' || code === 'EISDIR') { @@ -794,30 +816,40 @@ function installModuleLoaderOverrides() { // wrapLoaderMethod then falls through to the native binding. setLoaderOverrides({ internalModuleStat(filename) { - const result = findVFSForStat(filename); - return result !== null ? result.result : undefined; + const r = findVFSOrRoot(filename); + if (r === null) return undefined; + return r.vfs === null ? UV_ENOENT : vfsStat(r.vfs, filename); }, readFileSync(filename, options) { const pathStr = typeof filename === 'string' ? filename : (filename instanceof URL ? fileURLToPath(filename) : String(filename)); - const result = findVFSForRead(pathStr, options); - return result !== null ? result.content : undefined; + const r = findVFSOrRoot(pathStr); + if (r === null) return undefined; + if (r.vfs === null) throw createENOENT('open', pathStr); + return readVFS(r.vfs, pathStr, options); }, realpathSync(filename) { - return findVFSWith(filename, 'realpath', (vfs, n) => vfs.realpathSync(n)); + const r = findVFSOrRoot(filename); + if (r === null) return undefined; + if (r.vfs === null || !r.vfs.existsSync(filename)) { + throw createENOENT('realpath', filename); + } + return r.vfs.realpathSync(filename); }, getResolutionRoot(pathStr) { - const r = findVFS(pathStr); + const r = findVFSOrRoot(pathStr); if (r === null) return undefined; - const mountPoint = r.vfs.mountPoint; // The boundary is compared as a plain string prefix by the - // callers, so only report it when the input carries the mount - // point verbatim. - return StringPrototypeStartsWith(pathStr, mountPoint) ? - mountPoint : undefined; + // callers, so only report it when the input carries it verbatim. + // An unowned path stops at the reserved root itself, so no + // node_modules lookup walks out into the real file system. + const boundary = r.vfs === null ? + getNormalizedVfsRoot() : r.vfs.mountPoint; + return StringPrototypeStartsWith(pathStr, boundary) ? + boundary : undefined; }, legacyMainResolve(pkgPath, main, base) { - if (findVFS(pkgPath) === null) return undefined; + if (findVFSOrRoot(pkgPath) === null) return undefined; for (let i = 0; i < legacyMainResolveExtensions.length; i++) { const byMain = i <= kResolvedByMainIndexNode; @@ -835,14 +867,14 @@ function installModuleLoaderOverrides() { throw new ERR_MODULE_NOT_FOUND(initial, base, undefined); }, getFormatOfExtensionlessFile(filePath) { - let result; + const r = findVFSOrRoot(filePath); + if (r === null) return undefined; + let content; try { - result = findVFSForRead(filePath, null); + content = r.vfs === null ? null : readVFS(r.vfs, filePath, null); } catch { return internalConstants.EXTENSIONLESS_FORMAT_JAVASCRIPT; } - if (result === null) return undefined; - const content = result.content; // Wasm magic bytes: 0x00 0x61 0x73 0x6d if (content && content.length >= 4 && content[0] === 0x00 && content[1] === 0x61 && @@ -852,8 +884,9 @@ function installModuleLoaderOverrides() { return internalConstants.EXTENSIONLESS_FORMAT_JAVASCRIPT; }, readPackageJSON(jsonPath, isESM, base, specifier) { - const r = findVFS(jsonPath); + const r = findVFSOrRoot(jsonPath); if (r === null) return undefined; + if (r.vfs === null) return kLoaderOverrideNoResult; const { vfs } = r; if (vfsStat(vfs, jsonPath) !== 0) return kLoaderOverrideNoResult; let content; @@ -868,8 +901,9 @@ function installModuleLoaderOverrides() { content, jsonPath, isESM, base, specifier); }, getNearestParentPackageJSON(checkPath) { - const r = findVFS(checkPath); + const r = findVFSOrRoot(checkPath); if (r === null) return undefined; + if (r.vfs === null) return kLoaderOverrideNoResult; const found = findVFSPackageJSON(r.vfs, checkPath, r.normalized); return found.tuple ?? kLoaderOverrideNoResult; }, @@ -884,8 +918,11 @@ function installModuleLoaderOverrides() { } else { filePath = resolved; } - const r = findVFS(filePath); + const r = findVFSOrRoot(filePath); if (r === null) return undefined; + // The "not found" marker is the package.json beside the queried + // path, which is what the native binding reports for it. + if (r.vfs === null) return join(dirname(filePath), 'package.json'); const found = findVFSPackageJSON(r.vfs, filePath, r.normalized); if (found.tuple !== undefined) return found.tuple; return found.sentinel; @@ -901,8 +938,9 @@ function installModuleLoaderOverrides() { } else { filePath = url; } - const r = findVFS(filePath); + const r = findVFSOrRoot(filePath); if (r === null) return undefined; + if (r.vfs === null) return kLoaderOverrideNoResult; const found = findVFSPackageJSON(r.vfs, filePath, r.normalized); if (found.tuple !== undefined) { // Tuple shape: [name, main, type, imports, exports, filePath]. diff --git a/test/parallel/test-vfs-reserved-root-unowned.js b/test/parallel/test-vfs-reserved-root-unowned.js new file mode 100644 index 000000000000..e6a5b319f610 --- /dev/null +++ b/test/parallel/test-vfs-reserved-root-unowned.js @@ -0,0 +1,59 @@ +// Flags: --experimental-vfs --expose-internals +'use strict'; + +// The module loader manufactures paths under the reserved VFS root that no +// layer owns: resolving a mount point as a directory first probes the sibling +// names `.js`, `.json`, ..., and a package.json walk-up passes +// the parents of the mount point. Such paths cannot name anything real, but +// they must still be answered by the VFS instead of being handed to the native +// loader: on Windows the reserved root sits under `\\.\nul`, and +// `\\.\nul\` opens the NUL device, which stats as a character device +// and reads as empty. The native loader would then "find" a file at +// `.js` and reject the empty package.json above it as invalid JSON. + +require('../common'); +const assert = require('assert'); +const path = require('path'); +const { pathToFileURL } = require('url'); +const vfs = require('node:vfs'); +const { loaderMethods } = require('internal/modules/helpers'); +const { getNormalizedVfsRoot } = require('internal/vfs/router'); + +const layer = vfs.create(); +layer.writeFileSync('/index.js', 'module.exports = "ran";'); +const mountPoint = layer.mount(); + +const root = getNormalizedVfsRoot(); +const unowned = [ + `${mountPoint}.js`, + `${mountPoint}.json`, + `${mountPoint}.node`, + path.join(root, 'package.json'), + path.join(root, 'nope', 'index.js'), +]; + +for (const p of unowned) { + assert.ok(loaderMethods.internalModuleStat(p) < 0, p); + assert.throws(() => loaderMethods.readFileSync(p), { code: 'ENOENT' }, p); + assert.throws(() => loaderMethods.realpathSync(p), { code: 'ENOENT' }, p); + assert.strictEqual(loaderMethods.getNearestParentPackageJSON(p), undefined, p); + assert.strictEqual(loaderMethods.readPackageJSON(p, false), undefined, p); + assert.strictEqual(loaderMethods.getPackageType(pathToFileURL(p).href), undefined, p); + // The "not found" marker is the last candidate examined, like the native + // binding returns. + assert.strictEqual( + loaderMethods.getPackageScopeConfig(pathToFileURL(p).href), + path.join(path.dirname(p), 'package.json'), p); + // Upward walks (node_modules lookups) stop at the reserved root rather than + // continuing into the real file system. + assert.strictEqual(loaderMethods.getResolutionRoot(p), root, p); +} + +// Paths outside the reserved root are still left to the native loader. +assert.strictEqual(loaderMethods.getResolutionRoot(__filename), undefined); + +// The mount point itself resolves as a directory to its index through the +// layer, which is the sequence that produced the sibling probes above. +assert.strictEqual(require(mountPoint), 'ran'); + +layer.unmount(); From 619470a9ae37fa14df4287e2727a1b2560457ef4 Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Sun, 12 Jul 2026 15:34:00 +0200 Subject: [PATCH 004/217] stream: allocate stream read buffers from a slab Read buffers for streams that emit their data to JS were allocated per read: a 64KB backing store, tracked in a map, and then - since reads rarely fill the whole buffer - reallocated to the right size and copied. Allocate read buffers from a 64KB slab instead. Reads reserve the suggested size from the slab and JS receives a view over the slab's ArrayBuffer at the read's offset, using the offset mechanism that onStreamRead already supports. Unused reservation space is rewound when a read returns less than was reserved, so small reads (e.g. TLS records) share a slab. This removes the per-read allocations, the map bookkeeping and the resize copy. Signed-off-by: Matteo Collina PR-URL: https://github.com/nodejs/node/pull/64455 Reviewed-By: James M Snell Reviewed-By: Robert Nagy --- benchmark/net/tcp-raw-c2s.js | 10 +- benchmark/net/tcp-raw-pipe.js | 13 ++- benchmark/net/tcp-raw-s2c.js | 10 +- lib/internal/webstreams/adapters.js | 4 +- src/env.cc | 99 +++++++++++++++++++ src/env.h | 64 ++++++++++++ src/stream_base.cc | 33 ++++--- ...t-whatwg-webstreams-adapters-streambase.js | 15 ++- 8 files changed, 221 insertions(+), 27 deletions(-) diff --git a/benchmark/net/tcp-raw-c2s.js b/benchmark/net/tcp-raw-c2s.js index 5b174f50e81c..24e35906c925 100644 --- a/benchmark/net/tcp-raw-c2s.js +++ b/benchmark/net/tcp-raw-c2s.js @@ -23,7 +23,11 @@ function main({ dur, len, type }) { TCPConnectWrap, constants: TCPConstants, } = common.binding('tcp_wrap'); - const { WriteWrap } = common.binding('stream_wrap'); + const { + WriteWrap, + kReadBytesOrError, + streamBaseState, + } = common.binding('stream_wrap'); const PORT = common.PORT; const serverHandle = new TCP(TCPConstants.SERVER); @@ -55,9 +59,7 @@ function main({ dur, len, type }) { if (!buffer) fail('read'); - // Don't slice the buffer. The point of this is to isolate, not - // simulate real traffic. - bytes += buffer.byteLength; + bytes += streamBaseState[kReadBytesOrError]; }; clientHandle.readStart(); diff --git a/benchmark/net/tcp-raw-pipe.js b/benchmark/net/tcp-raw-pipe.js index fbaa21b5a309..36ddb822e04e 100644 --- a/benchmark/net/tcp-raw-pipe.js +++ b/benchmark/net/tcp-raw-pipe.js @@ -23,7 +23,12 @@ function main({ dur, len, type }) { TCPConnectWrap, constants: TCPConstants, } = common.binding('tcp_wrap'); - const { WriteWrap } = common.binding('stream_wrap'); + const { + WriteWrap, + kReadBytesOrError, + kArrayBufferOffset, + streamBaseState, + } = common.binding('stream_wrap'); const PORT = common.PORT; function fail(err, syscall) { @@ -50,9 +55,11 @@ function main({ dur, len, type }) { if (!buffer) fail('read'); + const nread = streamBaseState[kReadBytesOrError]; + const offset = streamBaseState[kArrayBufferOffset]; const writeReq = new WriteWrap(); writeReq.async = false; - err = clientHandle.writeBuffer(writeReq, Buffer.from(buffer)); + err = clientHandle.writeBuffer(writeReq, Buffer.from(buffer, offset, nread)); if (err) fail(err, 'write'); @@ -94,7 +101,7 @@ function main({ dur, len, type }) { if (!buffer) fail('read'); - bytes += buffer.byteLength; + bytes += streamBaseState[kReadBytesOrError]; }; connectReq.oncomplete = function(err) { diff --git a/benchmark/net/tcp-raw-s2c.js b/benchmark/net/tcp-raw-s2c.js index 3ca03529fcea..a847b8c28dcd 100644 --- a/benchmark/net/tcp-raw-s2c.js +++ b/benchmark/net/tcp-raw-s2c.js @@ -23,7 +23,11 @@ function main({ dur, len, type }) { TCPConnectWrap, constants: TCPConstants, } = common.binding('tcp_wrap'); - const { WriteWrap } = common.binding('stream_wrap'); + const { + WriteWrap, + kReadBytesOrError, + streamBaseState, + } = common.binding('stream_wrap'); const PORT = common.PORT; const serverHandle = new TCP(TCPConstants.SERVER); @@ -116,9 +120,7 @@ function main({ dur, len, type }) { if (!buffer) fail('read'); - // Don't slice the buffer. The point of this is to isolate, not - // simulate real traffic. - bytes += buffer.byteLength; + bytes += streamBaseState[kReadBytesOrError]; }; clientHandle.readStart(); diff --git a/lib/internal/webstreams/adapters.js b/lib/internal/webstreams/adapters.js index c00cc38ceef8..ef99ad1d39f6 100644 --- a/lib/internal/webstreams/adapters.js +++ b/lib/internal/webstreams/adapters.js @@ -94,6 +94,7 @@ const { const { WriteWrap, ShutdownWrap, + kArrayBufferOffset, kReadBytesOrError, kLastWriteWasAsync, streamBaseState, @@ -1121,7 +1122,8 @@ function newReadableStreamFromStreamBase(streamBase, strategy, options = kEmptyO return; } - controller.enqueue(arrayBuffer); + const offset = streamBaseState[kArrayBufferOffset]; + controller.enqueue(new Uint8Array(arrayBuffer, offset, nread)); if (controller.desiredSize <= 0) streamBase.readStop(); diff --git a/src/env.cc b/src/env.cc index 5626b2a7e10e..bac9a87f490f 100644 --- a/src/env.cc +++ b/src/env.cc @@ -822,6 +822,105 @@ void Environment::recycle_managed_buffer(std::unique_ptr bs) { managed_buffer_cache_ = std::move(bs); } +uv_buf_t StreamReadSlab::Allocate(Isolate* isolate, size_t suggested) { + DCHECK_GT(suggested, 0); + // Reads always get the full `suggested` size: handing out a smaller + // remainder would shrink the read() buffer and fragment large reads into + // more system calls, which costs more than the slab tail it saves. + size_t remaining = current_.bs ? current_.size() - current_.offset : 0; + if (remaining < suggested) { + // Retire the current slab; it stays alive through `retired_` if reads + // are still pending on it, or through JS views over it otherwise. + if (current_.bs && current_.pending > 0) + retired_.push_back(std::move(current_)); + current_ = Slab(); + std::unique_ptr bs = ArrayBuffer::NewBackingStore( + isolate, + std::max(kSlabSize, suggested), + BackingStoreInitializationMode::kUninitialized); + current_.bs = std::move(bs); + } + char* base = current_.data() + current_.offset; + current_.offset += suggested; + current_.last_base = base; + current_.last_end = current_.offset; + current_.pending++; + return uv_buf_init(base, suggested); +} + +StreamReadSlab::Slab* StreamReadSlab::FindSlab(const char* base) { + if (current_.Contains(base)) return ¤t_; + for (Slab& slab : retired_) + if (slab.Contains(base)) return &slab; + return nullptr; +} + +void StreamReadSlab::CompleteReservation(Slab* slab, + const uv_buf_t& buf, + size_t used) { + DCHECK_GT(slab->pending, 0); + slab->pending--; + if (buf.base == slab->last_base && slab->offset == slab->last_end) { + // This was the most recent reservation and nothing was reserved after + // it: rewind the unused remainder so it can be reserved again. + slab->offset = (buf.base - slab->data()) + used; + slab->last_end = slab->offset; + } + if (slab != ¤t_ && slab->pending == 0) { + for (auto it = retired_.begin(); it != retired_.end(); ++it) { + if (&*it == slab) { + retired_.erase(it); + break; + } + } + } +} + +bool StreamReadSlab::Commit(Isolate* isolate, + const uv_buf_t& buf, + size_t nread, + Local* ab, + size_t* offset) { + Slab* slab = FindSlab(buf.base); + if (slab == nullptr) return false; + DCHECK_LE(nread, buf.len); + + // A partial read would leave the rest of its reservation as waste once + // the slab retires - memory that counts towards V8's external memory and + // drives up GC frequency. If most of the reservation would be wasted, + // give the read a right-sized copy instead (as if it had never been read + // into the slab) and return its reservation in full. Reads that (mostly) + // fill their reservation get a zero-copy view into the slab. + if (nread < buf.len - buf.len / 4 && buf.base == slab->last_base && + slab->offset == slab->last_end) { + std::unique_ptr bs = ArrayBuffer::NewBackingStore( + isolate, nread, BackingStoreInitializationMode::kUninitialized); + memcpy(bs->Data(), buf.base, nread); + *ab = ArrayBuffer::New(isolate, std::move(bs)); + *offset = 0; + CompleteReservation(slab, buf, 0); + return true; + } + + *offset = buf.base - slab->data(); + if (slab->ab.IsEmpty()) { + *ab = ArrayBuffer::New(isolate, slab->bs); + slab->ab.Reset(isolate, *ab); + } else { + *ab = slab->ab.Get(isolate); + } + CompleteReservation(slab, buf, nread); + return true; +} + +bool StreamReadSlab::Release(const uv_buf_t& buf) { + if (buf.base == nullptr) return true; + Slab* slab = FindSlab(buf.base); + if (slab == nullptr) return false; + CompleteReservation(slab, buf, 0); + return true; +} + std::string Environment::GetExecPath(const std::vector& argv) { char exec_path_buf[2 * PATH_MAX]; size_t exec_path_len = sizeof(exec_path_buf); diff --git a/src/env.h b/src/env.h index 4410ccb3e101..6a94dae269d2 100644 --- a/src/env.h +++ b/src/env.h @@ -287,6 +287,63 @@ struct ContextInfo { class EnabledDebugList; +// A bump allocator for stream read buffers. Reads reserve a chunk of the +// current slab and, once completed, are handed to JS as a view (ArrayBuffer + +// offset) over the slab, so that no per-read allocation or copy is needed. +// Unused reservation space is rewound when a read returns fewer bytes than +// were reserved. Slabs with reads still pending when a new slab is started +// (possible when multiple reads are in flight, e.g. on Windows) are kept +// alive in `retired_` until those reads complete. +class StreamReadSlab { + public: + // Sized to match the read buffer size that libuv suggests for stream + // reads: a slab typically serves a single large read (still avoiding the + // copy that right-sizing the buffer would need), or many small ones. + // Larger slabs amortize allocations further, but stay alive (pinned by + // chunk views) long enough to be promoted to V8's old generation, where + // their external memory is only reclaimed by major GCs. + static constexpr size_t kSlabSize = 64 * 1024; + + // Reserve `suggested` bytes. Starts a new slab if the current one does + // not have enough space left. + uv_buf_t Allocate(v8::Isolate* isolate, size_t suggested); + // Commit a completed read of `nread` bytes into the buffer previously + // returned by Allocate(), rewinding the unused remainder of the + // reservation if possible. Returns the slab's ArrayBuffer and the offset + // of `buf.base` within it. Returns false if the buffer was not allocated + // from this slab (e.g. reads rerouted from another stream listener). + bool Commit(v8::Isolate* isolate, + const uv_buf_t& buf, + size_t nread, + v8::Local* ab, + size_t* offset); + // Return an unused reservation (failed or empty read). Returns false if + // the buffer was not allocated from this slab. + bool Release(const uv_buf_t& buf); + + private: + struct Slab { + std::shared_ptr bs; + v8::Global ab; + size_t offset = 0; // Bump pointer. + size_t pending = 0; // Reservations not yet committed/released. + char* last_base = nullptr; // Most recent reservation... + size_t last_end = 0; // ...and the bump pointer after it. + + char* data() const { return static_cast(bs->Data()); } + size_t size() const { return bs->ByteLength(); } + bool Contains(const char* p) const { + return bs && p >= data() && p < data() + size(); + } + }; + + Slab* FindSlab(const char* base); + void CompleteReservation(Slab* slab, const uv_buf_t& buf, size_t used); + + Slab current_; + std::vector retired_; +}; + namespace per_process { extern std::shared_ptr system_environment; } @@ -1082,6 +1139,10 @@ class Environment final : public MemoryRetainer { // Only buffers that were not exposed externally may be recycled. void recycle_managed_buffer(std::unique_ptr bs); + StreamReadSlab& stream_read_slab() { + return stream_read_slab_; + } + void AddUnmanagedFd(int fd); void RemoveUnmanagedFd(int fd); @@ -1309,6 +1370,9 @@ class Environment final : public MemoryRetainer { released_allocated_buffers_; std::unique_ptr managed_buffer_cache_; + // Used by EmitToJSStreamListener to allocate stream read buffers. + StreamReadSlab stream_read_slab_; + v8::CpuProfiler* cpu_profiler_ = nullptr; std::vector pending_profiles_; }; diff --git a/src/stream_base.cc b/src/stream_base.cc index 0791dcf81ebb..bd38c8c4cd6e 100644 --- a/src/stream_base.cc +++ b/src/stream_base.cc @@ -684,7 +684,7 @@ void StreamResource::ClearError() { uv_buf_t EmitToJSStreamListener::OnStreamAlloc(size_t suggested_size) { CHECK_NOT_NULL(stream_); Environment* env = static_cast(stream_)->stream_env(); - return env->allocate_managed_buffer(suggested_size); + return env->stream_read_slab().Allocate(env->isolate(), suggested_size); } void EmitToJSStreamListener::OnStreamRead(ssize_t nread, const uv_buf_t& buf_) { @@ -694,25 +694,34 @@ void EmitToJSStreamListener::OnStreamRead(ssize_t nread, const uv_buf_t& buf_) { Isolate* isolate = env->isolate(); HandleScope handle_scope(isolate); Context::Scope context_scope(env->context()); - std::unique_ptr bs = env->release_managed_buffer(buf_); if (nread <= 0) { - env->recycle_managed_buffer(std::move(bs)); + if (!env->stream_read_slab().Release(buf_)) + env->recycle_managed_buffer(env->release_managed_buffer(buf_)); if (nread < 0) stream->CallJSOnreadMethod(nread, Local()); return; } - CHECK_LE(static_cast(nread), bs->ByteLength()); - if (static_cast(nread) != bs->ByteLength()) { - std::unique_ptr old_bs = std::move(bs); - bs = ArrayBuffer::NewBackingStore( - isolate, nread, BackingStoreInitializationMode::kUninitialized); - memcpy(bs->Data(), old_bs->Data(), nread); - env->recycle_managed_buffer(std::move(old_bs)); + CHECK_LE(static_cast(nread), buf_.len); + size_t offset = 0; + Local ab; + if (!env->stream_read_slab().Commit(isolate, buf_, nread, &ab, &offset)) { + // The buffer was allocated through allocate_managed_buffer() by another + // stream listener (e.g. StreamPipe's) whose read was rerouted here after + // that listener was removed. + std::unique_ptr bs = env->release_managed_buffer(buf_); + CHECK_LE(static_cast(nread), bs->ByteLength()); + if (static_cast(nread) != bs->ByteLength()) { + std::unique_ptr old_bs = std::move(bs); + bs = ArrayBuffer::NewBackingStore( + isolate, nread, BackingStoreInitializationMode::kUninitialized); + memcpy(bs->Data(), old_bs->Data(), nread); + env->recycle_managed_buffer(std::move(old_bs)); + } + ab = ArrayBuffer::New(isolate, std::move(bs)); } - - stream->CallJSOnreadMethod(nread, ArrayBuffer::New(isolate, std::move(bs))); + stream->CallJSOnreadMethod(nread, ab, offset); } diff --git a/test/parallel/test-whatwg-webstreams-adapters-streambase.js b/test/parallel/test-whatwg-webstreams-adapters-streambase.js index bd864ab8c141..8a42dd6a9b76 100644 --- a/test/parallel/test-whatwg-webstreams-adapters-streambase.js +++ b/test/parallel/test-whatwg-webstreams-adapters-streambase.js @@ -35,6 +35,8 @@ const { { const buf = Buffer.from('hello'); const check = new Uint8Array(buf); + const buf2 = Buffer.from('world'); + const check2 = new Uint8Array(buf2); const stream = new JSStream(); @@ -47,13 +49,20 @@ const { assert.deepStrictEqual(new Uint8Array(value), check); reader.read().then(common.mustCall(({ done, value }) => { - assert(done); - assert.strictEqual(value, undefined); - })); + assert(!done); + assert.deepStrictEqual(new Uint8Array(value), check2); + reader.read().then(common.mustCall(({ done, value }) => { + assert(done); + assert.strictEqual(value, undefined); + })); + })); })); + // Two reads land in the same read buffer; the second chunk must be + // sliced from its offset within it. stream.readBuffer(buf); + stream.readBuffer(buf2); stream.emitEOF(); } From 15605210f82ef531181a1f651bd2f3db1d12e0a1 Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Sun, 12 Jul 2026 15:34:14 +0200 Subject: [PATCH 005/217] stream: create write request objects lazily Every stream write created a WriteWrap JS object up front, even though most writes complete synchronously via uv_try_write() and never use it. Let stream_base_commons pass null instead of a request object. StreamBase::Write() already creates the wrap object only when the write does not complete synchronously; return that object to JS (which attaches oncomplete/callback to it) and a plain error code otherwise. Writes that complete synchronously now cross the JS/C++ boundary once and allocate nothing. Callers that pass in a request object (child_process IPC, webstreams adapters) behave as before. Since Http2Stream::DoWrite() can invoke the completion callback synchronously - before JS has attached oncomplete - such completions are now recorded on the request object's writeStatus field and replayed by stream_base_commons after dispatch. This also replaces a Has() plus name-based MakeCallback() pair with a single Get(). Also pre-create the JS fields of WriteWrap instances in the object template, as was already done for ShutdownWrap, so that they are in-object properties. Signed-off-by: Matteo Collina PR-URL: https://github.com/nodejs/node/pull/64455 Reviewed-By: James M Snell Reviewed-By: Robert Nagy --- lib/internal/stream_base_commons.js | 141 ++++++++++++++-------------- src/env_properties.h | 1 + src/stream_base.cc | 105 +++++++++++++++++---- src/stream_base.h | 10 ++ src/stream_wrap.cc | 15 +++ 5 files changed, 185 insertions(+), 87 deletions(-) diff --git a/lib/internal/stream_base_commons.js b/lib/internal/stream_base_commons.js index 6d144f8a0fa6..7bf3a2bab638 100644 --- a/lib/internal/stream_base_commons.js +++ b/lib/internal/stream_base_commons.js @@ -8,11 +8,10 @@ const { const { Buffer } = require('buffer'); const { FastBuffer } = require('internal/buffer'); const { - WriteWrap, kReadBytesOrError, kArrayBufferOffset, kBytesWritten, - kLastWriteWasAsync, + kLastWriteErr, streamBaseState, } = internalBinding('stream_wrap'); const { UV_EOF } = internalBinding('uv'); @@ -43,41 +42,6 @@ const kBuffer = Symbol('kBuffer'); const kBufferGen = Symbol('kBufferGen'); const kBufferCb = Symbol('kBufferCb'); -function handleWriteReq(req, data, encoding) { - const { handle } = req; - - switch (encoding) { - case 'buffer': - { - const ret = handle.writeBuffer(req, data); - if (streamBaseState[kLastWriteWasAsync]) - req.buffer = data; - return ret; - } - case 'latin1': - case 'binary': - return handle.writeLatin1String(req, data); - case 'utf8': - case 'utf-8': - return handle.writeUtf8String(req, data); - case 'ascii': - return handle.writeAsciiString(req, data); - case 'ucs2': - case 'ucs-2': - case 'utf16le': - case 'utf-16le': - return handle.writeUcs2String(req, data); - default: - { - const buffer = Buffer.from(data, encoding); - const ret = handle.writeBuffer(req, buffer); - if (streamBaseState[kLastWriteWasAsync]) - req.buffer = buffer; - return ret; - } - } -} - function onWriteComplete(status) { debug('onWriteComplete', status, this.error); @@ -105,21 +69,8 @@ function onWriteComplete(status) { this.callback(null); } -function createWriteWrap(handle, callback) { - const req = new WriteWrap(); - - req.handle = handle; - req.oncomplete = onWriteComplete; - req.async = false; - req.bytes = 0; - req.buffer = null; - req.callback = callback; - - return req; -} - function writevGeneric(self, data, cb) { - const req = createWriteWrap(self[kHandle], cb); + const handle = self[kHandle]; const allBuffers = data.allBuffers; let chunks; if (allBuffers) { @@ -134,33 +85,87 @@ function writevGeneric(self, data, cb) { chunks[i * 2 + 1] = entry.encoding; } } - const err = req.handle.writev(req, chunks, allBuffers); - - // Retain chunks - if (err === 0) req._chunks = chunks; + const ret = handle.writev(null, chunks, allBuffers); - afterWriteDispatched(req, err, cb); - return req; + return afterWriteDispatched(handle, ret, chunks, cb); } function writeGeneric(self, data, encoding, cb) { - const req = createWriteWrap(self[kHandle], cb); - const err = handleWriteReq(req, data, encoding); + const handle = self[kHandle]; + let ret; + let buffer = null; - afterWriteDispatched(req, err, cb); - return req; + switch (encoding) { + case 'buffer': + buffer = data; + ret = handle.writeBuffer(null, data); + break; + case 'latin1': + case 'binary': + ret = handle.writeLatin1String(null, data); + break; + case 'utf8': + case 'utf-8': + ret = handle.writeUtf8String(null, data); + break; + case 'ascii': + ret = handle.writeAsciiString(null, data); + break; + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + ret = handle.writeUcs2String(null, data); + break; + default: + buffer = Buffer.from(data, encoding); + ret = handle.writeBuffer(null, buffer); + break; + } + + return afterWriteDispatched(handle, ret, buffer, cb); } -function afterWriteDispatched(req, err, cb) { - req.bytes = streamBaseState[kBytesWritten]; - req.async = !!streamBaseState[kLastWriteWasAsync]; +// `ret` is either a numeric error code (when the write - or its failure - +// completed synchronously and no write request was created) or the WriteWrap +// object of a dispatched write. +function afterWriteDispatched(handle, ret, buffer, cb) { + const bytes = streamBaseState[kBytesWritten]; + + if (typeof ret === 'number') { + // The write (or its failure) completed synchronously. + if (ret !== 0) + cb(new ErrnoException(ret, 'write')); + else if (typeof cb === 'function') + cb(); + return { async: false, bytes }; + } - if (err !== 0) - return cb(new ErrnoException(err, 'write', req.error)); + const req = ret; + const err = streamBaseState[kLastWriteErr]; + if (err !== 0) { + cb(new ErrnoException(err, 'write', req.error)); + return { async: false, bytes }; + } - if (!req.async && typeof req.callback === 'function') { - req.callback(); + req.handle = handle; + req.oncomplete = onWriteComplete; + req.callback = cb; + req.async = true; + req.bytes = bytes; + // Retain the data (or chunks) being written until the write completes. + req.buffer = buffer; + + // If the write completed synchronously inside the write call, before + // `oncomplete` could be attached, the completion status has been recorded; + // deliver it now. + const status = req.writeStatus; + if (status !== null) { + req.writeStatus = null; + req.oncomplete(status); } + + return req; } function onStreamRead(arrayBuffer) { diff --git a/src/env_properties.h b/src/env_properties.h index fc1772f926e5..bfb981d1c89b 100644 --- a/src/env_properties.h +++ b/src/env_properties.h @@ -407,6 +407,7 @@ V(writable_string, "writable") \ V(write_host_object_string, "_writeHostObject") \ V(write_queue_size_string, "writeQueueSize") \ + V(write_status_string, "writeStatus") \ V(zlib_string, "zlib") \ V(zstd_string, "zstd") diff --git a/src/stream_base.cc b/src/stream_base.cc index bd38c8c4cd6e..8b94f9eb4225 100644 --- a/src/stream_base.cc +++ b/src/stream_base.cc @@ -175,6 +175,23 @@ int StreamBase::Shutdown(const FunctionCallbackInfo& args) { void StreamBase::SetWriteResult(const StreamWriteResult& res) { env_->stream_base_state()[kBytesWritten] = res.bytes; env_->stream_base_state()[kLastWriteWasAsync] = res.async; + env_->stream_base_state()[kLastWriteErr] = res.err; +} + +// Finish a JS-initiated write. When the caller did not pass a request +// object (`lazy_req`), the write wrap object exists only if the write did +// not complete synchronously; hand it to JS so that it can attach its +// completion callback. Otherwise the numeric error code (via JSMethod) is +// all JS needs. +int StreamBase::FinishWrite(const v8::FunctionCallbackInfo& args, + const StreamWriteResult& res, + bool lazy_req) { + SetWriteResult(res); + if (lazy_req && res.wrap_obj) { + args.GetReturnValue().Set(res.wrap_obj->object()); + return kReturnValueSet; + } + return res.err; } int StreamBase::Writev(const FunctionCallbackInfo& args) { @@ -182,10 +199,13 @@ int StreamBase::Writev(const FunctionCallbackInfo& args) { Isolate* isolate = env->isolate(); Local context = env->context(); - CHECK(args[0]->IsObject()); CHECK(args[1]->IsArray()); - Local req_wrap_obj = args[0].As(); + // When no request object is passed in, one is created by Write() only if + // the write does not complete synchronously; see FinishWrite(). + const bool lazy_req = !args[0]->IsObject(); + Local req_wrap_obj; + if (!lazy_req) req_wrap_obj = args[0].As(); Local chunks = args[1].As(); bool all_buffers = args[2]->IsTrue(); @@ -287,20 +307,19 @@ int StreamBase::Writev(const FunctionCallbackInfo& args) { } StreamWriteResult res = Write(*bufs, count, nullptr, req_wrap_obj); - SetWriteResult(res); if (res.wrap != nullptr && storage_size > 0) res.wrap->SetBackingStore(std::move(bs)); - return res.err; + return FinishWrite(args, res, lazy_req); } - int StreamBase::WriteBuffer(const FunctionCallbackInfo& args) { - CHECK(args[0]->IsObject()); CHECK(args[1]->IsUint8Array()); Environment* env = Environment::GetCurrent(args); - Local req_wrap_obj = args[0].As(); + bool lazy_req = !args[0]->IsObject(); + Local req_wrap_obj; + if (!lazy_req) req_wrap_obj = args[0].As(); uv_buf_t buf; buf.base = Buffer::Data(args[1]); buf.len = Buffer::Length(args[1]); @@ -310,6 +329,16 @@ int StreamBase::WriteBuffer(const FunctionCallbackInfo& args) { if (args[2]->IsObject() && IsIPCPipe()) { Local send_handle_obj = args[2].As(); + if (lazy_req) { + // Sending a handle requires a request object up front to reference it. + if (!env->write_wrap_template() + ->NewInstance(env->context()) + .ToLocal(&req_wrap_obj)) { + return UV_EBUSY; + } + StreamReq::ResetObject(req_wrap_obj); + } + HandleWrap* wrap; ASSIGN_OR_RETURN_UNWRAP(&wrap, send_handle_obj, UV_EINVAL); send_handle = reinterpret_cast(wrap->GetHandle()); @@ -323,20 +352,18 @@ int StreamBase::WriteBuffer(const FunctionCallbackInfo& args) { } StreamWriteResult res = Write(&buf, 1, send_handle, req_wrap_obj); - SetWriteResult(res); - - return res.err; + return FinishWrite(args, res, lazy_req); } - template int StreamBase::WriteString(const FunctionCallbackInfo& args) { Environment* env = Environment::GetCurrent(args); Isolate* isolate = env->isolate(); - CHECK(args[0]->IsObject()); CHECK(args[1]->IsString()); - Local req_wrap_obj = args[0].As(); + const bool lazy_req = !args[0]->IsObject(); + Local req_wrap_obj; + if (!lazy_req) req_wrap_obj = args[0].As(); Local string = args[1].As(); Local send_handle_obj; if (args[2]->IsObject()) @@ -417,6 +444,16 @@ int StreamBase::WriteString(const FunctionCallbackInfo& args) { uv_stream_t* send_handle = nullptr; if (IsIPCPipe() && !send_handle_obj.IsEmpty()) { + if (lazy_req && req_wrap_obj.IsEmpty()) { + // Sending a handle requires a request object up front to reference it. + if (!env->write_wrap_template() + ->NewInstance(env->context()) + .ToLocal(&req_wrap_obj)) { + return UV_EBUSY; + } + StreamReq::ResetObject(req_wrap_obj); + } + HandleWrap* wrap; ASSIGN_OR_RETURN_UNWRAP(&wrap, send_handle_obj, UV_EINVAL); send_handle = reinterpret_cast(wrap->GetHandle()); @@ -432,11 +469,10 @@ int StreamBase::WriteString(const FunctionCallbackInfo& args) { StreamWriteResult res = Write(&buf, 1, send_handle, req_wrap_obj, try_write); res.bytes += synchronously_written; - SetWriteResult(res); if (res.wrap != nullptr) res.wrap->SetBackingStore(std::move(bs)); - return res.err; + return FinishWrite(args, res, lazy_req); } @@ -662,7 +698,8 @@ void StreamBase::JSMethod(const FunctionCallbackInfo& args) { if (!wrap->IsAlive()) return args.GetReturnValue().Set(UV_EINVAL); AsyncHooks::DefaultTriggerAsyncIdScope trigger_scope(wrap->GetAsyncWrap()); - args.GetReturnValue().Set((wrap->*Method)(args)); + int ret = (wrap->*Method)(args); + if (ret != kReturnValueSet) args.GetReturnValue().Set(ret); } int StreamResource::DoTryWrite(uv_buf_t** bufs, size_t* count) { @@ -771,20 +808,50 @@ void ReportWritesToJSStreamListener::OnStreamAfterReqFinished( CHECK(!async_wrap->persistent().IsEmpty()); Local req_wrap_obj = async_wrap->object(); + Local oncomplete; + if (!req_wrap_obj->Get(env->context(), env->oncomplete_string()) + .ToLocal(&oncomplete)) { + return; + } + + const char* msg = stream->Error(); + + if (!oncomplete->IsFunction()) { + // The completion callback has not been attached yet: the write finished + // synchronously inside the JS write call, before the request object was + // returned to JS. Record the status so that JS can replay the callback. + if (req_wrap_obj + ->Set(env->context(), + env->write_status_string(), + Integer::New(env->isolate(), status)) + .IsNothing()) { + return; + } + if (msg != nullptr) { + if (req_wrap_obj + ->Set(env->context(), + env->error_string(), + OneByteString(env->isolate(), msg)) + .IsNothing()) { + return; + } + stream->ClearError(); + } + return; + } + Local argv[] = { Integer::New(env->isolate(), status), stream->GetObject(), Undefined(env->isolate()) }; - const char* msg = stream->Error(); if (msg != nullptr) { argv[2] = OneByteString(env->isolate(), msg); stream->ClearError(); } - if (req_wrap_obj->Has(env->context(), env->oncomplete_string()).FromJust()) - async_wrap->MakeCallback(env->oncomplete_string(), arraysize(argv), argv); + async_wrap->MakeCallback(oncomplete.As(), arraysize(argv), argv); } void ReportWritesToJSStreamListener::OnStreamAfterWrite( diff --git a/src/stream_base.h b/src/stream_base.h index cb795a541297..450205a1b863 100644 --- a/src/stream_base.h +++ b/src/stream_base.h @@ -10,6 +10,8 @@ #include "v8.h" +#include // INT_MIN + namespace node { // Forward declarations @@ -405,14 +407,22 @@ class StreamBase : public StreamResource { kArrayBufferOffset, kBytesWritten, kLastWriteWasAsync, + kLastWriteErr, kNumStreamBaseStateFields }; private: + // Sentinel return value for JS methods that have set their own (object) + // return value and must not have it overwritten by JSMethod(). + static constexpr int kReturnValueSet = INT_MIN; + Environment* env_; EmitToJSStreamListener default_listener_; void SetWriteResult(const StreamWriteResult& res); + int FinishWrite(const v8::FunctionCallbackInfo& args, + const StreamWriteResult& res, + bool lazy_req); static void AddAccessor(v8::Isolate* isolate, v8::Local sig, enum v8::PropertyAttribute attributes, diff --git a/src/stream_wrap.cc b/src/stream_wrap.cc index 6b85d6533879..c7d3c4745511 100644 --- a/src/stream_wrap.cc +++ b/src/stream_wrap.cc @@ -95,6 +95,20 @@ void LibuvStreamWrap::Initialize(Local target, Local ww = FunctionTemplate::New(isolate, IsConstructCallCallback); ww->InstanceTemplate()->SetInternalFieldCount(WriteWrap::kInternalFieldCount); + // Pre-create the fields that JS attaches to write requests, so that they + // are in-object properties and the object shape stays monomorphic. + ww->InstanceTemplate()->Set(env->oncomplete_string(), v8::Null(isolate)); + ww->InstanceTemplate()->Set(FIXED_ONE_BYTE_STRING(isolate, "callback"), + v8::Null(isolate)); + ww->InstanceTemplate()->Set(env->handle_string(), v8::Null(isolate)); + ww->InstanceTemplate()->Set(FIXED_ONE_BYTE_STRING(isolate, "async"), + v8::False(isolate)); + ww->InstanceTemplate()->Set(FIXED_ONE_BYTE_STRING(isolate, "bytes"), + v8::Integer::New(isolate, 0)); + ww->InstanceTemplate()->Set(env->buffer_string(), v8::Null(isolate)); + // Slot for a completion that fires before JS attaches `oncomplete`; see + // ReportWritesToJSStreamListener::OnStreamAfterReqFinished(). + ww->InstanceTemplate()->Set(env->write_status_string(), v8::Null(isolate)); ww->Inherit(AsyncWrap::GetConstructorTemplate(env)); SetConstructorFunction(context, target, "WriteWrap", ww); env->set_write_wrap_template(ww->InstanceTemplate()); @@ -103,6 +117,7 @@ void LibuvStreamWrap::Initialize(Local target, NODE_DEFINE_CONSTANT(target, kArrayBufferOffset); NODE_DEFINE_CONSTANT(target, kBytesWritten); NODE_DEFINE_CONSTANT(target, kLastWriteWasAsync); + NODE_DEFINE_CONSTANT(target, kLastWriteErr); target ->Set(context, FIXED_ONE_BYTE_STRING(isolate, "streamBaseState"), From f24a495b36a33fc07fbb9800967f612a63fb2741 Mon Sep 17 00:00:00 2001 From: Shelley Vohr Date: Mon, 7 Sep 2026 19:55:48 +0200 Subject: [PATCH 006/217] build: fix quiet default for make builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Makefile documents quiet output unless V=1 and sets `V ?= 0`, but it forwards `V=$(V)` to the gyp-generated makefile, which tests `ifdef V`. "0" is a non-empty value there, so every make build has printed the full compiler command lines regardless. Default V to empty so the sub-make takes its quiet_ rules; `make V=1` and V=1 in the environment stay verbose, and the ninja and cpplint checks already compare against 1. Refs: https://github.com/nodejs/node/pull/26740 Refs: https://github.com/nodejs/build/issues/4419 Signed-off-by: Shelley Vohr PR-URL: https://github.com/nodejs/node/pull/65826 Reviewed-By: Filip Skokan Reviewed-By: James M Snell Reviewed-By: Gürgün Dayıoğlu --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index afa55b2da33e..8dec67750f0c 100644 --- a/Makefile +++ b/Makefile @@ -98,7 +98,7 @@ BUILD_RELEASE_FLAGS ?= $(BUILD_DOWNLOAD_FLAGS) $(BUILD_INTL_FLAGS) # Default to quiet/pretty builds. # To do verbose builds, run `make V=1` or set the V environment variable. -V ?= 0 +V ?= # Use -e to double check in case it's a broken link available-node = \ From 9fd3e6caa41cf5bd804fff4a3582256e1c5684f1 Mon Sep 17 00:00:00 2001 From: Shelley Vohr Date: Mon, 7 Sep 2026 23:21:55 +0200 Subject: [PATCH 007/217] inspector: fix crash when the IsolateData has no platform An embedder that creates its IsolateData without a MultiIsolatePlatform (allowed; node.h says only Workers need one) and keeps the inspector segfaulted on the first `console.log()`, `console.time()` or profiler use after a `node:inspector` session was connected: V8 calls the inspector client's `currentTimeMS()` there, and `NodeInspectorClient::currentTimeMS()` dereferenced `isolate_data()->platform()` unconditionally. Fall back to the wall clock when there is no platform, which is what `NodePlatform::CurrentClockTimeMillis()` returns anyway. Refs: https://github.com/nodejs/node/pull/21917 Signed-off-by: Shelley Vohr PR-URL: https://github.com/nodejs/node/pull/65818 Reviewed-By: James M Snell Reviewed-By: Chengzhong Wu --- src/inspector_agent.cc | 4 +++- test/cctest/test_environment.cc | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/inspector_agent.cc b/src/inspector_agent.cc index 4bb34e56bcbf..dcec24c16dba 100644 --- a/src/inspector_agent.cc +++ b/src/inspector_agent.cc @@ -808,7 +808,9 @@ class NodeInspectorClient : public V8InspectorClient { } double currentTimeMS() override { - return env_->isolate_data()->platform()->CurrentClockTimeMillis(); + MultiIsolatePlatform* platform = env_->isolate_data()->platform(); + if (platform == nullptr) return GetCurrentTimeInMicroseconds() / 1000; + return platform->CurrentClockTimeMillis(); } std::unique_ptr resourceNameToUrl( diff --git a/test/cctest/test_environment.cc b/test/cctest/test_environment.cc index 449d7d38750b..78eb5a79a1dc 100644 --- a/test/cctest/test_environment.cc +++ b/test/cctest/test_environment.cc @@ -698,6 +698,34 @@ TEST_F(EnvironmentTest, InspectorMultipleEmbeddedEnvironments) { CHECK_EQ(data.extracted_value, 42); CHECK_EQ(from_inspector->IntegerValue(context).FromJust(), 42); } + +TEST_F(EnvironmentTest, InspectorWithoutPlatform) { + const v8::HandleScope handle_scope(isolate_); + const Argv argv; + node::IsolateData* isolate_data = node::CreateIsolateData( + isolate_, &NodeTestFixture::current_loop, nullptr); + v8::Local context = node::NewContext(isolate_); + v8::Context::Scope context_scope(context); + std::vector args(*argv, *argv + 1); + node::Environment* env = + node::CreateEnvironment(isolate_data, context, args, args); + CHECK_NOT_NULL(env); + + v8::Local result = + node::LoadEnvironment(env, + "const { Session } = require('inspector');\n" + "const session = new Session();\n" + "session.connect();\n" + "console.time('t'); console.timeEnd('t');\n" + "session.disconnect();\n" + "return 42;") + .ToLocalChecked(); + EXPECT_EQ(result->Int32Value(context).FromJust(), 42); + + node::FreeEnvironment(env); + node::FreeIsolateData(isolate_data); +} + #endif // HAVE_INSPECTOR TEST_F(EnvironmentTest, ExitHandlerTest) { From 830ca7df7ac79c428a69bf22b73850805b7042f0 Mon Sep 17 00:00:00 2001 From: Shelley Vohr Date: Mon, 7 Sep 2026 23:32:59 +0200 Subject: [PATCH 008/217] src: fix null pointer call when running without a startup snapshot Without a startup snapshot (`--no-node-snapshot`, a `--without-node-snapshot` build, or an embedder Environment that was bootstrapped from scratch) starting a Worker made a member call through a null `SnapshotData*`, and so did `NodeMainInstance` while setting itself up. It only worked because the function called does not touch `this`; UBSan reports it for every such Worker. The call existed because `IsolateData::CreateIsolateData()` took an `EmbedderSnapshotData*` and unwrapped it straight away, so the two internal callers wrapped their possibly-null `SnapshotData*` with `AsEmbedderWrapper()` only for it to be unwrapped again. Let the internal function take the `SnapshotData*` itself, unwrap in the public `CreateIsolateData()` only, and drop `AsEmbedderWrapper()`, which has no other users. Refs: https://github.com/nodejs/node/pull/47731 Signed-off-by: Shelley Vohr PR-URL: https://github.com/nodejs/node/pull/65820 Reviewed-By: James M Snell Reviewed-By: Chengzhong Wu --- src/api/environment.cc | 6 +++++- src/env.cc | 4 +--- src/env.h | 3 +-- src/node_main_instance.cc | 10 +++++----- src/node_snapshotable.cc | 4 ---- src/node_worker.cc | 14 +++++++------- test/cctest/test_environment.cc | 12 ++++++++++++ 7 files changed, 31 insertions(+), 22 deletions(-) diff --git a/src/api/environment.cc b/src/api/environment.cc index 67a92f1cdc05..7e4c1c341f11 100644 --- a/src/api/environment.cc +++ b/src/api/environment.cc @@ -391,7 +391,11 @@ IsolateData* CreateIsolateData( ArrayBufferAllocator* allocator, const EmbedderSnapshotData* embedder_snapshot_data) { return IsolateData::CreateIsolateData( - isolate, loop, platform, allocator, embedder_snapshot_data); + isolate, + loop, + platform, + allocator, + SnapshotData::FromEmbedderWrapper(embedder_snapshot_data)); } void FreeIsolateData(IsolateData* isolate_data) { diff --git a/src/env.cc b/src/env.cc index bac9a87f490f..d35c28fcf7ed 100644 --- a/src/env.cc +++ b/src/env.cc @@ -600,10 +600,8 @@ IsolateData* IsolateData::CreateIsolateData( uv_loop_t* loop, MultiIsolatePlatform* platform, ArrayBufferAllocator* allocator, - const EmbedderSnapshotData* embedder_snapshot_data, + const SnapshotData* snapshot_data, std::shared_ptr options) { - const SnapshotData* snapshot_data = - SnapshotData::FromEmbedderWrapper(embedder_snapshot_data); if (options == nullptr) { options = per_process::cli_options->per_isolate->Clone(); } diff --git a/src/env.h b/src/env.h index 6a94dae269d2..29754ddb6371 100644 --- a/src/env.h +++ b/src/env.h @@ -143,7 +143,7 @@ class NODE_EXTERN_PRIVATE IsolateData : public MemoryRetainer { uv_loop_t* event_loop, MultiIsolatePlatform* platform = nullptr, ArrayBufferAllocator* node_allocator = nullptr, - const EmbedderSnapshotData* embedder_snapshot_data = nullptr, + const SnapshotData* snapshot_data = nullptr, std::shared_ptr options = nullptr); ~IsolateData(); @@ -662,7 +662,6 @@ struct SnapshotData { static bool FromBlob(SnapshotData* out, std::string_view in); static const SnapshotData* FromEmbedderWrapper( const EmbedderSnapshotData* data); - EmbedderSnapshotData::Pointer AsEmbedderWrapper() const; ~SnapshotData(); }; diff --git a/src/node_main_instance.cc b/src/node_main_instance.cc index 6f674df3ed0d..bc07d6aaf426 100644 --- a/src/node_main_instance.cc +++ b/src/node_main_instance.cc @@ -51,11 +51,11 @@ NodeMainInstance::NodeMainInstance(const SnapshotData* snapshot_data, // If the indexes are not nullptr, we are not deserializing isolate_data_.reset( - CreateIsolateData(isolate_, - event_loop, - platform, - array_buffer_allocator_.get(), - snapshot_data->AsEmbedderWrapper().get())); + IsolateData::CreateIsolateData(isolate_, + event_loop, + platform, + array_buffer_allocator_.get(), + snapshot_data)); isolate_data_->max_young_gen_size = isolate_params_->constraints.max_young_generation_size_in_bytes(); diff --git a/src/node_snapshotable.cc b/src/node_snapshotable.cc index e861e499534c..37123682dd16 100644 --- a/src/node_snapshotable.cc +++ b/src/node_snapshotable.cc @@ -628,10 +628,6 @@ const SnapshotData* SnapshotData::FromEmbedderWrapper( return data != nullptr ? data->impl_ : nullptr; } -EmbedderSnapshotData::Pointer SnapshotData::AsEmbedderWrapper() const { - return EmbedderSnapshotData::Pointer{new EmbedderSnapshotData(this, false)}; -} - bool SnapshotData::FromFile(SnapshotData* out, FILE* in) { return FromBlob(out, ReadFileSync(in)); } diff --git a/src/node_worker.cc b/src/node_worker.cc index f1a6e96e7f05..5617fd34b3f9 100644 --- a/src/node_worker.cc +++ b/src/node_worker.cc @@ -204,13 +204,13 @@ class WorkerThreadData { isolate->SetStackLimit(w->stack_base_); HandleScope handle_scope(isolate); - isolate_data_.reset(IsolateData::CreateIsolateData( - isolate, - &loop_, - w_->platform_, - allocator.get(), - w->snapshot_data()->AsEmbedderWrapper().get(), - std::move(w_->per_isolate_opts_))); + isolate_data_.reset( + IsolateData::CreateIsolateData(isolate, + &loop_, + w_->platform_, + allocator.get(), + w->snapshot_data(), + std::move(w_->per_isolate_opts_))); CHECK(isolate_data_); CHECK(!isolate_data_->is_building_snapshot()); isolate_data_->set_worker_context(w_); diff --git a/test/cctest/test_environment.cc b/test/cctest/test_environment.cc index 78eb5a79a1dc..c6c479a3a609 100644 --- a/test/cctest/test_environment.cc +++ b/test/cctest/test_environment.cc @@ -355,6 +355,18 @@ TEST_F(EnvironmentTest, MultipleEnvironmentsPerIsolate) { EXPECT_TRUE(called_cb_2); } +TEST_F(EnvironmentTest, WorkerInEnvironmentWithoutSnapshot) { + const v8::HandleScope handle_scope(isolate_); + const Argv argv; + Env env{handle_scope, argv}; + CHECK_NULL(isolate_data_->snapshot_data()); + node::LoadEnvironment(*env, + "const { Worker } = require('worker_threads');" + "new Worker('process.exit(0)', { eval: true });") + .ToLocalChecked(); + EXPECT_EQ(node::SpinEventLoop(*env).FromJust(), 0); +} + TEST_F(EnvironmentTest, NoEnvironmentSanity) { const v8::HandleScope handle_scope(isolate_); v8::Local context = v8::Context::New(isolate_); From 84ecb21343a34bb5f5b2d8a212c6b44880d5b56f Mon Sep 17 00:00:00 2001 From: Christian Aurich Date: Mon, 7 Sep 2026 20:22:00 -0300 Subject: [PATCH 009/217] quic: reject zero addressLRUSize SocketAddressLRU::Upsert always inserts an entry before evicting down to max_size_. With max_size_ == 0, it evicts the entry it just inserted and then accesses the now-missing key via map_[address]->second. operator[] recreates the key with a default-constructed std::list iterator, which is then dereferenced. This is undefined behavior, observed as a SIGSEGV in Endpoint::Receive on the first UDP packet accepted by a QuicEndpoint constructed with { addressLRUSize: 0 }. SocketAddressLRU has no useful semantics for a zero-capacity cache, and Upsert's callers rely on it returning a valid pointer. Reject 0 (and 0n) at the options-parsing boundary instead of changing Upsert's contract. Signed-off-by: Christian Aurich PR-URL: https://github.com/nodejs/node/pull/65827 Reviewed-By: James M Snell Reviewed-By: Tim Perry Reviewed-By: Trivikram Kamat --- doc/api/quic.md | 4 ++-- src/quic/endpoint.cc | 11 +++++++++++ test/parallel/test-quic-internal-endpoint-options.mjs | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/doc/api/quic.md b/doc/api/quic.md index c99cd5d9314c..77b014fbc23f 100644 --- a/doc/api/quic.md +++ b/doc/api/quic.md @@ -2675,8 +2675,8 @@ added: v23.8.0 The endpoint maintains an internal cache of validated socket addresses as a performance optimization. This option sets the maximum number of addresses -that are cached. This is an advanced option that users typically won't have -need to specify. +that are cached. The value must be greater than `0`. This is an advanced option +that users typically won't have need to specify. #### `endpointOptions.disableStatelessReset` diff --git a/src/quic/endpoint.cc b/src/quic/endpoint.cc index a6a4109509ae..8a92986c5a9c 100644 --- a/src/quic/endpoint.cc +++ b/src/quic/endpoint.cc @@ -249,6 +249,17 @@ Maybe Endpoint::Options::From(Environment* env, return Nothing(); } + // SocketAddressLRU::Upsert requires a positive capacity. With max_size_ == + // 0, the newly inserted entry is immediately evicted, and the final + // map_[address] creates a default list iterator that is then + // dereferenced, causing UB (observed as a SIGSEGV in Endpoint::Receive on + // the first accepted connection). + if (options.address_lru_size == 0) { + THROW_ERR_INVALID_ARG_VALUE( + env, "The addressLRUSize option must be greater than 0"); + return Nothing(); + } + Local address; if (!params->Get(env->context(), env->address_string()).ToLocal(&address)) { return Nothing(); diff --git a/test/parallel/test-quic-internal-endpoint-options.mjs b/test/parallel/test-quic-internal-endpoint-options.mjs index 29529c05e613..a81832c71094 100644 --- a/test/parallel/test-quic-internal-endpoint-options.mjs +++ b/test/parallel/test-quic-internal-endpoint-options.mjs @@ -56,7 +56,7 @@ const cases = [ valid: [ 1, 10, 100, 1000, 10000, 10000n, ], - invalid: [-1, -1n, 'a', null, false, true, {}, [], () => {}] + invalid: [-1, -1n, 0, 0n, 'a', null, false, true, {}, [], () => {}] }, { key: 'retryRate', From 79331f577924be6b04a812ac4008358585415673 Mon Sep 17 00:00:00 2001 From: Shelley Vohr Date: Tue, 8 Sep 2026 03:33:06 +0200 Subject: [PATCH 010/217] doc: note that FreeEnvironment() runs a shared event loop The embedder docs say each Environment has exactly one `uv_loop_t` and that an `IsolateData` can be shared between Environments, and `CreateIsolateData()` takes the loop, so several same-thread Environments naturally end up on one loop. Nothing mentions that `FreeEnvironment()` then runs that loop, with JavaScript disallowed on the isolate, until the freed Environment's handles are gone, so the other Environments' callbacks can fire inside it. Document that, and point at it from `FreeEnvironment()` in node.h. Signed-off-by: Shelley Vohr PR-URL: https://github.com/nodejs/node/pull/65691 Reviewed-By: Filip Skokan --- doc/api/embedding.md | 10 ++++++++++ src/node.h | 3 +++ 2 files changed, 13 insertions(+) diff --git a/doc/api/embedding.md b/doc/api/embedding.md index 0309e1120969..dfb84b49ef9b 100644 --- a/doc/api/embedding.md +++ b/doc/api/embedding.md @@ -93,6 +93,16 @@ to as `node::Environment`. Each `node::Environment` is associated with: that `node::IsolateData` is shared only among `node::Environment`s that use the same `v8::Isolate`, Node.js does not perform this check. +`node::Environment`s that share a `node::IsolateData` also share its +`uv_loop_t`. `node::FreeEnvironment()` runs that loop until the handles of the +`node::Environment` being freed have closed, and JavaScript execution is +disallowed on the whole `v8::Isolate` while it does, so pending timers, I/O +callbacks and thread pool completions that belong to other `node::Environment`s +on the same loop can run inside that call without being able to call into +JavaScript. `node::Environment`s that are freed independently of one another +should each use their own `uv_loop_t` and `node::IsolateData`, or the embedder +should make sure the others have no pending work when one of them is freed. + In order to set up a `v8::Isolate`, an `v8::ArrayBuffer::Allocator` needs to be provided. One possible choice is the default Node.js allocator, which can be created through `node::ArrayBufferAllocator::Create()`. Using the Node.js diff --git a/src/node.h b/src/node.h index 5e54a60fcf53..d6fc600864e1 100644 --- a/src/node.h +++ b/src/node.h @@ -882,6 +882,9 @@ NODE_EXTERN v8::MaybeLocal LoadEnvironment( const ModuleData* entry_point, EmbedderPreloadCallback preload = nullptr); +// Runs `env`'s event loop until its handles have closed, with JavaScript +// execution disallowed on the isolate; see doc/api/embedding.md if that loop +// is shared with other Environments. NODE_EXTERN void FreeEnvironment(Environment* env); // Set a callback that is called when process.exit() is called from JS, From 62fe96c166e65f641d89ba77c7478aa3082abf18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=9D=B4=ED=98=9C=EB=AF=B8?= <103042868+hyemimi@users.noreply.github.com> Date: Tue, 8 Sep 2026 11:34:31 +0900 Subject: [PATCH 011/217] typings: add missing sea binding properties Signed-off-by: hyemimi PR-URL: https://github.com/nodejs/node/pull/65815 Reviewed-By: Daeyeon Jeong Reviewed-By: James M Snell --- typings/internalBinding/sea.d.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/typings/internalBinding/sea.d.ts b/typings/internalBinding/sea.d.ts index 15f4430d87d1..258e7f112ed0 100644 --- a/typings/internalBinding/sea.d.ts +++ b/typings/internalBinding/sea.d.ts @@ -1,5 +1,8 @@ export interface SeaBinding { getAsset(key: string): ArrayBuffer | undefined; + getAssetKeys(): string[]; isExperimentalSeaWarningNeeded(): boolean; isSea(): boolean; + isVfsEnabled(): boolean; + mainCodePath?: string; } From c5c7036e7be1f7ec79c40c662066a17b655f2ab4 Mon Sep 17 00:00:00 2001 From: greenhead Date: Tue, 8 Sep 2026 11:34:41 +0900 Subject: [PATCH 012/217] typings: add timeoutInfo to TimersBinding Signed-off-by: greenhead PR-URL: https://github.com/nodejs/node/pull/65811 Refs: https://github.com/nodejs/node/pull/46014 Reviewed-By: Daeyeon Jeong Reviewed-By: James M Snell --- typings/internalBinding/timers.d.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/typings/internalBinding/timers.d.ts b/typings/internalBinding/timers.d.ts index e03deeeca240..ccdee1db5044 100644 --- a/typings/internalBinding/timers.d.ts +++ b/typings/internalBinding/timers.d.ts @@ -5,4 +5,5 @@ export interface TimersBinding { toggleTimerRef(value: boolean): void; toggleImmediateRef(value: boolean): void; immediateInfo: Uint32Array; + timeoutInfo: Int32Array; } From 2b502e0798fea2d259c846abd6165893f36f897a Mon Sep 17 00:00:00 2001 From: "Sparsh :)" <76697238+SparshGarg999@users.noreply.github.com> Date: Tue, 8 Sep 2026 10:23:09 +0530 Subject: [PATCH 013/217] fs: support removing read-only files in rmSync on Windows On Windows, libc++ std::filesystem::remove and remove_all do not automatically clear the read-only attribute before deleting a file (unlike MSVC STL). This causes fs.rmSync to fail with EPERM when trying to remove read-only files in environments where Node.js is built using clang libc++ (such as Electron). This commit introduces a Windows-specific helper ClearReadOnlyAttributeW which clears the FILE_ATTRIBUTE_READONLY attribute recursively (or for a single file) when operation_not_permitted is returned, allowing rmSync to successfully delete read-only files/folders. Fixes: https://github.com/nodejs/node/issues/64374 Signed-off-by: SparshGarg999 PR-URL: https://github.com/nodejs/node/pull/64453 Fixes: https://github.com/nodejs/node/issues/64374 Reviewed-By: Stefan Stojanovic --- src/node_file.cc | 54 +++++++++++++++++++++++++++++++++++++ test/parallel/test-fs-rm.js | 27 +++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/src/node_file.cc b/src/node_file.cc index b047c5a8e6a7..0179ad12c9d8 100644 --- a/src/node_file.cc +++ b/src/node_file.cc @@ -1763,6 +1763,41 @@ static void RMDir(const FunctionCallbackInfo& args) { } } +#ifdef _WIN32 +static void ClearReadOnlyAttributeWHelper(const wchar_t* path) { + DWORD attrs = GetFileAttributesW(path); + if (attrs != INVALID_FILE_ATTRIBUTES && (attrs & FILE_ATTRIBUTE_READONLY)) { + SetFileAttributesW(path, attrs & ~FILE_ATTRIBUTE_READONLY); + } +} + +static void ClearReadOnlyAttributeW(const std::filesystem::path& path, + bool recursive) { + std::error_code ec; + auto file_status = std::filesystem::symlink_status(path, ec); + if (ec) return; + + if (recursive && + file_status.type() == std::filesystem::file_type::directory) { + for (const auto& entry : std::filesystem::recursive_directory_iterator( + path, + std::filesystem::directory_options::skip_permission_denied, + ec)) { + std::error_code entry_ec; + auto entry_status = entry.symlink_status(entry_ec); + if (entry_ec) continue; + if (entry_status.type() != std::filesystem::file_type::symlink) { + ClearReadOnlyAttributeWHelper(entry.path().c_str()); + } + } + } + + if (file_status.type() != std::filesystem::file_type::symlink) { + ClearReadOnlyAttributeWHelper(path.c_str()); + } +} +#endif + static void RmSync(const FunctionCallbackInfo& args) { Environment* env = Environment::GetCurrent(args); Isolate* isolate = env->isolate(); @@ -1810,6 +1845,9 @@ static void RmSync(const FunctionCallbackInfo& args) { }; int i = 1; +#ifdef _WIN32 + bool cleared_readonly = false; +#endif while (maxRetries >= 0) { if (recursive) { @@ -1818,6 +1856,22 @@ static void RmSync(const FunctionCallbackInfo& args) { std::filesystem::remove(file_path, error); } +#ifdef _WIN32 + // On Windows, libc++ does not clear the read-only attribute before + // removing a file (unlike MSVC STL which does). Attempt to clear it + // manually when we get EPERM (operation_not_permitted) so that read-only + // files can be deleted, matching the behavior of official Node.js builds. + if (error == std::errc::operation_not_permitted && !cleared_readonly) { + cleared_readonly = true; + ClearReadOnlyAttributeW(file_path, recursive); + if (recursive) { + std::filesystem::remove_all(file_path, error); + } else { + std::filesystem::remove(file_path, error); + } + } +#endif // _WIN32 + if (!error || error == std::errc::no_such_file_or_directory) { return; } else if (!can_omit_error(error)) { diff --git a/test/parallel/test-fs-rm.js b/test/parallel/test-fs-rm.js index e92bf8c07971..1d0578e3004d 100644 --- a/test/parallel/test-fs-rm.js +++ b/test/parallel/test-fs-rm.js @@ -631,3 +631,30 @@ if (isGitPresent) { } } } + +{ + // Test that rmSync can delete read-only files (and directories containing read-only files recursively) + const dirname = nextDirPath(); + const filePath = path.join(dirname, 'readonly-file.txt'); + const recursiveDir = path.join(dirname, 'subdir'); + const recursiveFilePath = path.join(recursiveDir, 'readonly-nested.txt'); + + fs.mkdirSync(recursiveDir, { recursive: true }); + fs.writeFileSync(filePath, 'hello'); + fs.writeFileSync(recursiveFilePath, 'world'); + + // Make files read-only + fs.chmodSync(filePath, 0o444); + fs.chmodSync(recursiveFilePath, 0o444); + + // rmSync without recursive option on a file + fs.rmSync(filePath); + assert.strictEqual(fs.existsSync(filePath), false); + + // rmSync with recursive option on a directory containing a read-only file + fs.rmSync(recursiveDir, { recursive: true }); + assert.strictEqual(fs.existsSync(recursiveDir), false); + + // Clean up parent directory + fs.rmSync(dirname, { recursive: true, force: true }); +} From 5b6ac71fb2dcd5c2dca56bf56817999bfc9828d9 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Tue, 8 Sep 2026 10:50:21 +0200 Subject: [PATCH 014/217] lib: use Web IDL interface brand checks Use explicit brand predicates for interface conversion instead of prototype ancestry. Update CryptoKey and AbortSignal together with the shared converter contract. Read internal AbortSignal state during composition. Preserve genuine signals after prototype changes without invoking shadowed getters. Signed-off-by: Filip Skokan Assisted-by: GitHub Copilot PR-URL: https://github.com/nodejs/node/pull/65846 Reviewed-By: James M Snell Reviewed-By: Aviv Keller --- lib/internal/abort_controller.js | 31 ++++++------ lib/internal/crypto/webidl.js | 4 +- lib/internal/streams/iter/webidl.js | 7 ++- lib/internal/webidl.js | 7 ++- test/parallel/test-internal-webidl.js | 48 +++++++++++++++++-- .../test-webcrypto-cryptokey-brand-check.js | 5 +- test/parallel/test-webcrypto-webidl-brand.js | 38 +++++++++++++++ 7 files changed, 111 insertions(+), 29 deletions(-) create mode 100644 test/parallel/test-webcrypto-webidl-brand.js diff --git a/lib/internal/abort_controller.js b/lib/internal/abort_controller.js index 09b160e9fe5a..48dcdaafba2f 100644 --- a/lib/internal/abort_controller.js +++ b/lib/internal/abort_controller.js @@ -149,8 +149,8 @@ function refreshCompositeSignal(signal) { continue; } - if (sourceSignal.aborted) { - abortSignal(signal, sourceSignal.reason); + if (sourceSignal[kAborted]) { + abortSignal(signal, sourceSignal[kReason]); return; } } @@ -170,8 +170,8 @@ function followCompositeSignal(signal) { continue; } - if (sourceSignal.aborted) { - abortSignal(signal, sourceSignal.reason); + if (sourceSignal[kAborted]) { + abortSignal(signal, sourceSignal[kReason]); return; } @@ -217,6 +217,14 @@ function setWeakAbortSignalTimeout(weakRef, delay) { } class AbortSignal extends EventTarget { + #brand; + + static { + converters.AbortSignal = createInterfaceConverter( + 'AbortSignal', + (value) => typeof value === 'object' && value !== null && #brand in value, + ); + } /** * @param {symbol | undefined} dontThrowSymbol @@ -337,8 +345,9 @@ class AbortSignal extends EventTarget { gcPersistentSignals.add(signal); } - if (signal.aborted) { - abortSignal(resultSignal, signal.reason); + refreshCompositeSignal(signal); + if (signal[kAborted]) { + abortSignal(resultSignal, signal[kReason]); return resultSignal; } @@ -348,11 +357,6 @@ class AbortSignal extends EventTarget { } else if (!signal[kSourceSignals]) { continue; } else { - refreshCompositeSignal(signal); - if (signal.aborted) { - abortSignal(resultSignal, signal.reason); - return resultSignal; - } for (const sourceSignalWeakRef of signal[kSourceSignals]) { const sourceSignal = sourceSignalWeakRef.deref(); if (!sourceSignal) { @@ -360,8 +364,8 @@ class AbortSignal extends EventTarget { } assert(!sourceSignal[kComposite]); - if (sourceSignal.aborted) { - abortSignal(resultSignal, sourceSignal.reason); + if (sourceSignal[kAborted]) { + abortSignal(resultSignal, sourceSignal[kReason]); return resultSignal; } @@ -466,7 +470,6 @@ class AbortSignal extends EventTarget { } } -converters.AbortSignal = createInterfaceConverter('AbortSignal', AbortSignal.prototype); converters['sequence'] = createSequenceConverter(converters.AbortSignal); function ClonedAbortSignal() { diff --git a/lib/internal/crypto/webidl.js b/lib/internal/crypto/webidl.js index 6ec400256820..b9c65c91cfbd 100644 --- a/lib/internal/crypto/webidl.js +++ b/lib/internal/crypto/webidl.js @@ -18,10 +18,10 @@ const { const { isUint32, } = require('internal/validators'); -const { CryptoKey } = require('internal/crypto/webcrypto'); const { getCryptoKeyAlgorithm, getCryptoKeyType, + isCryptoKey, } = require('internal/crypto/keys'); const { bigIntArrayToUnsignedInt, @@ -642,7 +642,7 @@ converters.AesCtrParams = createDictionaryConverter( ]); converters.CryptoKey = createInterfaceConverter( - 'CryptoKey', CryptoKey.prototype); + 'CryptoKey', isCryptoKey); converters.EcdhKeyDeriveParams = createDictionaryConverter( 'EcdhKeyDeriveParams', [ diff --git a/lib/internal/streams/iter/webidl.js b/lib/internal/streams/iter/webidl.js index b971fa95071b..273a52443bf2 100644 --- a/lib/internal/streams/iter/webidl.js +++ b/lib/internal/streams/iter/webidl.js @@ -5,10 +5,10 @@ const { convertToInt, createDictionaryConverter, createEnumConverter, - createInterfaceConverter, createSequenceConverter, } = require('internal/webidl'); -const { AbortSignal } = require('internal/abort_controller'); +// Load AbortSignal to register its Web IDL converter in baseConverters. +require('internal/abort_controller'); const { isUint8Array } = require('internal/util/types'); const converters = { __proto__: null }; @@ -38,8 +38,7 @@ function allowStreamBufferOptions(options) { }; } -converters.AbortSignal = createInterfaceConverter( - 'AbortSignal', AbortSignal.prototype); +converters.AbortSignal = baseConverters.AbortSignal; converters.BackpressurePolicy = createEnumConverter('BackpressurePolicy', [ 'strict', 'unbounded', diff --git a/lib/internal/webidl.js b/lib/internal/webidl.js index 71513bbe86f4..2f86117afe9c 100644 --- a/lib/internal/webidl.js +++ b/lib/internal/webidl.js @@ -21,7 +21,6 @@ const { NumberMAX_SAFE_INTEGER, NumberMIN_SAFE_INTEGER, ObjectPrototypeHasOwnProperty, - ObjectPrototypeIsPrototypeOf, SafeArrayIterator, SafeSet, String, @@ -873,13 +872,13 @@ function createSequenceConverter(converter) { * Creates a converter for a Web IDL interface type. * @see https://webidl.spec.whatwg.org/#js-interface * @param {string} name Interface identifier. - * @param {object} prototype Interface prototype object. + * @param {(value: any) => boolean} brandCheck Interface brand predicate. * @returns {Converter} */ -function createInterfaceConverter(name, prototype) { +function createInterfaceConverter(name, brandCheck) { return (V, options = kEmptyObject) => { // Web IDL interface conversion step 1: return V if it implements I. - if (ObjectPrototypeIsPrototypeOf(prototype, V)) { + if (brandCheck(V)) { return V; } // Step 2: otherwise throw. diff --git a/test/parallel/test-internal-webidl.js b/test/parallel/test-internal-webidl.js index bd08648549be..406a625c26ab 100644 --- a/test/parallel/test-internal-webidl.js +++ b/test/parallel/test-internal-webidl.js @@ -1,7 +1,7 @@ // Flags: --expose-internals 'use strict'; -require('../common'); +const common = require('../common'); const assert = require('assert'); const vm = require('vm'); const webidl = require('internal/webidl'); @@ -511,10 +511,16 @@ assert.throws(() => webidl.requiredArguments(1, 2, opts), { } { - class Example {} + class Example { + #brand; + + static is(value) { + return typeof value === 'object' && value !== null && #brand in value; + } + } const converter = webidl.createInterfaceConverter( 'Example', - Example.prototype); + Example.is); const example = new Example(); assert.strictEqual(converter(example), example); @@ -523,6 +529,42 @@ assert.throws(() => webidl.requiredArguments(1, 2, opts), { code: 'ERR_INVALID_ARG_TYPE', message: 'Prefix: Context is not of type Example.', }); + assertInvalidArgType(() => converter({ __proto__: Example.prototype })); + assertInvalidArgType(() => converter(new Proxy(example, {}))); + Object.setPrototypeOf(example, null); + assert.strictEqual(converter(example), example); +} + +{ + const signal = AbortSignal.abort('reason'); + for (const value of [ + Object.create(AbortSignal.prototype, { aborted: { value: false } }), + { __proto__: signal }, + Object.create(AbortSignal.prototype, Object.getOwnPropertyDescriptors(signal)), + new Proxy(signal, {}), + ]) { + assertInvalidArgType(() => converters.AbortSignal(value)); + assertInvalidArgType(() => AbortSignal.any([value])); + } + + Object.setPrototypeOf(signal, null); + assert.strictEqual(converters.AbortSignal(signal), signal); + const composite = AbortSignal.any([signal]); + assert.strictEqual(composite.aborted, true); + assert.strictEqual(composite.reason, 'reason'); +} + +{ + const controller = new AbortController(); + Object.defineProperties(controller.signal, { + aborted: { get: common.mustNotCall('Unexpected aborted getter') }, + reason: { get: common.mustNotCall('Unexpected reason getter') }, + }); + const composite = AbortSignal.any([controller.signal]); + assert.strictEqual(composite.aborted, false); + controller.abort('reason'); + assert.strictEqual(composite.aborted, true); + assert.strictEqual(composite.reason, 'reason'); } { diff --git a/test/parallel/test-webcrypto-cryptokey-brand-check.js b/test/parallel/test-webcrypto-cryptokey-brand-check.js index 3fe8aaa181a2..9dd115f00721 100644 --- a/test/parallel/test-webcrypto-cryptokey-brand-check.js +++ b/test/parallel/test-webcrypto-cryptokey-brand-check.js @@ -54,6 +54,7 @@ const { subtle } = globalThis.crypto; assert.strictEqual(Object.getPrototypeOf(internalProto), CryptoKey.prototype); const invalidThis = { code: 'ERR_INVALID_THIS', name: 'TypeError' }; + const invalidArgType = { code: 'ERR_INVALID_ARG_TYPE', name: 'TypeError' }; // Plain object receiver. Object.entries(getters).forEach(([, getter]) => { @@ -94,10 +95,10 @@ const { subtle } = globalThis.crypto; assert.strictEqual(isCryptoKey(spoofed), false); await assert.rejects( subtle.sign('HMAC', spoofed, Buffer.from('payload')), - invalidThis); + invalidArgType); await assert.rejects( subtle.exportKey('jwk', spoofed), - invalidThis); + invalidArgType); // Subvert `instanceof CryptoKey` via Symbol.hasInstance, then // invoke the native getters on a forged object. The C++ tag diff --git a/test/parallel/test-webcrypto-webidl-brand.js b/test/parallel/test-webcrypto-webidl-brand.js new file mode 100644 index 000000000000..d4d785684906 --- /dev/null +++ b/test/parallel/test-webcrypto-webidl-brand.js @@ -0,0 +1,38 @@ +// Flags: --expose-internals +'use strict'; + +const common = require('../common'); +if (!common.hasCrypto) + common.skip('missing crypto'); + +const assert = require('assert'); +const { subtle } = globalThis.crypto; +const { CryptoKey } = require('internal/crypto/keys'); +const { converters } = require('internal/crypto/webidl'); + +async function main() { + const bytes = new Uint8Array(16); + const key = await subtle.importKey('raw', bytes, 'AES-GCM', true, ['encrypt']); + + for (const value of [ + { __proto__: CryptoKey.prototype }, + { __proto__: key }, + Object.create(CryptoKey.prototype, Object.getOwnPropertyDescriptors(key)), + new Proxy(key, {}), + ]) { + assert.throws(() => converters.CryptoKey(value), { + name: 'TypeError', + code: 'ERR_INVALID_ARG_TYPE', + }); + await assert.rejects(subtle.exportKey('raw', value), { + name: 'TypeError', + code: 'ERR_INVALID_ARG_TYPE', + }); + } + + Object.setPrototypeOf(key, null); + assert.strictEqual(converters.CryptoKey(key), key); + assert.deepStrictEqual(new Uint8Array(await subtle.exportKey('raw', key)), bytes); +} + +main().then(common.mustCall()); From bdacc08fbefe6ad9b1e139f09792c269eaaf05ff Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Tue, 8 Sep 2026 10:50:33 +0200 Subject: [PATCH 015/217] lib: validate sequence iterator objects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reject primitive iterator factory results before accessing next, as required by GetIteratorFromMethod. Signed-off-by: Filip Skokan PR-URL: https://github.com/nodejs/node/pull/65844 Reviewed-By: Jason Zhang Reviewed-By: James M Snell Reviewed-By: Gürgün Dayıoğlu Reviewed-By: Daeyeon Jeong --- lib/internal/webidl.js | 7 +++++- test/parallel/test-internal-webidl.js | 33 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/lib/internal/webidl.js b/lib/internal/webidl.js index 2f86117afe9c..ab13f17f4a0e 100644 --- a/lib/internal/webidl.js +++ b/lib/internal/webidl.js @@ -835,7 +835,12 @@ function createSequenceConverter(converter) { // Step 4 and create-sequence step 1: get the iterator record. const iterator = FunctionPrototypeCall(method, V); - const nextMethod = iterator?.next; + if (type(iterator) !== 'Object') { + throw makeException( + 'cannot be converted to sequence.', + options); + } + const nextMethod = iterator.next; if (typeof nextMethod !== 'function') { throw makeException( 'cannot be converted to sequence.', diff --git a/test/parallel/test-internal-webidl.js b/test/parallel/test-internal-webidl.js index 406a625c26ab..f52896d5210f 100644 --- a/test/parallel/test-internal-webidl.js +++ b/test/parallel/test-internal-webidl.js @@ -510,6 +510,39 @@ assert.throws(() => webidl.requiredArguments(1, 2, opts), { }), []); } +for (const [prototype, value] of [ + [Number.prototype, 1], + [String.prototype, 'iterator'], + [Boolean.prototype, true], + [BigInt.prototype, 1n], + [Symbol.prototype, Symbol()], +]) { + let nextReads = 0; + Object.defineProperty(prototype, 'next', { + configurable: true, + get() { + nextReads++; + return () => ({ done: true }); + }, + }); + try { + const iterable = { [Symbol.iterator]: () => value }; + assertInvalidArgType(() => converters['sequence'](iterable)); + assertInvalidArgType(() => structuredClone(null, { transfer: iterable })); + assert.strictEqual(nextReads, 0); + } finally { + delete prototype.next; + } +} + +{ + function iterator() {} + iterator.next = () => ({ done: true }); + assert.deepStrictEqual(converters['sequence']({ + [Symbol.iterator]: () => iterator, + }), []); +} + { class Example { #brand; From 412399efd32a751a44dd5a661ca477cb59e9a8ae Mon Sep 17 00:00:00 2001 From: Mike McCready <66998419+MikeMcC399@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:24:14 +0200 Subject: [PATCH 016/217] doc: clarify supported Python releases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exclude Python pre-release versions Signed-off-by: Mike McCready <66998419+MikeMcC399@users.noreply.github.com> PR-URL: https://github.com/nodejs/node/pull/65850 Fixes: https://github.com/nodejs/node/issues/65599 Refs: https://github.com/nodejs/node/pull/65750 Reviewed-By: Richard Lau Reviewed-By: Gürgün Dayıoğlu --- BUILDING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILDING.md b/BUILDING.md index e477d46863f4..b46abe291e6d 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -241,7 +241,7 @@ tarball and/or browse the git repository checked out at the relevant tag. ### Prerequisites -* [A supported version of Python][Python versions] for building and testing. +* [A supported version of Python][Python versions] (excludes pre-release versions) for building and testing. * A Rust toolchain if [building Node.js with Temporal support](#building-nodejs-with-temporal-support). * Memory: at least 8GB of RAM is typically required when compiling with 4 parallel jobs (e.g: `make -j4`). From e379c26a9220d9dca6f774702ea24a07873ed8c7 Mon Sep 17 00:00:00 2001 From: Philipp Dunkel Date: Tue, 8 Sep 2026 13:35:41 +0200 Subject: [PATCH 017/217] vfs: align virtual file handles with open(2) A file descriptor obtained on a mounted path answers several `node:fs` calls differently from one on a real file, in both providers: * `writeFileSync(path, data, { flag: 'r+' })` replaces the whole file instead of overwriting bytes from offset 0 and keeping the tail. * Numeric open flags are mapped by treating any write-ish bit as "w": `O_WRONLY` alone truncates, and `O_RDONLY | O_CREAT` opens the file write-only and truncates it. * A handle opened with "a+" starts its read offset at the end of the file, so the first read returns nothing; O_APPEND only affects writes. The ZipProvider handle additionally: * throws EISDIR instead of EBADF when reading a write-only handle or writing a read-only one; * leaves stale bytes in place when `ftruncate` grows a file that was previously shrunk, where real files read back as zeros; * rejects a BigInt `position` with a TypeError from mixing number and BigInt arithmetic. This adds a test that runs the same sequence of calls against a memory mount and a ZIP mount and expects the real-fs result, so every divergence shows up as its own failing case. Proposed solution: decode numeric flags bit by bit (O_TRUNC decides truncation, O_CREAT decides creation, O_WRONLY/O_RDWR decide access) instead of collapsing them to a flag string; keep the read offset at 0 for append handles and only force writes to the end; make the handle `writeFile` for non-truncating flags write at offset 0 without shrinking; in the ZIP handle use EBADF for access-mode violations, zero-fill on growth in `#doTruncate`, and coerce `position` with `Number()` as the memory handle does. Signed-off-by: Philipp Dunkel PR-URL: https://github.com/nodejs/node/pull/65854 Reviewed-By: James M Snell Reviewed-By: Matteo Collina --- lib/internal/vfs/file_handle.js | 107 +++++++++------- lib/internal/vfs/providers/memory.js | 59 ++------- lib/internal/vfs/providers/ziparchive.js | 106 +++++++--------- test/parallel/test-vfs-handle-semantics.js | 115 ++++++++++++++++++ test/parallel/test-vfs-zip-provider-handle.js | 14 ++- 5 files changed, 240 insertions(+), 161 deletions(-) create mode 100644 test/parallel/test-vfs-handle-semantics.js diff --git a/lib/internal/vfs/file_handle.js b/lib/internal/vfs/file_handle.js index 7b60c9def2b5..a65fe5f99386 100644 --- a/lib/internal/vfs/file_handle.js +++ b/lib/internal/vfs/file_handle.js @@ -27,6 +27,52 @@ const kFlags = Symbol('kFlags'); const kMode = Symbol('kMode'); const kPosition = Symbol('kPosition'); const kClosed = Symbol('kClosed'); +const kAccess = Symbol('kAccess'); + +const { stringToFlags } = require('internal/fs/utils'); +const { + fs: { O_APPEND, O_CREAT, O_EXCL, O_RDONLY, O_RDWR, O_TRUNC, O_WRONLY }, +} = internalBinding('constants'); + +/** + * Decodes open flags into what they ask for. Both the string spellings and + * the numeric `fs.constants` values are accepted. The bits decide, because + * several numeric combinations (`O_WRONLY` alone, `O_RDONLY | O_CREAT`) + * have no string spelling, and collapsing them to the nearest one changes + * their meaning: a plain `O_WRONLY` must neither create nor truncate. + * @param {string|number} flags + * @returns {{ readable: boolean, writable: boolean, create: boolean, + * exclusive: boolean, truncate: boolean, append: boolean }} + */ +function decodeOpenFlags(flags) { + const bits = typeof flags === 'number' ? flags : stringToFlags(flags); + const access = bits & (O_RDONLY | O_WRONLY | O_RDWR); + return { + __proto__: null, + readable: access !== O_WRONLY, + writable: access !== O_RDONLY, + create: (bits & O_CREAT) !== 0, + exclusive: (bits & O_EXCL) !== 0, + truncate: (bits & O_TRUNC) !== 0, + append: (bits & O_APPEND) !== 0, + }; +} + +/** + * The string spelling closest to numeric flags, for `handle.flags`, which + * has always been a string. Behaviour is never derived from it. + * @param {string|number} flags + * @returns {string} + */ +function flagsToString(flags) { + if (typeof flags !== 'number') return flags; + const { readable, writable, exclusive, truncate, append } = decodeOpenFlags(flags); + const plus = readable && writable ? '+' : ''; + const x = exclusive ? 'x' : ''; + if (append) return `a${x}${plus}`; + if (truncate) return `w${x}${plus}`; + return writable ? 'r+' : 'r'; +} function isCurrentPosition(position) { return position === null || position === undefined || position === -1; @@ -44,7 +90,8 @@ class VirtualFileHandle { */ constructor(path, flags, mode) { this[kPath] = path; - this[kFlags] = flags; + this[kAccess] = decodeOpenFlags(flags); + this[kFlags] = flagsToString(flags); this[kMode] = mode ?? 0o644; this[kPosition] = 0; this[kClosed] = false; @@ -387,19 +434,15 @@ class MemoryFileHandle extends VirtualFileHandle { this.#entry = entry; this.#getStats = getStats; - // Handle different open modes - if (flags === 'w' || flags === 'w+' || - flags === 'wx' || flags === 'wx+') { - // Write mode: truncate + // O_TRUNC empties the file at open time. O_APPEND does not move the + // read offset: it only forces writes to the end, so the position stays + // at 0 and reads start from the beginning as they do on a real file. + if (this[kAccess].truncate) { this.#content = Buffer.alloc(0); this.#size = 0; if (entry) { entry.content = this.#content; } - } else if (flags === 'a' || flags === 'a+' || - flags === 'ax' || flags === 'ax+') { - // Append mode: position at end - this.position = this.#size; } } @@ -407,7 +450,7 @@ class MemoryFileHandle extends VirtualFileHandle { * Throws EBADF if the handle was not opened for writing. */ #checkWritable() { - if (this.flags === 'r') { + if (!this[kAccess].writable) { throw createEBADF('write'); } } @@ -416,8 +459,7 @@ class MemoryFileHandle extends VirtualFileHandle { * Throws EBADF if the handle was not opened for reading. */ #checkReadable() { - const f = this.flags; - if (f === 'w' || f === 'a' || f === 'wx' || f === 'ax') { + if (!this[kAccess].readable) { throw createEBADF('read'); } } @@ -427,8 +469,7 @@ class MemoryFileHandle extends VirtualFileHandle { * @returns {boolean} */ #isAppend() { - const f = this.flags; - return f === 'a' || f === 'a+' || f === 'ax' || f === 'ax+'; + return this[kAccess].append; } /** @@ -605,42 +646,16 @@ class MemoryFileHandle extends VirtualFileHandle { } /** - * Writes data to the file synchronously. - * Replaces content in 'w' mode, appends in 'a' mode. + * Writes data to the file synchronously, from the current position (the + * end, in append mode), the way `filehandle.writeFile()` does. Whether + * earlier content is discarded was decided by the open flags: "w" has + * already truncated, "r+" overwrites in place and keeps any tail. * @param {Buffer|string} data The data to write * @param {object} [options] Options */ writeFileSync(data, options) { - this.#checkClosed('write'); - this.#checkWritable(); - const buffer = typeof data === 'string' ? Buffer.from(data, options?.encoding) : data; - - // In append mode, append to existing content - if (this.#isAppend()) { - const neededSize = this.#size + buffer.length; - if (neededSize > this.#content.length) { - const newCapacity = MathMax(neededSize, this.#content.length * 2); - const newContent = Buffer.alloc(newCapacity); - this.#content.copy(newContent, 0, 0, this.#size); - this.#content = newContent; - } - buffer.copy(this.#content, this.#size); - this.#size = neededSize; - } else { - this.#content = Buffer.from(buffer); - this.#size = buffer.length; - } - - // Update the entry's content, mtime, and ctime - if (this.#entry) { - const now = DateNow(); - this.#entry.content = this.#content.subarray(0, this.#size); - this.#entry.mtime = now; - this.#entry.ctime = now; - } - - this.position = this.#size; + this.writeSync(buffer, 0, buffer.length, null); } /** @@ -721,4 +736,6 @@ class MemoryFileHandle extends VirtualFileHandle { module.exports = { VirtualFileHandle, MemoryFileHandle, + decodeOpenFlags, + kAccess, }; diff --git a/lib/internal/vfs/providers/memory.js b/lib/internal/vfs/providers/memory.js index ce59a0611616..2fd9a7d78cae 100644 --- a/lib/internal/vfs/providers/memory.js +++ b/lib/internal/vfs/providers/memory.js @@ -16,7 +16,7 @@ const { Buffer } = require('buffer'); const { isPromise } = require('util/types'); const { posix: pathPosix } = require('path'); const { VirtualProvider } = require('internal/vfs/provider'); -const { MemoryFileHandle } = require('internal/vfs/file_handle'); +const { MemoryFileHandle, decodeOpenFlags } = require('internal/vfs/file_handle'); const { VFSWatcher, VFSStatWatcher, @@ -46,45 +46,12 @@ const { Dirent } = require('internal/fs/utils'); const { kEmptyObject } = require('internal/util'); const { fs: { - O_APPEND, - O_CREAT, - O_EXCL, - O_RDWR, - O_TRUNC, - O_WRONLY, UV_DIRENT_FILE, UV_DIRENT_DIR, UV_DIRENT_LINK, }, } = internalBinding('constants'); -/** - * Converts numeric flags to a string representation. - * If already a string, returns as-is. - * @param {string|number} flags The flags to normalize - * @returns {string} Normalized string flags - */ -function normalizeFlags(flags) { - if (typeof flags === 'string') return flags; - if (typeof flags !== 'number') return 'r'; - - const rdwr = (flags & O_RDWR) !== 0; - const append = (flags & O_APPEND) !== 0; - const excl = (flags & O_EXCL) !== 0; - const write = (flags & O_WRONLY) !== 0 || - (flags & O_CREAT) !== 0 || - (flags & O_TRUNC) !== 0; - - if (append) { - return 'a' + (excl ? 'x' : '') + (rdwr ? '+' : ''); - } - if (write) { - return 'w' + (excl ? 'x' : '') + (rdwr ? '+' : ''); - } - if (rdwr) return 'r+'; - return 'r'; -} - /** * Converts a time argument (Date, number, or string) to milliseconds. * Numbers are treated as seconds (matching Node.js utimes convention). @@ -496,33 +463,23 @@ class MemoryProvider extends VirtualProvider { openSync(path, flags, mode) { const normalized = this.#normalizePath(path); + const access = decodeOpenFlags(flags); - // Normalize numeric flags to string - flags = normalizeFlags(flags); - - // Handle create and exclusive modes - const isCreate = flags === 'w' || flags === 'w+' || - flags === 'a' || flags === 'a+' || - flags === 'wx' || flags === 'wx+' || - flags === 'ax' || flags === 'ax+'; - const isExclusive = flags === 'wx' || flags === 'wx+' || - flags === 'ax' || flags === 'ax+'; - const isWritable = flags !== 'r'; - - // Check readonly for any writable mode - if (this.readonly && isWritable) { + // Creating a file is a write even when the handle itself is read-only. + if (this.readonly && (access.writable || access.create)) { throw createEROFS('open', path); } let entry; try { entry = this.#getEntry(normalized, 'open'); - // Exclusive flag: file must not exist - if (isExclusive) { + // O_EXCL only means anything together with O_CREAT: the file must + // not exist yet. + if (access.create && access.exclusive) { throw createEEXIST('open', path); } } catch (err) { - if (err.code !== 'ENOENT' || !isCreate) throw err; + if (err.code !== 'ENOENT' || !access.create) throw err; // Create the file const parent = this.#ensureParent(normalized, false, 'open'); const name = pathPosix.basename(normalized); diff --git a/lib/internal/vfs/providers/ziparchive.js b/lib/internal/vfs/providers/ziparchive.js index f369cdd22f76..402c92b8b659 100644 --- a/lib/internal/vfs/providers/ziparchive.js +++ b/lib/internal/vfs/providers/ziparchive.js @@ -5,6 +5,7 @@ const { ArrayPrototypePush, MathMax, MathMin, + Number, StringPrototypeIndexOf, StringPrototypeSlice, StringPrototypeStartsWith, @@ -20,8 +21,13 @@ const { }, } = require('internal/errors'); const { VirtualProvider } = require('internal/vfs/provider'); -const { VirtualFileHandle } = require('internal/vfs/file_handle'); const { + VirtualFileHandle, + decodeOpenFlags, + kAccess, +} = require('internal/vfs/file_handle'); +const { + createEBADF, createEEXIST, createEISDIR, createENOENT, @@ -33,12 +39,6 @@ const { createFileStats, createDirectoryStats } = require('internal/vfs/stats'); const { Dirent } = require('internal/fs/utils'); const { fs: { - O_APPEND, - O_CREAT, - O_EXCL, - O_RDWR, - O_TRUNC, - O_WRONLY, UV_DIRENT_DIR, UV_DIRENT_FILE, }, @@ -51,41 +51,30 @@ function normalize(vfsPath) { return StringPrototypeStartsWith(vfsPath, '/') ? StringPrototypeSlice(vfsPath, 1) : vfsPath; } -// Converts numeric open flags (e.g. `fs.constants.O_RDWR`) to the flag strings -// the helpers below understand, so a caller passing `node:fs`-style numeric -// flags through the VFS is handled the same way `fs` would. Strings pass -// through unchanged; anything else falls back to 'r'. -function normalizeFlags(flags) { - if (typeof flags === 'string') return flags; - if (typeof flags !== 'number') return 'r'; - const rdwr = (flags & O_RDWR) !== 0; - const append = (flags & O_APPEND) !== 0; - const excl = (flags & O_EXCL) !== 0; - const write = (flags & O_WRONLY) !== 0 || (flags & O_CREAT) !== 0 || (flags & O_TRUNC) !== 0; - if (append) return 'a' + (excl ? 'x' : '') + (rdwr ? '+' : ''); - if (write) return 'w' + (excl ? 'x' : '') + (rdwr ? '+' : ''); - if (rdwr) return 'r+'; - return 'r'; -} - +// The open flags decide by their bits, not by a string spelling: numeric +// `fs.constants` combinations such as `O_WRONLY` alone or `O_RDONLY | O_CREAT` +// have no string form, so `decodeOpenFlags` is the one place that interprets +// them and these helpers are thin readers over it. They accept both strings +// and numbers so call sites need not care which they were given. function isCurrentPosition(position) { return position === null || position === undefined || position === -1; } function isWriteTruncate(flags) { - return flags === 'w' || flags === 'w+' || flags === 'wx' || flags === 'wx+'; + return decodeOpenFlags(flags).truncate; } -function isAppend(flags) { - return flags === 'a' || flags === 'a+' || flags === 'ax' || flags === 'ax+'; +function isExclusive(flags) { + const access = decodeOpenFlags(flags); + return access.create && access.exclusive; } -function isReadableFlag(flags) { - return flags !== 'w' && flags !== 'a' && flags !== 'wx' && flags !== 'ax'; +function mustExist(flags) { + return !decodeOpenFlags(flags).create; } function isWritableFlag(flags) { - return flags !== 'r'; + return decodeOpenFlags(flags).writable; } /** @@ -131,14 +120,15 @@ class ZipFileHandle extends VirtualFileHandle { this.#name = name; this.#buffer = initial; this.#size = initial.length; - if (isAppend(flags)) this.position = this.#size; } + // Access-mode violations are EBADF, as for any descriptor opened without + // the needed access; EISDIR is for directories only. #checkReadable() { - if (!isReadableFlag(this.flags)) throw createEISDIR('read', this.path); + if (!this[kAccess].readable) throw createEBADF('read'); } #checkWritable() { - if (!isWritableFlag(this.flags)) throw createEISDIR('write', this.path); + if (!this[kAccess].writable) throw createEBADF('write'); } #ensureCapacity(size) { if (size <= this.#buffer.length) return; @@ -151,7 +141,9 @@ class ZipFileHandle extends VirtualFileHandle { #doRead(buffer, offset, length, position) { this.#checkReadable(); const useCurrent = isCurrentPosition(position); - const pos = useCurrent ? this.position : position; + // `position` may be a BigInt, which fs allows; the arithmetic below is + // on numbers. + const pos = useCurrent ? this.position : Number(position); const available = MathMax(0, this.#size - pos); const bytesRead = MathMin(length, available); if (bytesRead > 0) this.#buffer.copy(buffer, offset, pos, pos + bytesRead); @@ -161,14 +153,16 @@ class ZipFileHandle extends VirtualFileHandle { async read(buffer, offset, length, position) { return this.#doRead(buffer, offset, length, position); } + // The synchronous form reports the count alone, like `fs.readSync` and the + // memory handle; the `{ bytesRead, buffer }` shape belongs to the promise. readSync(buffer, offset, length, position) { - return this.#doRead(buffer, offset, length, position); + return this.#doRead(buffer, offset, length, position).bytesRead; } #doWrite(buffer, offset, length, position) { this.#checkWritable(); const useCurrent = isCurrentPosition(position); - const pos = isAppend(this.flags) ? this.#size : (useCurrent ? this.position : position); + const pos = this[kAccess].append ? this.#size : (useCurrent ? this.position : Number(position)); this.#ensureCapacity(pos + length); buffer.copy(this.#buffer, pos, offset, offset + length); if (pos + length > this.#size) this.#size = pos + length; @@ -180,7 +174,7 @@ class ZipFileHandle extends VirtualFileHandle { return this.#doWrite(buffer, offset, length, position); } writeSync(buffer, offset, length, position) { - return this.#doWrite(buffer, offset, length, position); + return this.#doWrite(buffer, offset, length, position).bytesWritten; } #doReadFile(options) { @@ -196,22 +190,14 @@ class ZipFileHandle extends VirtualFileHandle { return this.#doReadFile(options); } - // Replaces content, except in append mode ('a'/'a+'/'ax'/'ax+'), where it - // appends to the existing content instead - matching MemoryFileHandle and - // what makes `appendFile()`/`appendFileSync()` (built on this, by - // VirtualProvider's defaults) actually append. + // Writes from the current position (the end, in append mode), the way + // `filehandle.writeFile()` does. Whether earlier content is discarded was + // decided by the open flags: "w" has already truncated, "r+" overwrites in + // place and keeps any tail. This is what makes `appendFile()` (built on + // this by VirtualProvider's defaults) actually append. #doWriteFile(data, options) { - this.#checkWritable(); const content = typeof data === 'string' ? Buffer.from(data, options?.encoding) : Buffer.from(data); - if (isAppend(this.flags)) { - this.#ensureCapacity(this.#size + content.length); - content.copy(this.#buffer, this.#size); - this.#size += content.length; - } else { - this.#buffer = content; - this.#size = content.length; - } - this.#dirty = true; + this.#doWrite(content, 0, content.length, null); } async writeFile(data, options) { this.#doWriteFile(data, options); @@ -233,6 +219,10 @@ class ZipFileHandle extends VirtualFileHandle { #doTruncate(len) { this.#checkWritable(); this.#ensureCapacity(len); + // Growing exposes bytes past the old size. A fresh buffer is zeroed, but + // one that was shrunk earlier still holds the cut-off content, which a + // real file never hands back. + if (len > this.#size) this.#buffer.fill(0, this.#size, len); this.#size = len; this.#dirty = true; } @@ -330,20 +320,19 @@ class ZipProvider extends VirtualProvider { } async open(path, flags, mode) { - flags = normalizeFlags(flags); const name = normalize(path); const fileEntry = await this.#getEntry(name); if (fileEntry === null && this.#isDirectory(name)) { throw createEISDIR('open', path); } const exists = fileEntry !== null; - if (isWritableFlag(flags) && this.readonly) { + if ((isWritableFlag(flags) || !mustExist(flags)) && this.readonly) { throw createEROFS('open', path); } - if ((flags === 'wx' || flags === 'wx+' || flags === 'ax' || flags === 'ax+') && exists) { + if (isExclusive(flags) && exists) { throw createEEXIST('open', path); } - if (!exists && (flags === 'r' || flags === 'r+')) { + if (!exists && mustExist(flags)) { throw createENOENT('open', path); } let initial = EMPTY_BUFFER; @@ -353,20 +342,19 @@ class ZipProvider extends VirtualProvider { return new ZipFileHandle(path, flags, mode, this.#source, name, initial); } openSync(path, flags, mode) { - flags = normalizeFlags(flags); const name = normalize(path); const fileEntry = this.#getEntrySync(name); if (fileEntry === null && this.#isDirectory(name)) { throw createEISDIR('open', path); } const exists = fileEntry !== null; - if (isWritableFlag(flags) && this.readonly) { + if ((isWritableFlag(flags) || !mustExist(flags)) && this.readonly) { throw createEROFS('open', path); } - if ((flags === 'wx' || flags === 'wx+' || flags === 'ax' || flags === 'ax+') && exists) { + if (isExclusive(flags) && exists) { throw createEEXIST('open', path); } - if (!exists && (flags === 'r' || flags === 'r+')) { + if (!exists && mustExist(flags)) { throw createENOENT('open', path); } let initial = EMPTY_BUFFER; diff --git a/test/parallel/test-vfs-handle-semantics.js b/test/parallel/test-vfs-handle-semantics.js new file mode 100644 index 000000000000..3cd38fb7f833 --- /dev/null +++ b/test/parallel/test-vfs-handle-semantics.js @@ -0,0 +1,115 @@ +// Flags: --experimental-vfs +'use strict'; + +// A file handle on a mounted path must answer the same `node:fs` calls the +// way a descriptor on a real file does. These cases run against both the +// memory provider and the ZipProvider, and state the real-fs outcome as the +// expectation. Cases are independent so the runner reports each one. + +require('../common'); +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const zlib = require('zlib'); +const vfs = require('node:vfs'); +const { test } = require('node:test'); + +const { O_WRONLY, O_RDONLY, O_CREAT } = fs.constants; + +// Each provider is described by a function that mounts a fresh layer holding +// one file `f` with the given content and returns that file's path. +const providers = { + memory(content) { + const layer = vfs.create(); + layer.writeFileSync('/f', content); + return path.join(layer.mount(), 'f'); + }, + zip(content) { + const entry = zlib.ZipEntry.createSync('f', Buffer.from(content)); + const chunks = []; + for (const chunk of zlib.createZipArchiveSync([entry])) chunks.push(chunk); + const provider = new vfs.ZipProvider(new zlib.ZipBuffer(Buffer.concat(chunks))); + return path.join(vfs.create(provider).mount(), 'f'); + }, +}; + +for (const { 0: name, 1: fileWith } of Object.entries(providers)) { + test(`${name}: reading a write-only handle fails with EBADF`, () => { + const fd = fs.openSync(fileWith('x'), 'w'); + try { + assert.throws(() => fs.readSync(fd, Buffer.alloc(4), 0, 4, 0), { code: 'EBADF' }); + } finally { + fs.closeSync(fd); + } + }); + + test(`${name}: writing a read-only handle fails with EBADF`, () => { + const fd = fs.openSync(fileWith('x'), 'r'); + try { + assert.throws(() => fs.writeSync(fd, Buffer.from('y')), { code: 'EBADF' }); + } finally { + fs.closeSync(fd); + } + }); + + test(`${name}: extending a file with ftruncate zero-fills the new region`, () => { + const file = fileWith('hello world'); + const fd = fs.openSync(file, 'r+'); + fs.ftruncateSync(fd, 5); + fs.ftruncateSync(fd, 11); + fs.closeSync(fd); + // Shrinking then growing must not resurrect the bytes that were cut off. + assert.strictEqual(fs.readFileSync(file, 'latin1'), 'hello\0\0\0\0\0\0'); + }); + + test(`${name}: readSync accepts a BigInt position`, () => { + const fd = fs.openSync(fileWith('hello'), 'r'); + try { + const buf = Buffer.alloc(4); + const n = fs.readSync(fd, buf, 0, 4, 1n); + assert.strictEqual(buf.toString('utf8', 0, n), 'ello'); + } finally { + fs.closeSync(fd); + } + }); + + test(`${name}: numeric O_WRONLY does not truncate`, () => { + const file = fileWith('hello'); + const fd = fs.openSync(file, O_WRONLY); + fs.writeSync(fd, Buffer.from('J'), 0, 1, 0); + fs.closeSync(fd); + // Only O_TRUNC truncates. + assert.strictEqual(fs.readFileSync(file, 'utf8'), 'Jello'); + }); + + test(`${name}: numeric O_RDONLY | O_CREAT opens an existing file readable and intact`, () => { + const file = fileWith('hello'); + const fd = fs.openSync(file, O_RDONLY | O_CREAT); + try { + const buf = Buffer.alloc(5); + const n = fs.readSync(fd, buf, 0, 5, 0); + assert.strictEqual(buf.toString('utf8', 0, n), 'hello'); + } finally { + fs.closeSync(fd); + } + assert.strictEqual(fs.readFileSync(file, 'utf8'), 'hello'); + }); + + test(`${name}: an "a+" handle reads from the start of the file`, () => { + const fd = fs.openSync(fileWith('abc'), 'a+'); + try { + const buf = Buffer.alloc(3); + // O_APPEND only moves writes to the end; the read offset starts at 0. + const n = fs.readSync(fd, buf, 0, 3, null); + assert.strictEqual(buf.toString('utf8', 0, n), 'abc'); + } finally { + fs.closeSync(fd); + } + }); + + test(`${name}: writeFileSync with flag "r+" overwrites in place without truncating`, () => { + const file = fileWith('hello world'); + fs.writeFileSync(file, 'HEY', { flag: 'r+' }); + assert.strictEqual(fs.readFileSync(file, 'utf8'), 'HEYlo world'); + }); +} diff --git a/test/parallel/test-vfs-zip-provider-handle.js b/test/parallel/test-vfs-zip-provider-handle.js index 3c88eacbb8c9..4263864e2410 100644 --- a/test/parallel/test-vfs-zip-provider-handle.js +++ b/test/parallel/test-vfs-zip-provider-handle.js @@ -98,7 +98,7 @@ function buildArchiveSync(entries, comment) { const handle = provider.openSync('/b.txt', 'r+'); const buf = Buffer.alloc(4); - const { bytesRead } = handle.readSync(buf, 0, 4, 2); + const bytesRead = handle.readSync(buf, 0, 4, 2); assert.strictEqual(bytesRead, 4); assert.strictEqual(buf.toString(), '2345'); @@ -124,7 +124,8 @@ function buildArchiveSync(entries, comment) { const provider = new vfs.ZipProvider(zip); const handle = await provider.open('/c.txt', 'a'); - assert.strictEqual(handle.position, 2); // Positioned at EOF on open + // O_APPEND only forces writes to the end; the read offset starts at 0. + assert.strictEqual(handle.position, 0); // Even with an explicit (wrong) position, append mode writes at the end. await handle.write(Buffer.from('z'), 0, 1, 0); @@ -259,7 +260,7 @@ function buildArchiveSync(entries, comment) { assert.throws(() => provider.rmdirSync('/file.txt'), { code: 'ENOTDIR' }); })().then(common.mustCall()); -// --- open(): EEXIST/ENOENT/EISDIR-on-wrong-direction, called directly on +// --- open(): EEXIST/ENOENT/EBADF-on-wrong-direction, called directly on // the provider so the router can't short-circuit before delegating --------- (async () => { const archive = await buildArchive([await zlib.ZipEntry.create('a.txt', Buffer.from('x'))]); @@ -271,12 +272,13 @@ function buildArchiveSync(entries, comment) { await assert.rejects(provider.open('/missing.txt', 'r'), { code: 'ENOENT' }); assert.throws(() => provider.openSync('/missing.txt', 'r'), { code: 'ENOENT' }); - // A handle opened write-only can't be read from, and vice versa. + // A handle opened write-only can't be read from, and vice versa: EBADF, + // as for any descriptor without the needed access. const writeOnly = await provider.open('/w.txt', 'w'); - await assert.rejects(writeOnly.read(Buffer.alloc(1), 0, 1, 0), { code: 'EISDIR' }); + await assert.rejects(writeOnly.read(Buffer.alloc(1), 0, 1, 0), { code: 'EBADF' }); await writeOnly.close(); const readOnly = await provider.open('/a.txt', 'r'); - await assert.rejects(readOnly.write(Buffer.alloc(1), 0, 1, 0), { code: 'EISDIR' }); + await assert.rejects(readOnly.write(Buffer.alloc(1), 0, 1, 0), { code: 'EBADF' }); await readOnly.close(); })().then(common.mustCall()); From f744023ccea33c46f03249f0d04d5a0bdf6599a3 Mon Sep 17 00:00:00 2001 From: Mike McCready <66998419+MikeMcC399@users.noreply.github.com> Date: Tue, 8 Sep 2026 14:04:56 +0200 Subject: [PATCH 018/217] doc: expand revert commit collaborator instructions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Advise to add dont-land labels to commit and revert commit pairs to prevent redundant backporting. Signed-off-by: Mike McCready <66998419+MikeMcC399@users.noreply.github.com> PR-URL: https://github.com/nodejs/node/pull/65848 Reviewed-By: Luigi Pinca Reviewed-By: Gürgün Dayıoğlu --- doc/contributing/collaborator-guide.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/contributing/collaborator-guide.md b/doc/contributing/collaborator-guide.md index e9dc344dcd38..2966f4dfeec0 100644 --- a/doc/contributing/collaborator-guide.md +++ b/doc/contributing/collaborator-guide.md @@ -490,6 +490,10 @@ generated commit message will not have a subsystem and might violate line length rules. That is OK. Append the reason for the revert and any `Refs` or `Fixes` metadata. Raise a pull request like any other change. +Apply `dont-land-on-v?.x` labels to the revert pull request and to its +corresponding original pull request, +unless the original pull request has already been backported. + ### Introducing new modules Treat commits that introduce new core modules with extra care. From cd5a7611e9b36e12f13555df86bd587299d13398 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Tue, 1 Sep 2026 14:07:50 +0200 Subject: [PATCH 019/217] crypto: use primordials in HKDF info validation Signed-off-by: Filip Skokan PR-URL: https://github.com/nodejs/node/pull/65704 Reviewed-By: Rafael Gonzaga --- lib/internal/crypto/hkdf.js | 49 ++++++++++++++++++------------- test/parallel/test-crypto-hkdf.js | 17 +++++++++++ 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/lib/internal/crypto/hkdf.js b/lib/internal/crypto/hkdf.js index fcff7070164f..8673f9b46817 100644 --- a/lib/internal/crypto/hkdf.js +++ b/lib/internal/crypto/hkdf.js @@ -4,6 +4,8 @@ const { ArrayBuffer, FunctionPrototypeCall, PromiseResolve, + TypedArrayPrototypeGetByteLength, + Uint8Array, } = primordials; const { @@ -22,6 +24,7 @@ const { const { kMaxLength } = require('buffer'); const { + getBufferSourceByteLength, jobPromise, normalizeHashName, toBuf, @@ -41,6 +44,7 @@ const { const { isAnyArrayBuffer, isArrayBufferView, + isSharedArrayBuffer, } = require('internal/util/types'); const { @@ -51,6 +55,13 @@ const { hideStackFrames, } = require('internal/errors'); +function getByteSourceByteLength(source) { + if (isSharedArrayBuffer(source)) { + return TypedArrayPrototypeGetByteLength(new Uint8Array(source)); + } + return getBufferSourceByteLength(source); +} + const validateParameters = hideStackFrames((hash, key, salt, info, length) => { validateString.withoutStackTrace(hash, 'digest'); key = prepareKey(key); @@ -61,11 +72,12 @@ const validateParameters = hideStackFrames((hash, key, salt, info, length) => { // Coerce -0 to +0. length += 0; - if (info.byteLength > 1024) { + const infoByteLength = getByteSourceByteLength(info); + if (infoByteLength > 1024) { throw new ERR_OUT_OF_RANGE.HideStackFramesError( 'info', 'must not contain more than 1024 bytes', - info.byteLength); + infoByteLength); } return { @@ -103,18 +115,20 @@ function prepareKey(key) { return key; } -function hkdf(hash, key, salt, info, length, callback) { - ({ - hash, - key, - salt, - info, - length, - } = validateParameters(hash, key, salt, info, length)); +function createHkdfJob(mode, params) { + return new HKDFJob( + mode, + params.hash, + params.key, + params.salt, + params.info, + params.length); +} +function hkdf(hash, key, salt, info, length, callback) { + const params = validateParameters(hash, key, salt, info, length); validateFunction(callback, 'callback'); - - const job = new HKDFJob(kCryptoJobAsync, hash, key, salt, info, length); + const job = createHkdfJob(kCryptoJobAsync, params); job.ondone = (error, bits) => { if (error) return FunctionPrototypeCall(callback, job, error); @@ -125,15 +139,8 @@ function hkdf(hash, key, salt, info, length, callback) { } function hkdfSync(hash, key, salt, info, length) { - ({ - hash, - key, - salt, - info, - length, - } = validateParameters(hash, key, salt, info, length)); - - const job = new HKDFJob(kCryptoJobSync, hash, key, salt, info, length); + const params = validateParameters(hash, key, salt, info, length); + const job = createHkdfJob(kCryptoJobSync, params); const { 0: err, 1: bits } = job.run(); if (err !== undefined) throw err; diff --git a/test/parallel/test-crypto-hkdf.js b/test/parallel/test-crypto-hkdf.js index bfde3b324331..1e2069d2e701 100644 --- a/test/parallel/test-crypto-hkdf.js +++ b/test/parallel/test-crypto-hkdf.js @@ -108,6 +108,23 @@ const { hasOpenSSL } = require('../common/crypto'); code: 'ERR_OUT_OF_RANGE' }); + { + const info = new Uint8Array(2048); + Object.defineProperty(info, 'byteLength', { + __proto__: null, + get() { + return 1; + }, + }); + + const calls = [ + () => hkdf('sha256', 'a', '', info, 10, common.mustNotCall()), + () => hkdfSync('sha256', 'a', '', info, 10), + ]; + for (const call of calls) + assert.throws(call, { code: 'ERR_OUT_OF_RANGE' }); + } + assert.throws( () => hkdf('sha512', 'a', '', '', 64 * 255 + 1, common.mustNotCall()), { code: 'ERR_CRYPTO_INVALID_KEYLEN' From e44669bb7d64efa8fb4807524c36fdb5e12dc80e Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Tue, 1 Sep 2026 14:08:14 +0200 Subject: [PATCH 020/217] crypto: validate the limit of PBKDF2 iterations Reject iteration counts outside of OpenSSL supported range Signed-off-by: Filip Skokan PR-URL: https://github.com/nodejs/node/pull/65704 Reviewed-By: Rafael Gonzaga --- lib/internal/crypto/webidl.js | 6 ++++++ test/fixtures/webcrypto/supports-level-2.mjs | 4 ++++ test/parallel/test-webcrypto-derivebits.js | 22 ++++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/lib/internal/crypto/webidl.js b/lib/internal/crypto/webidl.js index b9c65c91cfbd..688a8111bae7 100644 --- a/lib/internal/crypto/webidl.js +++ b/lib/internal/crypto/webidl.js @@ -16,6 +16,7 @@ const { kEmptyObject, } = require('internal/util'); const { + isInt32, isUint32, } = require('internal/validators'); const { @@ -513,6 +514,11 @@ converters.Pbkdf2Params = createDictionaryConverter( validator: (V, dict) => { if (V === 0) throw lazyDOMException('iterations cannot be zero', 'OperationError'); + if (!isInt32(V)) { + throw lazyDOMException( + 'iterations exceeds the implementation limit', + 'NotSupportedError'); + } }, required: true, }, diff --git a/test/fixtures/webcrypto/supports-level-2.mjs b/test/fixtures/webcrypto/supports-level-2.mjs index 3d0f8a630394..deec07dda904 100644 --- a/test/fixtures/webcrypto/supports-level-2.mjs +++ b/test/fixtures/webcrypto/supports-level-2.mjs @@ -124,6 +124,9 @@ export const vectors = { [true, { name: 'PBKDF2', hash: 'SHA-256', salt: Buffer.alloc(0), iterations: 1 }, { name: 'HMAC', hash: 'SHA-256' }], + [false, + { name: 'PBKDF2', hash: 'SHA-256', salt: Buffer.alloc(0), iterations: 2 ** 31 }, + { name: 'AES-CBC', length: 128 }], [false, { name: 'PBKDF2', hash: 'SHA-256', salt: Buffer.alloc(0), iterations: 1 }, 'HKDF'], @@ -183,6 +186,7 @@ export const vectors = { [true, { name: 'PBKDF2', hash: 'SHA-256', salt: Buffer.alloc(0), iterations: 1 }, 8], [true, { name: 'PBKDF2', hash: 'SHA-256', salt: Buffer.alloc(0), iterations: 1 }, 0], [false, { name: 'PBKDF2', hash: 'SHA-256', salt: Buffer.alloc(0), iterations: 0 }, 8], + [false, { name: 'PBKDF2', hash: 'SHA-256', salt: Buffer.alloc(0), iterations: 2 ** 31 }, 8], [false, { name: 'PBKDF2', hash: 'SHA-256', salt: Buffer.alloc(0), iterations: 1 }, null], [false, { name: 'PBKDF2', hash: 'SHA-256', salt: Buffer.alloc(0), iterations: 1 }, 7], [false, { name: 'PBKDF2', hash: 'Invalid', salt: Buffer.alloc(0), iterations: 1 }, 8], diff --git a/test/parallel/test-webcrypto-derivebits.js b/test/parallel/test-webcrypto-derivebits.js index 6ef2227ab2d2..6b0b9d0e36e5 100644 --- a/test/parallel/test-webcrypto-derivebits.js +++ b/test/parallel/test-webcrypto-derivebits.js @@ -120,6 +120,28 @@ const rejectsXCurves = hasFIPS(3, 5); } } +// Test PBKDF2 rejects iteration counts beyond the native signed int range +{ + async function test() { + const key = await subtle.importKey( + 'raw', + new Uint8Array([1]), + 'PBKDF2', + false, + ['deriveBits']); + await assert.rejects( + subtle.deriveBits({ + name: 'PBKDF2', + hash: 'SHA-256', + salt: new Uint8Array([2]), + iterations: 2 ** 31, + }, key, 8), + { name: 'NotSupportedError' }); + } + + test().then(common.mustCall()); +} + // Test X25519 and X448 bit derivation { async function test(name) { From 080e76b3d7878685c1f60e3f2d1c2dfbb2d82647 Mon Sep 17 00:00:00 2001 From: Guy Bedford Date: Mon, 27 Jul 2026 12:39:30 -0700 Subject: [PATCH 021/217] net: support sending net.BoundSocket to threads and child processes This adds support for transferring net.BoundSocket instances to other threads via the worker_threads postMessage() transfer list, and for sending them to child processes as the sendHandle argument of subprocess.send(), following on from the BoundSocket introduction. A BoundSocket reserves a port synchronously at construction time. Making it transferable means a port can be reserved on one thread or process and the bound (but not yet listening or connected) TCP handle handed off to another to listen or connect on, without racing on the bind. For threads, BoundSocket implements kTransfer/kTransferList/ kDeserialize, moving the underlying TCP handle with the same mechanism used for net.Socket and net.Server transfer. For child processes, the handleConversion entry reuses the same transfer protocol on the sending side and the same _TransferredBoundSocket deserialization path on the receiving side; the underlying transport is that of cluster's shared-handle scheduling: SCM_RIGHTS on Unix and WSADuplicateSocket on Windows, both of which carry bind state. In both cases the source instance is left in the adopted state: address(), fd() and close() throw ERR_SOCKET_HANDLE_ADOPTED. Transfer requires an un-adopted, open TCP handle, otherwise ERR_WORKER_HANDLE_NOT_TRANSFERABLE is thrown; pipe (path) binds are not transferable and throw ERR_INVALID_HANDLE_TYPE when sent over IPC. On the receiving side the local address is re-derived from the handle rather than trusted from serialized state. Signed-off-by: Guy Bedford PR-URL: https://github.com/nodejs/node/pull/64725 Reviewed-By: Matteo Collina Reviewed-By: James M Snell --- doc/api/child_process.md | 16 +++- doc/api/errors.md | 12 +-- doc/api/net.md | 15 ++++ doc/api/worker_threads.md | 4 +- lib/internal/child_process.js | 65 +++++++++++---- lib/net.js | 65 ++++++++++++++- .../test-child-process-send-boundsocket.js | 79 +++++++++++++++++++ .../test-net-boundsocket-transfer-worker.js | 64 +++++++++++++++ test/parallel/test-net-transfer-guards.js | 39 ++++++++- 9 files changed, 329 insertions(+), 30 deletions(-) create mode 100644 test/parallel/test-child-process-send-boundsocket.js create mode 100644 test/parallel/test-net-boundsocket-transfer-worker.js diff --git a/doc/api/child_process.md b/doc/api/child_process.md index 138fb52f8612..bdf7d1f98b40 100644 --- a/doc/api/child_process.md +++ b/doc/api/child_process.md @@ -1581,7 +1581,7 @@ added: v0.5.9 * `message` {Object} A parsed JSON object or primitive value. * `sendHandle` {Handle|undefined} `undefined` or a [`net.Socket`][], - [`net.Server`][], or [`dgram.Socket`][] object. + [`net.Server`][], [`net.BoundSocket`][], or [`dgram.Socket`][] object. The `'message'` event is triggered when a child process uses [`process.send()`][] to send messages. @@ -1891,6 +1891,9 @@ subprocess.ref(); + +* Extends: {errors.Error} + +The Web IDL {DOMException} class. These errors are thrown by web-platform APIs +in Node.js such as [`fetch()`][], {AbortController}, {AbortSignal}, and Web +Streams. For details, see also [`Class: DOMException`][] on the Globals page. + +The [`domException.name`][] property identifies the type of the exception (for +example, `'AbortError'`). Unlike most errors in Node.js, the +[`domException.code`][] property is a number that corresponds to a +[legacy error code name][Web IDL error names] (for example, `20` for +`ABORT_ERR`). + +Node.js-specific APIs that support {AbortSignal} (such as +[`events.once()`][]) throw a Node.js `AbortError` (a native {errors.Error} with +`name` of `'AbortError'` and `code` of [`'ABORT_ERR'`][ABORT_ERR]) rather than a +{DOMException}. To identify abort errors in either case, checking +`err?.name === 'AbortError'` is sufficient. + +See also [`ABORT_ERR`][]. + ## Class: `RangeError` * Extends: {errors.Error} @@ -4661,6 +4687,7 @@ The public key in the certificate SubjectPublicKeyInfo could not be read. An error occurred trying to allocate memory. This should never happen. +[ABORT_ERR]: #abort_err [ES Module]: esm.md [ICU]: intl.md#internationalization-support [JSON Web Key Elliptic Curve Registry]: https://www.iana.org/assignments/jose/jose.xhtml#web-key-elliptic-curve @@ -4674,6 +4701,7 @@ An error occurred trying to allocate memory. This should never happen. [V8's stack trace API]: https://v8.dev/docs/stack-trace-api [WHATWG Supported Encodings]: util.md#whatwg-supported-encodings [WHATWG URL API]: url.md#the-whatwg-url-api +[Web IDL error names]: https://webidl.spec.whatwg.org/#dfn-error-names-table [`"exports"`]: packages.md#exports [`"imports"`]: packages.md#imports [`'uncaughtException'`]: process.md#event-uncaughtexception @@ -4681,7 +4709,9 @@ An error occurred trying to allocate memory. This should never happen. [`--force-fips`]: cli.md#--force-fips [`--no-addons`]: cli.md#--no-addons [`--unhandled-rejections`]: cli.md#--unhandled-rejectionsmode +[`ABORT_ERR`]: #abort_err [`BoundSocket`]: net.md#class-netboundsocket +[`Class: DOMException`]: globals.md#class-domexception [`Class: assert.AssertionError`]: assert.md#class-assertassertionerror [`ERR_INCOMPATIBLE_OPTION_PAIR`]: #err_incompatible_option_pair [`ERR_INVALID_ARG_TYPE`]: #err_invalid_arg_type @@ -4710,10 +4740,13 @@ An error occurred trying to allocate memory. This should never happen. [`dgram.createSocket()`]: dgram.md#dgramcreatesocketoptions-callback [`dgram.disconnect()`]: dgram.md#socketdisconnect [`dgram.remoteAddress()`]: dgram.md#socketremoteaddress +[`domException.code`]: https://developer.mozilla.org/en-US/docs/Web/API/DOMException/code [`domException.name`]: https://developer.mozilla.org/en-US/docs/Web/API/DOMException/name [`errno`(3) man page]: https://man7.org/linux/man-pages/man3/errno.3.html [`error.code`]: #errorcode [`error.message`]: #errormessage +[`events.once()`]: events.md#eventsonceemitter-name-options +[`fetch()`]: globals.md#fetch [`fs.Dir`]: fs.md#class-fsdir [`fs.cp()`]: fs.md#fscpsrc-dest-options-callback [`fs.readFileSync`]: fs.md#fsreadfilesyncpath-options From 833c58d9dc6961c2764a064559de2a912055851e Mon Sep 17 00:00:00 2001 From: Marten Richter Date: Tue, 8 Sep 2026 19:43:27 +0200 Subject: [PATCH 031/217] quic: fix crash in onStreamClose onStreamClose could crash, if this[kOwner] was not set. Signed-off-by: Marten Richter PR-URL: https://github.com/nodejs/node/pull/65861 Reviewed-By: James M Snell Reviewed-By: Xuguang Mei --- lib/internal/quic/quic.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/internal/quic/quic.js b/lib/internal/quic/quic.js index 8ecd5fc776a3..04e99ecc3232 100644 --- a/lib/internal/quic/quic.js +++ b/lib/internal/quic/quic.js @@ -962,9 +962,10 @@ setCallbacks({ // Called when the stream C++ handle has been closed. The error is // either undefined (clean close) or a raw array [type, code, reason] // from QuicError::ToV8Value. Convert to a proper Node.js Error. + if (!this[kOwner]) return; if (error !== undefined) { error = convertQuicError(error); - } else if (this[kOwner] && !this[kOwner].destroyed) { + } else if (!this[kOwner].destroyed) { // The stream is closing cleanly, but it may have been reset by the // peer (ReceiveStreamReset) or locally (resetStream). The C++ side // records the reset code in state.resetCode. If set, surface the From ae29cc92eac26569121c7a4cfe7c362c6535ad2d Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sun, 6 Sep 2026 10:46:57 -0700 Subject: [PATCH 032/217] quic: split headers out from src/quic/stream.{h/cc} As part of the effort to get better separation between the generalized QUIC streams and HTTP/3 streams, separate out header handling from the C++ class. Signed-off-by: James M Snell Assisted-by: Opencode PR-URL: https://github.com/nodejs/node/pull/65863 Reviewed-By: Xuguang Mei Reviewed-By: Tim Perry --- lib/internal/quic/quic.js | 54 +++- lib/internal/quic/state.js | 40 --- src/quic/application.cc | 2 +- src/quic/application.h | 30 +- src/quic/bindingdata.cc | 50 +++ src/quic/bindingdata.h | 2 + src/quic/defs.h | 11 - src/quic/http3.cc | 294 ++++++++++++------ src/quic/streams.cc | 181 +---------- src/quic/streams.h | 87 ++---- .../parallel/test-quic-h3-header-interest.mjs | 76 +++++ 11 files changed, 440 insertions(+), 387 deletions(-) create mode 100644 test/parallel/test-quic-h3-header-interest.mjs diff --git a/lib/internal/quic/quic.js b/lib/internal/quic/quic.js index 04e99ecc3232..b6bb31aa97b4 100644 --- a/lib/internal/quic/quic.js +++ b/lib/internal/quic/quic.js @@ -50,7 +50,9 @@ let debug = require('internal/util/debuglog').debuglog('quic', (fn) => { const { Endpoint: Endpoint_, + sendHeaders, setCallbacks, + setHeadersInterest, // The constants to be exposed to end users for various options. CC_ALGO_RENO_STR: CC_ALGO_RENO, @@ -1304,6 +1306,17 @@ function parseHeaderPairs(pairs) { return block; } +function updateHeaderInterest(handle, inner) { + if (handle === undefined) return; + setHeadersInterest( + handle, + inner.onheaders !== undefined || + inner.ontrailers !== undefined || + inner.oninfo !== undefined, + inner.onwanttrailers !== undefined || inner.pendingTrailers !== undefined, + ); +} + /** * Applies session and stream callbacks from an options object to a session. * @param {QuicSession} session @@ -1828,13 +1841,12 @@ class QuicStream { const inner = this.#inner; if (fn === undefined) { inner.onheaders = undefined; - inner.state.wantsHeaders = false; } else { validateFunction(fn, 'onheaders'); assertHeadersSupported(inner.session); inner.onheaders = FunctionPrototypeBind(fn, this); - inner.state.wantsHeaders = true; } + updateHeaderInterest(this.#handle, inner); } /** @type {Function|undefined} */ @@ -1853,6 +1865,7 @@ class QuicStream { assertHeadersSupported(inner.session); inner.oninfo = FunctionPrototypeBind(fn, this); } + updateHeaderInterest(this.#handle, inner); } /** @type {Function|undefined} */ @@ -1871,6 +1884,7 @@ class QuicStream { assertHeadersSupported(inner.session); inner.ontrailers = FunctionPrototypeBind(fn, this); } + updateHeaderInterest(this.#handle, inner); } /** @type {Function|undefined} */ @@ -1884,13 +1898,12 @@ class QuicStream { const inner = this.#inner; if (fn === undefined) { inner.onwanttrailers = undefined; - inner.state.wantsTrailers = false; } else { validateFunction(fn, 'onwanttrailers'); assertHeadersSupported(inner.session); inner.onwanttrailers = FunctionPrototypeBind(fn, this); - inner.state.wantsTrailers = true; } + updateHeaderInterest(this.#handle, inner); } /** @@ -1919,10 +1932,12 @@ class QuicStream { assertHeadersSupported(inner.session); if (headers === undefined) { inner.pendingTrailers = undefined; + updateHeaderInterest(this.#handle, inner); return; } validateObject(headers, 'headers'); inner.pendingTrailers = headers; + updateHeaderInterest(this.#handle, inner); } /** @@ -2105,7 +2120,8 @@ class QuicStream { const headerString = buildNgHeaderString( headers, assertValidPseudoHeader, true /* strictSingleValueFields */); const flags = terminal ? kHeadersFlagsTerminal : kHeadersFlagsNone; - return this.#handle.sendHeaders(kHeadersKindInitial, headerString, flags); + return sendHeaders( + this.#handle, kHeadersKindInitial, headerString, flags); } /** @@ -2124,8 +2140,8 @@ class QuicStream { validateObject(headers, 'headers'); const headerString = buildNgHeaderString( headers, assertValidPseudoHeader, true); - return this.#handle.sendHeaders( - kHeadersKindHints, headerString, kHeadersFlagsNone); + return sendHeaders( + this.#handle, kHeadersKindHints, headerString, kHeadersFlagsNone); } /** @@ -2144,8 +2160,8 @@ class QuicStream { } validateObject(headers, 'headers'); const headerString = buildNgHeaderString(headers); - return this.#handle.sendHeaders( - kHeadersKindTrailing, headerString, kHeadersFlagsNone); + return sendHeaders( + this.#handle, kHeadersKindTrailing, headerString, kHeadersFlagsNone); } /** @@ -2563,7 +2579,7 @@ class QuicStream { assertValidPseudoHeader, true, // This could become an option in future ); - return this.#handle.sendHeaders(kind, headerString, flags); + return sendHeaders(this.#handle, kind, headerString, flags); } [kFinishClose](error) { @@ -2671,7 +2687,6 @@ class QuicStream { switch (kindName) { case 'initial': - assert(inner.onheaders, 'Unexpected stream headers event'); inner.headers ??= block; if (onStreamHeadersChannel.hasSubscribers) { onStreamHeadersChannel.publish({ @@ -2681,7 +2696,8 @@ class QuicStream { headers: block, }); } - safeCallbackInvoke(inner.onheaders, this, block); + if (inner.onheaders) + safeCallbackInvoke(inner.onheaders, this, block); break; case 'trailing': if (onStreamTrailersChannel.hasSubscribers) { @@ -2717,8 +2733,20 @@ class QuicStream { // nghttp3 is asking us to provide trailers to send. // Check for pre-set pendingTrailers first, then the callback. if (inner.pendingTrailers) { - this.sendTrailers(inner.pendingTrailers); + let sent; + try { + sent = this.sendTrailers(inner.pendingTrailers); + } catch (error) { + this.destroy(error); + return; + } + if (!sent) { + this.destroy(new ERR_QUIC_STREAM_ABORTED( + 'Failed to submit trailing headers')); + return; + } inner.pendingTrailers = undefined; + updateHeaderInterest(this.#handle, inner); } else if (typeof inner.onwanttrailers === 'function') { safeCallbackInvoke(inner.onwanttrailers, this); } diff --git a/lib/internal/quic/state.js b/lib/internal/quic/state.js index 769fee687498..21db0f0e3d59 100644 --- a/lib/internal/quic/state.js +++ b/lib/internal/quic/state.js @@ -101,10 +101,8 @@ const { IDX_STATE_STREAM_HAS_OUTBOUND, IDX_STATE_STREAM_HAS_READER, IDX_STATE_STREAM_WANTS_BLOCK, - IDX_STATE_STREAM_WANTS_HEADERS, IDX_STATE_STREAM_WANTS_RESET, IDX_STATE_STREAM_WANTS_STOP_SENDING, - IDX_STATE_STREAM_WANTS_TRAILERS, IDX_STATE_STREAM_RECEIVED_EARLY_DATA, IDX_STATE_STREAM_WRITE_DESIRED_SIZE, IDX_STATE_STREAM_BUDGET, @@ -145,10 +143,8 @@ assert(IDX_STATE_STREAM_RESET !== undefined); assert(IDX_STATE_STREAM_HAS_OUTBOUND !== undefined); assert(IDX_STATE_STREAM_HAS_READER !== undefined); assert(IDX_STATE_STREAM_WANTS_BLOCK !== undefined); -assert(IDX_STATE_STREAM_WANTS_HEADERS !== undefined); assert(IDX_STATE_STREAM_WANTS_RESET !== undefined); assert(IDX_STATE_STREAM_WANTS_STOP_SENDING !== undefined); -assert(IDX_STATE_STREAM_WANTS_TRAILERS !== undefined); assert(IDX_STATE_STREAM_WRITE_DESIRED_SIZE !== undefined); assert(IDX_STATE_STREAM_RESET_CODE !== undefined); @@ -824,20 +820,6 @@ class QuicStreamState { DataViewPrototypeSetUint8(handle, this.#offset + IDX_STATE_STREAM_WANTS_BLOCK, val ? 1 : 0); } - /** @type {boolean} */ - get wantsHeaders() { - const handle = this.#handle; - if (handle === undefined) return undefined; - return DataViewPrototypeGetUint8(handle, this.#offset + IDX_STATE_STREAM_WANTS_HEADERS) !== 0; - } - - /** @type {boolean} */ - set wantsHeaders(val) { - const handle = this.#handle; - if (handle === undefined) return; - DataViewPrototypeSetUint8(handle, this.#offset + IDX_STATE_STREAM_WANTS_HEADERS, val ? 1 : 0); - } - /** @type {boolean} */ get wantsReset() { const handle = this.#handle; @@ -870,20 +852,6 @@ class QuicStreamState { val ? 1 : 0); } - /** @type {boolean} */ - get wantsTrailers() { - const handle = this.#handle; - if (handle === undefined) return undefined; - return DataViewPrototypeGetUint8(handle, this.#offset + IDX_STATE_STREAM_WANTS_TRAILERS) !== 0; - } - - /** @type {boolean} */ - set wantsTrailers(val) { - const handle = this.#handle; - if (handle === undefined) return; - DataViewPrototypeSetUint8(handle, this.#offset + IDX_STATE_STREAM_WANTS_TRAILERS, val ? 1 : 0); - } - /** @type {boolean} */ get early() { const handle = this.#handle; @@ -948,8 +916,6 @@ class QuicStreamState { wantsBlock, wantsReset, wantsStopSending, - wantsHeaders, - wantsTrailers, early, resetCode, writeDesiredSize, @@ -969,8 +935,6 @@ class QuicStreamState { wantsBlock, wantsReset, wantsStopSending, - wantsHeaders, - wantsTrailers, early, resetCode: `${resetCode}`, writeDesiredSize, @@ -1007,8 +971,6 @@ class QuicStreamState { wantsBlock, wantsReset, wantsStopSending, - wantsHeaders, - wantsTrailers, early, resetCode, writeDesiredSize, @@ -1028,8 +990,6 @@ class QuicStreamState { wantsBlock, wantsReset, wantsStopSending, - wantsHeaders, - wantsTrailers, early, resetCode, writeDesiredSize, diff --git a/src/quic/application.cc b/src/quic/application.cc index f83b64bad49e..1e78b7931ce1 100644 --- a/src/quic/application.cc +++ b/src/quic/application.cc @@ -117,7 +117,7 @@ Maybe Session::Application_Options::From( // Ensure the advertised max_field_section_size in SETTINGS is at least // as large as max_header_length. Otherwise the peer would be told to - // restrict headers to a smaller size than what CanAddHeader accepts. + // restrict headers to a smaller size than what the HTTP/3 stream accepts. if (options.max_field_section_size < options.max_header_length) { options.max_field_section_size = options.max_header_length; } diff --git a/src/quic/application.h b/src/quic/application.h index df472e0a9fd7..0ae4ab53ab7e 100644 --- a/src/quic/application.h +++ b/src/quic/application.h @@ -11,6 +11,17 @@ namespace node::quic { +enum class HeadersKind : uint8_t { + HINTS, + INITIAL, + TRAILING, +}; + +enum class HeadersFlags : uint8_t { + NONE, + TERMINAL, +}; + // An Application implements the ALPN-protocol specific semantics on behalf // of a QUIC Session. class Session::Application : public MemoryRetainer { @@ -95,13 +106,10 @@ class Session::Application : public MemoryRetainer { // Application. virtual bool AcknowledgeStreamData(stream_id id, size_t datalen); - // Called to determine if a Header can be added to this application. - // Applications that do not support headers will always return false. - virtual bool CanAddHeader(size_t current_count, - size_t current_headers_length, - size_t this_header_length) { - return false; - } + // Called when a pending transport stream receives its stream ID. Protocols + // can use this to flush operations that require an opened stream. Returns + // false if deferred application data could not be submitted. + virtual bool StreamOpened(Stream& stream) { return true; } // Called when ngtcp2 reports NGTCP2_ERR_STREAM_SHUT_WR for a stream. // Applications that manage their own framing (e.g., HTTP/3) must inform @@ -173,13 +181,19 @@ class Session::Application : public MemoryRetainer { // Submits an outbound block of headers for the given stream. Not all // Application types will support headers, in which case this function // should return false. - virtual bool SendHeaders(const Stream& stream, + virtual bool SendHeaders(Stream& stream, HeadersKind kind, const v8::Local& headers, HeadersFlags flags = HeadersFlags::NONE) { return false; } + // Updates JavaScript callback interest for an application's stream header + // events. Applications without header semantics ignore this. + virtual void SetHeadersInterest(Stream& stream, + bool wants_headers, + bool wants_trailers) {} + // Returns true if the application protocol supports sending and // receiving headers on streams (e.g. HTTP/3). Applications that // do not support headers should return false (the default). diff --git a/src/quic/bindingdata.cc b/src/quic/bindingdata.cc index 31467a8477a7..d5711d88cb92 100644 --- a/src/quic/bindingdata.cc +++ b/src/quic/bindingdata.cc @@ -1,6 +1,7 @@ #if HAVE_OPENSSL && HAVE_QUIC #include "guard.h" #ifndef OPENSSL_NO_QUIC +#include #include #include #include @@ -13,13 +14,16 @@ #include #include #include +#include "application.h" #include "bindingdata.h" #include "session.h" #include "session_manager.h" +#include "streams.h" namespace node { using mem::kReserveSizeAndAlign; +using v8::Array; using v8::DictionaryTemplate; using v8::Function; using v8::FunctionTemplate; @@ -288,6 +292,26 @@ void nghttp3_debug_log(const char* fmt, va_list args) { void BindingData::InitPerContext(Realm* realm, Local target) { nghttp3_set_debug_vprintf_callback(nghttp3_debug_log); SetMethod(realm->context(), target, "setCallbacks", SetCallbacks); + SetMethod(realm->context(), target, "sendHeaders", SendHeaders); + SetMethod(realm->context(), target, "setHeadersInterest", SetHeadersInterest); + + constexpr int QUIC_STREAM_HEADERS_KIND_HINTS = + static_cast(HeadersKind::HINTS); + constexpr int QUIC_STREAM_HEADERS_KIND_INITIAL = + static_cast(HeadersKind::INITIAL); + constexpr int QUIC_STREAM_HEADERS_KIND_TRAILING = + static_cast(HeadersKind::TRAILING); + constexpr int QUIC_STREAM_HEADERS_FLAGS_NONE = + static_cast(HeadersFlags::NONE); + constexpr int QUIC_STREAM_HEADERS_FLAGS_TERMINAL = + static_cast(HeadersFlags::TERMINAL); + + NODE_DEFINE_CONSTANT(target, QUIC_STREAM_HEADERS_KIND_HINTS); + NODE_DEFINE_CONSTANT(target, QUIC_STREAM_HEADERS_KIND_INITIAL); + NODE_DEFINE_CONSTANT(target, QUIC_STREAM_HEADERS_KIND_TRAILING); + NODE_DEFINE_CONSTANT(target, QUIC_STREAM_HEADERS_FLAGS_NONE); + NODE_DEFINE_CONSTANT(target, QUIC_STREAM_HEADERS_FLAGS_TERMINAL); + Realm::GetCurrent(realm->context())->AddBindingData(target); } @@ -295,6 +319,32 @@ void BindingData::RegisterExternalReferences( ExternalReferenceRegistry* registry) { registry->Register(IllegalConstructor); registry->Register(SetCallbacks); + registry->Register(SendHeaders); + registry->Register(SetHeadersInterest); +} + +JS_METHOD_IMPL(BindingData::SendHeaders) { + Stream* stream; + ASSIGN_OR_RETURN_UNWRAP(&stream, args[0]); + CHECK(args[1]->IsUint32()); // Kind + CHECK(args[2]->IsArray()); // Headers + CHECK(args[3]->IsUint32()); // Flags + + HeadersKind kind = FromV8Value(args[1]); + Local headers = args[2].As(); + HeadersFlags flags = FromV8Value(args[3]); + + args.GetReturnValue().Set(stream->session().application().SendHeaders( + *stream, kind, headers, flags)); +} + +JS_METHOD_IMPL(BindingData::SetHeadersInterest) { + Stream* stream; + ASSIGN_OR_RETURN_UNWRAP(&stream, args[0]); + CHECK(args[1]->IsBoolean()); + CHECK(args[2]->IsBoolean()); + stream->session().application().SetHeadersInterest( + *stream, args[1]->IsTrue(), args[2]->IsTrue()); } BindingData::BindingData(Realm* realm, Local object) diff --git a/src/quic/bindingdata.h b/src/quic/bindingdata.h index e4763b5b0d3e..e56c861261ad 100644 --- a/src/quic/bindingdata.h +++ b/src/quic/bindingdata.h @@ -294,6 +294,8 @@ class BindingData final // Installs the set of JavaScript callback functions that are used to // bridge out to the JS API. JS_METHOD(SetCallbacks); + JS_METHOD(SendHeaders); + JS_METHOD(SetHeadersInterest); // Lazily-created per-Realm SessionManager. Centralizes CID -> Session // routing so that any endpoint can route packets to any session. diff --git a/src/quic/defs.h b/src/quic/defs.h index 5184288475b1..32b3a10d7b80 100644 --- a/src/quic/defs.h +++ b/src/quic/defs.h @@ -300,17 +300,6 @@ enum class Direction : uint8_t { UNIDIRECTIONAL, }; -enum class HeadersKind : uint8_t { - HINTS, - INITIAL, - TRAILING, -}; - -enum class HeadersFlags : uint8_t { - NONE, - TERMINAL, -}; - enum class StreamPriority : uint8_t { DEFAULT = NGHTTP3_DEFAULT_URGENCY, LOW = NGHTTP3_URGENCY_LOW, diff --git a/src/quic/http3.cc b/src/quic/http3.cc index 6e1d0c44a048..43ead1b8164e 100644 --- a/src/quic/http3.cc +++ b/src/quic/http3.cc @@ -22,7 +22,11 @@ namespace node { using v8::Array; +using v8::Global; +using v8::Integer; using v8::Local; +using v8::LocalVector; +using v8::Value; namespace quic { @@ -138,6 +142,25 @@ struct Http3HeaderTraits { using Http3Header = NgHeader; +struct Http3StreamState final : public StreamApplicationState { + struct PendingHeaders final { + HeadersKind kind; + Global headers; + HeadersFlags flags; + + PendingHeaders(HeadersKind kind, Global headers, HeadersFlags flags) + : kind(kind), headers(std::move(headers)), flags(flags) {} + DISALLOW_COPY_AND_MOVE(PendingHeaders) + }; + + std::vector> pending_headers; + std::vector> headers; + HeadersKind headers_kind = HeadersKind::INITIAL; + size_t headers_length = 0; + bool wants_headers = false; + bool wants_trailers = false; +}; + // Implements the low-level HTTP/3 Application semantics. class Http3ApplicationImpl final : public Session::Application { public: @@ -334,17 +357,6 @@ class Http3ApplicationImpl final : public Session::Application { return nghttp3_conn_add_ack_offset(*this, id, datalen) == 0; } - bool CanAddHeader(size_t current_count, - size_t current_headers_length, - size_t this_header_length) override { - // We cannot add the header if we've either reached - // * the max number of header pairs or - // * the max number of header bytes (name + value combined) - return (current_count < options_.max_header_pairs) && - (current_headers_length + this_header_length) <= - options_.max_header_length; - } - bool stream_fin_managed_by_application() const override { return true; } void StreamWriteShut(stream_id id) override { @@ -537,75 +549,49 @@ class Http3ApplicationImpl final : public Session::Application { Application::ReceiveStreamStopSending(stream, std::move(error)); } - bool SendHeaders(const Stream& stream, - HeadersKind kind, - const Local& headers, - HeadersFlags flags = HeadersFlags::NONE) override { - Session::SendPendingDataScope send_scope(&session()); - Http3Headers nva(env(), headers); + bool StreamOpened(Stream& stream) override { + auto* state = GetStreamState(stream); + if (state == nullptr || state->pending_headers.empty()) return true; - switch (kind) { - case HeadersKind::HINTS: { - if (!session().is_server()) { - // Client side cannot send hints - return false; - } - Debug(&session(), - "Submitting %" PRIu64 " early hints for stream %" PRIu64, - stream.id()); - return nghttp3_conn_submit_info( - *this, stream.id(), nva.data(), nva.length()) == 0; - break; + decltype(state->pending_headers) pending; + state->pending_headers.swap(pending); + Session::SendPendingDataScope send_scope(&session()); + for (auto& headers : pending) { + if (!SubmitHeaders(stream, + headers->kind, + headers->headers.Get(env()->isolate()), + headers->flags)) { + return false; } - case HeadersKind::INITIAL: { - static constexpr nghttp3_data_reader reader = {on_read_data_callback}; - const nghttp3_data_reader* reader_ptr = nullptr; - - // If the terminal flag is set, that means that we know we're only - // sending headers and no body and the stream writable side should be - // closed immediately because there is no nghttp3_data_reader provided. - if (flags != HeadersFlags::TERMINAL) { - reader_ptr = &reader; - } + } + return true; + } - if (session().is_server()) { - // If this is a server, we're submitting a response... - Debug(&session(), - "Submitting %" PRIu64 " response headers for stream %" PRIu64, - nva.length(), - stream.id()); - return nghttp3_conn_submit_response(*this, - stream.id(), - nva.data(), - nva.length(), - reader_ptr) == 0; - } else { - // Otherwise we're submitting a request... - Debug(&session(), - "Submitting %" PRIu64 " request headers for stream %" PRIu64, - nva.length(), - stream.id()); - return nghttp3_conn_submit_request(*this, - stream.id(), - nva.data(), - nva.length(), - reader_ptr, - const_cast(&stream)) == 0; - } - break; - } - case HeadersKind::TRAILING: { - Debug(&session(), - "Submitting %" PRIu64 " trailing headers for stream %" PRIu64, - nva.length(), - stream.id()); - return nghttp3_conn_submit_trailers( - *this, stream.id(), nva.data(), nva.length()) == 0; - break; - } + bool SendHeaders(Stream& stream, + HeadersKind kind, + const Local& headers, + HeadersFlags flags = HeadersFlags::NONE) override { + if (kind == HeadersKind::HINTS && !session().is_server()) return false; + + if (stream.is_pending()) { + Debug(&session(), "Enqueuing headers for pending HTTP/3 stream"); + auto& state = GetOrCreateStreamState(stream); + state.pending_headers.push_back( + std::make_unique( + kind, Global(env()->isolate(), headers), flags)); + return true; } - return false; + Session::SendPendingDataScope send_scope(&session()); + return SubmitHeaders(stream, kind, headers, flags); + } + + void SetHeadersInterest(Stream& stream, + bool wants_headers, + bool wants_trailers) override { + auto& state = GetOrCreateStreamState(stream); + state.wants_headers = wants_headers; + state.wants_trailers = wants_trailers; } void SetStreamPriority(const Stream& stream, @@ -642,8 +628,7 @@ class Http3ApplicationImpl final : public Session::Application { // PRIORITY_UPDATE frames). Client-side priority is tracked by the // Stream itself and returned directly from GetPriority in streams.cc. if (!session().is_server()) { - auto& stored = stream.stored_priority(); - return {stored.priority, stored.flags}; + return {stream.priority(), stream.priority_flags()}; } nghttp3_pri pri; if (nghttp3_conn_get_stream_priority(*this, &pri, stream.id()) == 0) { @@ -730,7 +715,7 @@ class Http3ApplicationImpl final : public Session::Application { // for the next writev_stream in the send loop. if (pending_trailers_stream_ == data->id) { pending_trailers_stream_ = -1; - if (data->stream) data->stream->EmitWantTrailers(); + if (data->stream) EmitWantTrailers(*data->stream); } return true; } @@ -750,6 +735,137 @@ class Http3ApplicationImpl final : public Session::Application { id == qpack_enc_stream_id_; } + bool SubmitHeaders(Stream& stream, + HeadersKind kind, + const Local& headers, + HeadersFlags flags) { + Http3Headers nva(env(), headers); + + switch (kind) { + case HeadersKind::HINTS: { + if (!session().is_server()) return false; + Debug(&session(), + "Submitting %" PRIu64 " early hints for stream %" PRIu64, + stream.id()); + return nghttp3_conn_submit_info( + *this, stream.id(), nva.data(), nva.length()) == 0; + } + case HeadersKind::INITIAL: { + static constexpr nghttp3_data_reader reader = {on_read_data_callback}; + const nghttp3_data_reader* reader_ptr = nullptr; + if (flags != HeadersFlags::TERMINAL) reader_ptr = &reader; + + if (session().is_server()) { + Debug(&session(), + "Submitting %" PRIu64 " response headers for stream %" PRIu64, + nva.length(), + stream.id()); + return nghttp3_conn_submit_response(*this, + stream.id(), + nva.data(), + nva.length(), + reader_ptr) == 0; + } + + Debug(&session(), + "Submitting %" PRIu64 " request headers for stream %" PRIu64, + nva.length(), + stream.id()); + return nghttp3_conn_submit_request(*this, + stream.id(), + nva.data(), + nva.length(), + reader_ptr, + &stream) == 0; + } + case HeadersKind::TRAILING: { + Debug(&session(), + "Submitting %" PRIu64 " trailing headers for stream %" PRIu64, + nva.length(), + stream.id()); + return nghttp3_conn_submit_trailers( + *this, stream.id(), nva.data(), nva.length()) == 0; + } + } + + return false; + } + + Http3StreamState* GetStreamState(Stream& stream) const { + return static_cast(stream.application_state()); + } + + Http3StreamState& GetOrCreateStreamState(Stream& stream) { + if (stream.application_state() == nullptr) { + stream.set_application_state(std::make_unique()); + } + return *GetStreamState(stream); + } + + void BeginHeaders(Stream& stream, HeadersKind kind) { + auto& state = GetOrCreateStreamState(stream); + state.headers_length = 0; + state.headers.clear(); + state.headers_kind = kind; + } + + bool AddHeader(Stream& stream, std::unique_ptr header) { + auto& state = GetOrCreateStreamState(stream); + size_t length = header->length(); + if (state.headers.size() >= options_.max_header_pairs || + state.headers_length + length > options_.max_header_length) { + return false; + } + state.headers_length += length; + state.headers.push_back(std::move(header)); + return true; + } + + void EmitHeaders(Stream& stream) { + auto& state = GetOrCreateStreamState(stream); + stream.RecordReceivedActivity(); + if (!env()->can_call_into_js() || !state.wants_headers) { + state.headers.clear(); + return; + } + + CallbackScope cb_scope(&stream); + auto& binding = BindingData::Get(env()); + size_t count = state.headers.size() * 2; + LocalVector values(env()->isolate(), count); + + for (size_t i = 0; i < state.headers.size(); i++) { + Local name; + Local value; + if (!state.headers[i]->GetName(&binding).ToLocal(&name) || + !state.headers[i]->GetValue(&binding).ToLocal(&value)) [[unlikely]] { + state.headers.clear(); + return; + } + values[i * 2] = name; + values[i * 2 + 1] = value; + } + + state.headers.clear(); + Local argv[] = { + Array::New(env()->isolate(), values.data(), count), + Integer::NewFromUnsigned(env()->isolate(), + static_cast(state.headers_kind))}; + stream.MakeCallback( + binding.stream_headers_callback(), arraysize(argv), argv); + } + + void EmitWantTrailers(Stream& stream) { + auto* state = GetStreamState(stream); + if (!env()->can_call_into_js() || state == nullptr || + !state->wants_trailers) { + return; + } + CallbackScope cb_scope(&stream); + stream.MakeCallback( + BindingData::Get(env()).stream_trailers_callback(), 0, nullptr); + } + void BuildOriginPayload() { // Build the serialized ORIGIN frame payload from the SNI configuration. // Each origin entry is: 2-byte BE length + origin string. @@ -831,7 +947,7 @@ class Http3ApplicationImpl final : public Session::Application { "HTTP/3 application beginning initial block of headers for stream " "%" PRIi64, id); - stream->BeginHeaders(HeadersKind::INITIAL); + BeginHeaders(*stream, HeadersKind::INITIAL); } void OnReceiveHeader(stream_id id, std::unique_ptr header) { @@ -843,7 +959,7 @@ class Http3ApplicationImpl final : public Session::Application { Debug(&session(), "HTTP/3 application switching to hints headers for stream %" PRIi64, stream->id()); - stream->set_headers_kind(HeadersKind::HINTS); + GetOrCreateStreamState(*stream).headers_kind = HeadersKind::HINTS; } IF_QUIC_DEBUG(env()) { Debug(&session(), @@ -851,7 +967,7 @@ class Http3ApplicationImpl final : public Session::Application { header->name(), header->value()); } - stream->AddHeader(std::move(header)); + AddHeader(*stream, std::move(header)); } void OnEndHeaders(stream_id id, int fin) { @@ -861,8 +977,8 @@ class Http3ApplicationImpl final : public Session::Application { Debug(&session(), "HTTP/3 application received end of headers for stream %" PRIi64, id); - stream->EmitHeaders(); - // EmitHeaders() calls into JavaScript, which can synchronously destroy the + EmitHeaders(*stream); + // EmitHeaders calls into JavaScript, which can synchronously destroy the // stream. Its arena-backed state is released by Destroy(), so do not touch // the stream again if that happened. if (stream->is_destroyed()) return; @@ -884,7 +1000,7 @@ class Http3ApplicationImpl final : public Session::Application { Debug(&session(), "HTTP/3 application beginning block of trailers for stream %" PRIi64, id); - stream->BeginHeaders(HeadersKind::TRAILING); + BeginHeaders(*stream, HeadersKind::TRAILING); } void OnReceiveTrailer(stream_id id, std::unique_ptr header) { @@ -897,7 +1013,7 @@ class Http3ApplicationImpl final : public Session::Application { header->name(), header->value()); } - stream->AddHeader(std::move(header)); + AddHeader(*stream, std::move(header)); } void OnEndTrailers(stream_id id, int fin) { @@ -907,8 +1023,8 @@ class Http3ApplicationImpl final : public Session::Application { Debug(&session(), "HTTP/3 application received end of trailers for stream %" PRIi64, id); - stream->EmitHeaders(); - // EmitHeaders() calls into JavaScript, which can synchronously destroy the + EmitHeaders(*stream); + // EmitHeaders calls into JavaScript, which can synchronously destroy the // stream. Its arena-backed state is released by Destroy(), so do not touch // the stream again if that happened. if (stream->is_destroyed()) return; @@ -1112,7 +1228,7 @@ class Http3ApplicationImpl final : public Session::Application { if (stream->is_eos()) { *pflags |= NGHTTP3_DATA_FLAG_EOF; - if (stream->wants_trailers()) { + if (app.GetOrCreateStreamState(*stream).wants_trailers) { *pflags |= NGHTTP3_DATA_FLAG_NO_END_STREAM; app.pending_trailers_stream_ = id; } @@ -1131,7 +1247,7 @@ class Http3ApplicationImpl final : public Session::Application { return; case bob::Status::STATUS_EOS: *pflags |= NGHTTP3_DATA_FLAG_EOF; - if (stream->wants_trailers()) { + if (app.GetOrCreateStreamState(*stream).wants_trailers) { *pflags |= NGHTTP3_DATA_FLAG_NO_END_STREAM; app.pending_trailers_stream_ = id; } diff --git a/src/quic/streams.cc b/src/quic/streams.cc index 1f8761c75049..9a11bb4d591e 100644 --- a/src/quic/streams.cc +++ b/src/quic/streams.cc @@ -26,12 +26,10 @@ using v8::BackingStore; using v8::BackingStoreInitializationMode; using v8::BigInt; using v8::FunctionCallbackInfo; -using v8::Global; using v8::HandleScope; using v8::Integer; using v8::Just; using v8::Local; -using v8::LocalVector; using v8::Maybe; using v8::Nothing; using v8::Object; @@ -57,14 +55,10 @@ namespace quic { V(HAS_READER, has_reader, uint8_t) \ /* Set when the stream has a block event handler */ \ V(WANTS_BLOCK, wants_block, uint8_t) \ - /* Set when the stream has a headers event handler */ \ - V(WANTS_HEADERS, wants_headers, uint8_t) \ /* Set when the stream has a reset event handler */ \ V(WANTS_RESET, wants_reset, uint8_t) \ /* Set when the stream has a stop sending event handler */ \ V(WANTS_STOP_SENDING, wants_stop_sending, uint8_t) \ - /* Set when the stream has a trailers event handler */ \ - V(WANTS_TRAILERS, wants_trailers, uint8_t) \ /* True when 0-RTT early data was received */ \ V(RECEIVED_EARLY_DATA, received_early_data, uint8_t) \ V(WRITE_DESIRED_SIZE, write_desired_size, uint32_t) \ @@ -98,7 +92,6 @@ namespace quic { #define STREAM_JS_METHODS(V) \ V(AttachSource, attachSource, false) \ V(Destroy, destroy, false) \ - V(SendHeaders, sendHeaders, false) \ V(StopSending, stopSending, false) \ V(ResetStream, resetStream, false) \ V(SetPriority, setPriority, false) \ @@ -227,17 +220,6 @@ void PendingStream::reject(QuicError error) { stream_->Destroy(error); } -struct Stream::PendingHeaders { - HeadersKind kind; - Global headers; - HeadersFlags flags; - PendingHeaders(HeadersKind kind_, Global headers_, HeadersFlags flags_) - : kind(kind_), headers(std::move(headers_)), flags(flags_) {} - DISALLOW_COPY_AND_MOVE(PendingHeaders) -}; - -// ============================================================================ - struct Stream::State { #define V(_, name, type) type name; STREAM_STATE(V) @@ -445,37 +427,6 @@ struct Stream::Impl { } } - // Sends a block of headers to the peer. If the stream is not yet open, - // the headers will be queued and sent immediately when the stream is - // opened. Returns false if the application does not support headers. - JS_METHOD(SendHeaders) { - Stream* stream; - ASSIGN_OR_RETURN_UNWRAP(&stream, args.This()); - CHECK(args[0]->IsUint32()); // Kind - CHECK(args[1]->IsArray()); // Headers - CHECK(args[2]->IsUint32()); // Flags - - HeadersKind kind = FromV8Value(args[0]); - Local headers = args[1].As(); - HeadersFlags flags = FromV8Value(args[2]); - - // If the stream is pending, the headers will be queued until the - // stream is opened, at which time the queued header block will be - // immediately sent when the stream is opened. If we already know - // that the application does not support headers, return false - // immediately so the JS side can throw an appropriate error. - if (stream->is_pending()) { - if (!stream->session().application().SupportsHeaders()) { - return args.GetReturnValue().Set(false); - } - stream->EnqueuePendingHeaders(kind, headers, flags); - return args.GetReturnValue().Set(true); - } - - args.GetReturnValue().Set(stream->session().application().SendHeaders( - *stream, kind, headers, flags)); - } - // Tells the peer to stop sending data for this stream. This has the effect // of shutting down the readable side of the stream for this peer. Any data // that has already been received is still readable. @@ -1057,25 +1008,6 @@ void Stream::InitPerContext(Realm* realm, Local target) { #undef V NODE_DEFINE_CONSTANT(target, IDX_STATS_STREAM_COUNT); - - constexpr int QUIC_STREAM_HEADERS_KIND_HINTS = - static_cast(HeadersKind::HINTS); - constexpr int QUIC_STREAM_HEADERS_KIND_INITIAL = - static_cast(HeadersKind::INITIAL); - constexpr int QUIC_STREAM_HEADERS_KIND_TRAILING = - static_cast(HeadersKind::TRAILING); - - constexpr int QUIC_STREAM_HEADERS_FLAGS_NONE = - static_cast(HeadersFlags::NONE); - constexpr int QUIC_STREAM_HEADERS_FLAGS_TERMINAL = - static_cast(HeadersFlags::TERMINAL); - - NODE_DEFINE_CONSTANT(target, QUIC_STREAM_HEADERS_KIND_HINTS); - NODE_DEFINE_CONSTANT(target, QUIC_STREAM_HEADERS_KIND_INITIAL); - NODE_DEFINE_CONSTANT(target, QUIC_STREAM_HEADERS_KIND_TRAILING); - - NODE_DEFINE_CONSTANT(target, QUIC_STREAM_HEADERS_FLAGS_NONE); - NODE_DEFINE_CONSTANT(target, QUIC_STREAM_HEADERS_FLAGS_TERMINAL); } Stream* Stream::From(void* stream_user_data) { @@ -1243,25 +1175,6 @@ void Stream::NotifyStreamOpened(stream_id id) { *this, priority_.priority, priority_.flags); priority_.pending = false; } - if (!pending_headers_queue_.empty()) { - if (!session().application().SupportsHeaders()) { - // Headers were enqueued while the application was not yet known - // (headers_supported == 0), and the negotiated application does - // not support headers. This is a fatal mismatch. - Destroy(QuicError::ForApplication( - session().application().GetInternalErrorCode())); - return; - } - decltype(pending_headers_queue_) queue; - pending_headers_queue_.swap(queue); - for (auto& headers : queue) { - session().application().SendHeaders( - *this, - headers->kind, - headers->headers.Get(env()->isolate()), - headers->flags); - } - } // If the stream is not a local unidirectional stream and is_readable is // false, then we should shutdown the streams readable side now. if (!is_local_unidirectional() && !is_readable()) { @@ -1277,6 +1190,15 @@ void Stream::NotifyStreamOpened(stream_id id) { // since the stream likely hasn't had any opporunity to get blocked // yet, but just for completeness, let's make sure. if (outbound_) session().ResumeStream(id); + + // This may make application data sendable, so keep it as the final action: + // sending can eventually call into JavaScript and destroy the stream. + BaseObjectPtr self(this); + auto& application = session().application(); + error_code internal_error = application.GetInternalErrorCode(); + if (!application.StreamOpened(*this) && !is_destroyed()) { + Destroy(QuicError::ForApplication(internal_error)); + } } void Stream::NotifyReadableEnded(error_code code) { @@ -1291,14 +1213,6 @@ void Stream::NotifyWritableEnded(error_code code) { ngtcp2_conn_shutdown_stream_write(session(), 0, id(), code); } -void Stream::EnqueuePendingHeaders(HeadersKind kind, - Local headers, - HeadersFlags flags) { - Debug(this, "Enqueuing headers for pending stream"); - pending_headers_queue_.push_back(std::make_unique( - kind, Global(env()->isolate(), headers), flags)); -} - bool Stream::is_pending() const { return state()->pending; } @@ -1336,6 +1250,10 @@ uint64_t Stream::last_activity_timestamp() const { return ts != 0 ? ts : stats()->created_at; } +void Stream::RecordReceivedActivity() { + STAT_RECORD_TIMESTAMP(Stats, received_at); +} + bool Stream::is_local_unidirectional() const { return direction() == Direction::UNIDIRECTIONAL && ngtcp2_conn_is_local_stream(*session_, id()); @@ -1350,10 +1268,6 @@ bool Stream::is_eos() const { return state()->fin_sent; } -bool Stream::wants_trailers() const { - return state()->wants_trailers; -} - void Stream::set_early() { state()->received_early_data = 1; } @@ -1587,28 +1501,6 @@ int Stream::DoPull(bob::Next next, return outbound_->Pull(std::move(next), options, data, count, max_count_hint); } -void Stream::BeginHeaders(HeadersKind kind) { - headers_length_ = 0; - headers_.clear(); - set_headers_kind(kind); -} - -void Stream::set_headers_kind(HeadersKind kind) { - headers_kind_ = kind; -} - -bool Stream::AddHeader(std::unique_ptr
header) { - size_t len = header->length(); - if (!session_->application().CanAddHeader( - headers_.size(), headers_length_, len)) { - return false; - } - - headers_length_ += len; - headers_.push_back(std::move(header)); - return true; -} - void Stream::Acknowledge(size_t datalen) { if (outbound_ == nullptr) return; @@ -1680,6 +1572,7 @@ void Stream::Destroy(QuicError error) { // We are going to release our reference to the outbound_ queue here. outbound_.reset(); + application_state_.reset(); // EndReadable() above already flushed accumulated data. Just release // the ring buffer memory. @@ -1959,42 +1852,6 @@ void Stream::EmitClose(const QuicError& error) { MakeCallback(BindingData::Get(env()).stream_close_callback(), 1, &err); } -void Stream::EmitHeaders() { - STAT_RECORD_TIMESTAMP(Stats, received_at); - // state()->wants_headers will be set from the javascript side if the - // stream object has a handler for the headers event. - if (!env()->can_call_into_js() || !state()->wants_headers) { - headers_.clear(); - return; - } - CallbackScope cb_scope(this); - - auto& binding = BindingData::Get(env()); - size_t count = headers_.size() * 2; - LocalVector values(env()->isolate(), count); - - for (size_t i = 0; i < headers_.size(); i++) { - Local name; - Local value; - if (!headers_[i]->GetName(&binding).ToLocal(&name) || - !headers_[i]->GetValue(&binding).ToLocal(&value)) [[unlikely]] { - headers_.clear(); - return; - } - values[i * 2] = name; - values[i * 2 + 1] = value; - } - - headers_.clear(); - - Local argv[] = { - Array::New(env()->isolate(), values.data(), count), - Integer::NewFromUnsigned(env()->isolate(), - static_cast(headers_kind_))}; - - MakeCallback(binding.stream_headers_callback(), arraysize(argv), argv); -} - void Stream::EmitReset(const QuicError& error) { // state()->wants_reset will be set from the javascript side if the // stream object has a handler for the reset event. @@ -2019,16 +1876,6 @@ void Stream::EmitStopSending(const QuicError& error) { MakeCallback(BindingData::Get(env()).stream_stop_sending_callback(), 1, &err); } -void Stream::EmitWantTrailers() { - // state()->wants_trailers will be set from the javascript side if the - // stream object has a handler for the trailers event. - if (!env()->can_call_into_js() || !state()->wants_trailers) { - return; - } - CallbackScope cb_scope(this); - MakeCallback(BindingData::Get(env()).stream_trailers_callback(), 0, nullptr); -} - // ============================================================================ void Stream::Schedule(Queue* queue) { diff --git a/src/quic/streams.h b/src/quic/streams.h index cd4849aae277..1ea85f216524 100644 --- a/src/quic/streams.h +++ b/src/quic/streams.h @@ -10,7 +10,6 @@ #include #include #include -#include #include #include "bindingdata.h" #include "data.h" @@ -22,6 +21,14 @@ namespace node::quic { class Session; class Stream; +// Optional per-stream state owned by the negotiated application protocol. +// Stream deliberately treats this as opaque so protocol semantics do not +// become part of the transport stream abstraction. +class StreamApplicationState { + public: + virtual ~StreamApplicationState() = default; +}; + // An elastic ring buffer used by Stream to coalesce received data before // flushing it into the DataQueue. This avoids creating many small V8 // BackingStore allocations from per-QUIC-frame ngtcp2 callbacks. Data is @@ -179,11 +186,6 @@ class PendingStream final { // that the stream is gone. Any data that has already been received and is in // the inbound queue is preserved and may be read by the application. // -// QUIC streams in general do not have headers. Some QUIC applications, however, -// may associate headers with the stream (HTTP/3 for instance). As a -// convenience, the Stream class will hold onto these headers for the -// application. -// // Streams may be created in a pending state. This means that while the Stream // object is created, it has not yet been opened in ngtcp2 and therefore has // no official status yet. Certain operations can still be performed on the @@ -202,8 +204,6 @@ class Stream final : public AsyncWrap, public Ngtcp2Source, public DataQueue::BackpressureListener { public: - using Header = NgHeaderBase; - // Acquire a DataQueue from the given value if it is valid. The return // follows the typical V8 rules for Maybe types. If an error occurs, // the Maybe will be empty and an exception will be set on the isolate. @@ -263,6 +263,10 @@ class Stream final : public AsyncWrap, // otherwise falls back to created_at. Returns 0 if neither is set. uint64_t last_activity_timestamp() const; + // Records protocol-level receive activity that does not pass through + // ReceiveData(), such as application framing metadata. + void RecordReceivedActivity(); + // True if this stream was created in a pending state and is still waiting // to be created. bool is_pending() const; @@ -276,9 +280,6 @@ class Stream final : public AsyncWrap, // data to be acknowledged by the remote peer. bool is_eos() const; - // True if the stream wants to send trailing headers after the body. - bool wants_trailers() const; - // Marks this stream as having received 0-RTT early data. void set_early(); @@ -297,6 +298,17 @@ class Stream final : public AsyncWrap, // Returns the Blob::Reader for the inbound data, or nullptr. Blob::Reader* reader() const; + StreamApplicationState* application_state() const { + return application_state_.get(); + } + void set_application_state( + std::unique_ptr application_state) { + application_state_ = std::move(application_state); + } + + StreamPriority priority() const { return priority_.priority; } + StreamPriorityFlags priority_flags() const { return priority_.flags; } + // Called by the session/application to indicate that the specified number // of bytes have been acknowledged by the peer. void Acknowledge(size_t datalen); @@ -308,6 +320,10 @@ class Stream final : public AsyncWrap, // acknowledged to have been received by the peer. void Commit(size_t datalen, bool fin = false); + // Updates the write_desired_size state field based on current flow control + // and outbound buffer state. Emits drain if transitioning from 0 to > 0. + void UpdateWriteDesiredSize(); + void EndWritable(); void EndReadable(std::optional maybe_final_size = std::nullopt); void EntryRead(size_t amount) override; @@ -355,18 +371,8 @@ class Stream final : public AsyncWrap, // that has already been received is still readable. void SendStopSending(error_code code); - // Currently, only HTTP/3 streams support headers. These methods are here - // to support that. They are not used when using any other QUIC application. - - void BeginHeaders(HeadersKind kind); - void set_headers_kind(HeadersKind kind); - // Returns false if the header cannot be added. This will typically happen - // if the application does not support headers, a maximum number of headers - // have already been added, or the maximum total header length is reached. - bool AddHeader(std::unique_ptr
header); - // TODO(@jasnell): Implement MemoryInfo to track outbound_, inbound_, - // reader_, headers_, and pending_headers_queue_. + // reader_, and application_state_. SET_NO_MEMORY_INFO() SET_MEMORY_INFO_NAME(Stream) SET_SELF_SIZE(Stream) @@ -387,7 +393,6 @@ class Stream final : public AsyncWrap, private: struct Impl; - struct PendingHeaders; class Outbound; @@ -437,11 +442,6 @@ class Stream final : public AsyncWrap, // Notifies the JavaScript side that the peer asked it to stop sending. void EmitStopSending(const QuicError& error); - // Notifies the JavaScript side that the application is ready to receive - // trailing headers. Any trailing headers must be sent immediately, and - // synchronously when this callback is triggered. - void EmitWantTrailers(); - // Notifies the JavaScript side that sending data on the stream has been // blocked because of flow control restriction. void EmitBlocked(); @@ -450,23 +450,12 @@ class Stream final : public AsyncWrap, // for more data. Fires when write_desired_size transitions from 0 to > 0. void EmitDrain(); - // Updates the write_desired_size state field based on current flow control - // and outbound buffer state. Emits drain if transitioning from 0 to > 0. - void UpdateWriteDesiredSize(); - - // Delivers the set of inbound headers that have been collected. - void EmitHeaders(); - void NotifyReadableEnded(error_code code); void NotifyWritableEnded(error_code code); // When a pending stream is finally opened, the NotifyStreamOpened method // will be called and the id will be assigned. void NotifyStreamOpened(stream_id id); - void EnqueuePendingHeaders(HeadersKind kind, - v8::Local headers, - HeadersFlags flags); - ArenaSlotBase stats_slot_; ArenaSlotBase state_slot_; BaseObjectWeakPtr session_; @@ -474,6 +463,7 @@ class Stream final : public AsyncWrap, std::shared_ptr inbound_; BaseObjectWeakPtr reader_; std::unique_ptr recv_accumulator_; + std::unique_ptr application_state_; // Bytes delivered to ReceiveData() that still hold inbound flow control // credit. Returned incrementally as the consumer reads them, and in bulk @@ -488,7 +478,6 @@ class Stream final : public AsyncWrap, // and the stream id will be assigned. std::optional> maybe_pending_stream_ = std::nullopt; - std::vector> pending_headers_queue_; error_code pending_close_read_code_ = 0; error_code pending_close_write_code_ = 0; @@ -499,26 +488,8 @@ class Stream final : public AsyncWrap, }; StoredPriority priority_; - const StoredPriority& stored_priority() const { return priority_; } - - // The headers_ field holds a block of headers that have been received and - // are being buffered for delivery to the JavaScript side. Headers are - // stored as C++ objects during collection (AddHeader) and converted to - // V8 strings only when emitted (EmitHeaders), avoiding StrongRootAllocator - // mutex contention on the per-header hot path. - std::vector> headers_; - - // The headers_kind_ field indicates the kind of headers that are being - // buffered. - HeadersKind headers_kind_ = HeadersKind::INITIAL; - - // The headers_length_ field holds the total length of the headers that have - // been buffered. - size_t headers_length_ = 0; - friend struct Impl; friend class PendingStream; - friend class Http3ApplicationImpl; friend class DefaultApplication; public: diff --git a/test/parallel/test-quic-h3-header-interest.mjs b/test/parallel/test-quic-h3-header-interest.mjs new file mode 100644 index 000000000000..4fe05b6f6d3e --- /dev/null +++ b/test/parallel/test-quic-h3-header-interest.mjs @@ -0,0 +1,76 @@ +// Flags: --experimental-quic --experimental-stream-iter --no-warnings + +// Verify HTTP/3 header interest is tracked independently of onheaders and +// that pre-set trailing headers keep the response open until they are sent. + +import { hasQuic, skip, mustCall } from '../common/index.mjs'; +import assert from 'node:assert'; +import * as fixtures from '../common/fixtures.mjs'; + +if (!hasQuic) { + skip('QUIC is not enabled'); +} + +const { createPrivateKey } = await import('node:crypto'); +const { listen, connect } = await import('node:quic'); +const { bytes } = await import('stream/iter'); + +const key = createPrivateKey(fixtures.readKey('agent1-key.pem')); +const cert = fixtures.readKey('agent1-cert.pem'); +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); +const serverDone = Promise.withResolvers(); + +const serverEndpoint = await listen(mustCall(async (serverSession) => { + serverSession.onstream = mustCall(async (stream) => { + await stream.closed; + serverSession.close(); + serverDone.resolve(); + }); +}), { + sni: { '*': { keys: [key], certs: [cert] } }, + onheaders: mustCall(function() { + this.sendInformationalHeaders({ + ':status': '103', + 'link': '; rel=preload', + }); + this.sendHeaders({ ':status': '200' }); + this.pendingTrailers = { 'x-checksum': 'abc123' }; + const writer = this.writer; + writer.writeSync(encoder.encode('body')); + writer.endSync(); + }), +}); + +const clientSession = await connect(serverEndpoint.address, { + servername: 'localhost', + verifyPeer: 'manual', +}); +await clientSession.opened; + +const infoReceived = Promise.withResolvers(); +const trailersReceived = Promise.withResolvers(); +const stream = await clientSession.createBidirectionalStream({ + headers: { + ':method': 'GET', + ':path': '/', + ':scheme': 'https', + ':authority': 'localhost', + }, + oninfo: mustCall((headers) => { + assert.strictEqual(headers[':status'], 103); + infoReceived.resolve(); + }), + ontrailers: mustCall((headers) => { + assert.strictEqual(headers['x-checksum'], 'abc123'); + trailersReceived.resolve(); + }), +}); + +assert.strictEqual(decoder.decode(await bytes(stream)), 'body'); +await Promise.all([infoReceived.promise, trailersReceived.promise]); +assert.strictEqual(stream.headers[':status'], 200); + +await Promise.all([stream.closed, serverDone.promise]); +await clientSession.close(); +await serverEndpoint.close(); From 29d3406b6c9e2a541e8007125103744a8e79ea2c Mon Sep 17 00:00:00 2001 From: Philipp Dunkel Date: Sun, 6 Sep 2026 11:34:15 +0200 Subject: [PATCH 033/217] vfs: apply open(2) effects to ZipProvider handles A ZipProvider handle keeps its content in memory and adds the entry to the archive when it is closed, and only if something was written. The effects a real `open(2)` has at open time are therefore lost, and so is the metadata the entry already carried: * `open(path, 'w')` followed by `close()` neither truncates an existing entry nor creates a missing one; the same holds for "a" on a missing file. Tools that touch or truncate by open-then-close do nothing. * Rewriting an entry (append, or an in-place write through "r+") re-adds it with the `mode` argument `open()` received (fs's default 0o666), not the mode the entry had, so a 0o755 script silently loses its executable bit. * `fstat` on a handle reports that same `open()` mode and the current time instead of the entry's mode and modification time. * Renaming a file onto an existing directory succeeds and leaves a name that is both a file and a directory; real file systems refuse with EISDIR. This adds a test for each of these against a mounted ZipBuffer, stating the real-fs outcome as the expectation. Proposed solution: mark the handle dirty at open time when the flags imply creation or truncation, so close always commits; carry the existing entry's mode and modification time on the handle, use them for `fstat` and for the re-added entry, and only fall back to the `open()` mode for a newly created entry; and reject `rename` onto an existing directory with EISDIR before touching the archive. Signed-off-by: Philipp Dunkel PR-URL: https://github.com/nodejs/node/pull/65853 Reviewed-By: James M Snell Reviewed-By: Matteo Collina --- test/parallel/test-vfs-zip-provider-commit.js | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 test/parallel/test-vfs-zip-provider-commit.js diff --git a/test/parallel/test-vfs-zip-provider-commit.js b/test/parallel/test-vfs-zip-provider-commit.js new file mode 100644 index 000000000000..6a2874be4ee0 --- /dev/null +++ b/test/parallel/test-vfs-zip-provider-commit.js @@ -0,0 +1,102 @@ +// Flags: --experimental-vfs +'use strict'; + +// A ZipProvider handle commits its content to the archive when it is closed. +// The effects `open(2)` has at open time, and the metadata an entry already +// carries, must survive that model: opening with "w" creates or truncates +// even without a write, rewriting an entry keeps its mode, fstat reports the +// entry's mode, and a file cannot be renamed onto a directory. Each case +// states the real-fs outcome as the expectation. Cases are independent so +// the runner reports each one. + +require('../common'); +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const zlib = require('zlib'); +const vfs = require('node:vfs'); +const { test } = require('node:test'); + +// Builds a writable in-memory archive from [name, content, options] triples +// and mounts it, returning the mount point. +function mountZip(entries) { + const list = entries.map(({ 0: name, 1: content, 2: options }) => + zlib.ZipEntry.createSync(name, Buffer.from(content), options)); + const chunks = []; + for (const chunk of zlib.createZipArchiveSync(list)) chunks.push(chunk); + const provider = new vfs.ZipProvider(new zlib.ZipBuffer(Buffer.concat(chunks))); + return vfs.create(provider).mount(); +} + +test('opening an existing file with "w" truncates it even without a write', () => { + const file = path.join(mountZip([['f.txt', 'hello']]), 'f.txt'); + fs.closeSync(fs.openSync(file, 'w')); + assert.strictEqual(fs.readFileSync(file, 'utf8'), ''); +}); + +test('opening a new file with "w" creates it even without a write', () => { + const file = path.join(mountZip([['f.txt', 'hello']]), 'new.txt'); + fs.closeSync(fs.openSync(file, 'w')); + assert.strictEqual(fs.existsSync(file), true); +}); + +test('opening a new file with "a" creates it even without a write', () => { + const file = path.join(mountZip([['f.txt', 'hello']]), 'log.txt'); + fs.closeSync(fs.openSync(file, 'a')); + assert.strictEqual(fs.existsSync(file), true); +}); + +test('appending keeps the entry mode', () => { + const file = path.join(mountZip([['x.sh', 'a', { mode: 0o755 }]]), 'x.sh'); + assert.strictEqual(fs.statSync(file).mode & 0o777, 0o755); + fs.appendFileSync(file, 'b'); + assert.strictEqual(fs.statSync(file).mode & 0o777, 0o755); + assert.strictEqual(fs.readFileSync(file, 'utf8'), 'ab'); +}); + +test('an in-place write keeps the entry mode', () => { + const file = path.join(mountZip([['x.sh', 'abc', { mode: 0o755 }]]), 'x.sh'); + const fd = fs.openSync(file, 'r+'); + fs.writeSync(fd, Buffer.from('Z'), 0, 1, 0); + fs.closeSync(fd); + assert.strictEqual(fs.statSync(file).mode & 0o777, 0o755); +}); + +test('a new file gets the mode passed to open', () => { + const file = path.join(mountZip([['f.txt', 'hello']]), 'new.sh'); + const fd = fs.openSync(file, 'w', 0o700); + fs.writeSync(fd, Buffer.from('#!')); + fs.closeSync(fd); + assert.strictEqual(fs.statSync(file).mode & 0o777, 0o700); +}); + +test('fstat reports the entry mode, not the open() mode argument', () => { + const file = path.join(mountZip([['x.sh', 'a', { mode: 0o755 }]]), 'x.sh'); + const fd = fs.openSync(file, 'r'); + try { + assert.strictEqual(fs.fstatSync(fd).mode & 0o777, 0o755); + } finally { + fs.closeSync(fd); + } +}); + +test('fstat reports the entry modification time', () => { + const modified = new Date('2020-01-02T03:04:05Z'); + const file = path.join(mountZip([['f.txt', 'a', { modified }]]), 'f.txt'); + const fd = fs.openSync(file, 'r'); + try { + // ZIP timestamps have two-second resolution, so compare at that grain. + assert.strictEqual(Math.floor(fs.fstatSync(fd).mtimeMs / 2000), + Math.floor(modified.getTime() / 2000)); + } finally { + fs.closeSync(fd); + } +}); + +test('renaming a file onto an existing directory fails with EISDIR', () => { + const mount = mountZip([['dir/', ''], ['f', 'x']]); + assert.throws(() => fs.renameSync(path.join(mount, 'f'), path.join(mount, 'dir')), + { code: 'EISDIR' }); + assert.strictEqual(fs.statSync(path.join(mount, 'dir')).isDirectory(), true); + assert.strictEqual(fs.readFileSync(path.join(mount, 'f'), 'utf8'), 'x'); +}); From 5c4d3190284db7f1030ceaedc27f081c8286a354 Mon Sep 17 00:00:00 2001 From: Philipp Dunkel Date: Sun, 6 Sep 2026 13:09:41 +0200 Subject: [PATCH 034/217] vfs: commit ZipProvider handles the way open(2) does A ZipProvider handle adds its entry to the archive when it is closed. Give that model the effects a real open(2) has up front, and keep the metadata an entry already carries: * A handle whose flags create or truncate the file starts out dirty, so closing it without a write still creates the missing entry or truncates the existing one. * The handle remembers the entry's own mode and modification time. The re-added entry keeps that mode instead of taking the `mode` argument `open()` was given (fs's default 0o666), so a 0o755 script survives an append or an in-place write; only a newly created file takes the mode from `open()`. * `fstat` reports that mode and, until the handle has changed the file, that modification time. * `rename` refuses to move a file onto an existing directory with EISDIR before touching the archive. Signed-off-by: Philipp Dunkel PR-URL: https://github.com/nodejs/node/pull/65853 Reviewed-By: James M Snell Reviewed-By: Matteo Collina --- lib/internal/vfs/providers/ziparchive.js | 32 +++++++++++++++++++----- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/lib/internal/vfs/providers/ziparchive.js b/lib/internal/vfs/providers/ziparchive.js index 402c92b8b659..e5ecfc7afd78 100644 --- a/lib/internal/vfs/providers/ziparchive.js +++ b/lib/internal/vfs/providers/ziparchive.js @@ -104,6 +104,8 @@ class ZipFileHandle extends VirtualFileHandle { #buffer; #size; #dirty = false; + #entryMode; + #modified; /** * @param {string} path @@ -114,8 +116,15 @@ class ZipFileHandle extends VirtualFileHandle { * @param {Buffer} initial The entry's current decompressed content, or an * empty buffer for a new/truncated file */ - constructor(path, flags, mode, source, name, initial) { + constructor(path, flags, mode, source, name, initial, entry = null, dirty = false) { super(path, flags, mode); + // An existing entry keeps its own mode and modification time across a + // rewrite; only a newly created file takes the mode open() was given. + this.#entryMode = entry === null ? this.mode : (entry.mode || 0o644); + this.#modified = entry === null ? null : entry.modified; + // Creation and truncation take effect at open time on a real file, so + // such a handle is committed on close even when nothing is written. + this.#dirty = dirty; this.#source = source; this.#name = name; this.#buffer = initial; @@ -206,8 +215,13 @@ class ZipFileHandle extends VirtualFileHandle { this.#doWriteFile(data, options); } + // Reports the entry's own mode and, until the handle has changed the file, + // its own modification time; once dirty the file is as new as its close. #doStat() { - return createFileStats(this.#size, { mode: this.mode }); + return createFileStats(this.#size, { + mode: this.#entryMode, + mtimeMs: this.#dirty || this.#modified === null ? undefined : this.#modified.getTime(), + }); } async stat(options) { return this.#doStat(); @@ -235,13 +249,13 @@ class ZipFileHandle extends VirtualFileHandle { async close() { if (this.#dirty && isWritableFlag(this.flags)) { - await this.#source.add(this.#name, this.#buffer.subarray(0, this.#size), { mode: this.mode }); + await this.#source.add(this.#name, this.#buffer.subarray(0, this.#size), { mode: this.#entryMode }); } await super.close(); } closeSync() { if (this.#dirty && isWritableFlag(this.flags)) { - this.#source.addSync(this.#name, this.#buffer.subarray(0, this.#size), { mode: this.mode }); + this.#source.addSync(this.#name, this.#buffer.subarray(0, this.#size), { mode: this.#entryMode }); } super.closeSync(); } @@ -339,7 +353,8 @@ class ZipProvider extends VirtualProvider { if (exists && !isWriteTruncate(flags)) { initial = await fileEntry.content(); } - return new ZipFileHandle(path, flags, mode, this.#source, name, initial); + return new ZipFileHandle(path, flags, mode, this.#source, name, initial, + fileEntry, !exists || isWriteTruncate(flags)); } openSync(path, flags, mode) { const name = normalize(path); @@ -361,7 +376,8 @@ class ZipProvider extends VirtualProvider { if (exists && !isWriteTruncate(flags)) { initial = fileEntry.contentSync(); } - return new ZipFileHandle(path, flags, mode, this.#source, name, initial); + return new ZipFileHandle(path, flags, mode, this.#source, name, initial, + fileEntry, !exists || isWriteTruncate(flags)); } async stat(path, options) { @@ -519,6 +535,9 @@ class ZipProvider extends VirtualProvider { const newName = normalize(newPath); const entry = await this.#getEntry(oldName); if (entry === null) throw createENOENT('rename', oldPath); + // A file cannot take a directory's name; the archive would otherwise + // hold both under it. + if (this.#isDirectory(newName)) throw createEISDIR('rename', newPath); const content = await entry.content(); await this.#source.add(newName, content, { mode: entry.mode || undefined, @@ -533,6 +552,7 @@ class ZipProvider extends VirtualProvider { const newName = normalize(newPath); const entry = this.#getEntrySync(oldName); if (entry === null) throw createENOENT('rename', oldPath); + if (this.#isDirectory(newName)) throw createEISDIR('rename', newPath); const content = entry.contentSync(); this.#source.addSync(newName, content, { mode: entry.mode || undefined, From 08f96f142e84b0a146ec42f7431b6dcb84defa14 Mon Sep 17 00:00:00 2001 From: Philipp Dunkel Date: Sun, 6 Sep 2026 16:32:10 +0200 Subject: [PATCH 035/217] vfs: give ZipProvider option bags a null prototype The options passed to `createFileStats()` and to the archive's `add()` and `addSync()` are plain literals, so a property added to `Object.prototype` would reach those callees as if it had been passed on purpose. Create them with a null prototype so only the fields set here are visible. Refs: https://github.com/nodejs/node/pull/65853 Signed-off-by: Philipp Dunkel PR-URL: https://github.com/nodejs/node/pull/65853 Reviewed-By: James M Snell Reviewed-By: Matteo Collina --- lib/internal/vfs/providers/ziparchive.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/internal/vfs/providers/ziparchive.js b/lib/internal/vfs/providers/ziparchive.js index e5ecfc7afd78..9f4ecceee460 100644 --- a/lib/internal/vfs/providers/ziparchive.js +++ b/lib/internal/vfs/providers/ziparchive.js @@ -219,6 +219,7 @@ class ZipFileHandle extends VirtualFileHandle { // its own modification time; once dirty the file is as new as its close. #doStat() { return createFileStats(this.#size, { + __proto__: null, mode: this.#entryMode, mtimeMs: this.#dirty || this.#modified === null ? undefined : this.#modified.getTime(), }); @@ -249,13 +250,15 @@ class ZipFileHandle extends VirtualFileHandle { async close() { if (this.#dirty && isWritableFlag(this.flags)) { - await this.#source.add(this.#name, this.#buffer.subarray(0, this.#size), { mode: this.#entryMode }); + await this.#source.add(this.#name, this.#buffer.subarray(0, this.#size), + { __proto__: null, mode: this.#entryMode }); } await super.close(); } closeSync() { if (this.#dirty && isWritableFlag(this.flags)) { - this.#source.addSync(this.#name, this.#buffer.subarray(0, this.#size), { mode: this.#entryMode }); + this.#source.addSync(this.#name, this.#buffer.subarray(0, this.#size), + { __proto__: null, mode: this.#entryMode }); } super.closeSync(); } From 7f5168149c15ffa0239f7541071662d72f6564af Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Mon, 7 Sep 2026 10:45:23 +0200 Subject: [PATCH 036/217] test: fix flaky test-bench-stream testDeliveryDoesNotConsumeTimeout gave the benchmark a 20ms timeout while stalling the consumer for 50ms. Only delivery time is credited back to the deadline, so the 32 samples still had to run within 20ms, which is about a 10x margin on an idle machine and not enough on a loaded CI runner. Scale the timeout and the stall together so the benchmark's own work gets 500ms of headroom while the consumer still stalls for longer than the timeout. Signed-off-by: Matteo Collina PR-URL: https://github.com/nodejs/node/pull/65874 Reviewed-By: Filip Skokan Reviewed-By: Paolo Insogna Reviewed-By: James M Snell --- test/parallel/test-bench-stream.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/parallel/test-bench-stream.js b/test/parallel/test-bench-stream.js index 655a1a229130..8fe0870ae55a 100644 --- a/test/parallel/test-bench-stream.js +++ b/test/parallel/test-bench-stream.js @@ -4,7 +4,7 @@ const common = require('../common'); const assert = require('assert'); const { createRunner } = require('node:bench'); -const { setImmediate, setTimeout } = require('timers/promises'); +const { setImmediate } = require('timers/promises'); function recordSample(b) { b.record({ @@ -151,15 +151,21 @@ async function testCancellationCompletesBenchmarks() { async function testDeliveryDoesNotConsumeTimeout() { const runner = createRunner({ yieldBetweenSamples: false }); + // The timeout only has to cover the benchmark's own work, which is 32 samples + // that do nothing but record a fixed value. Keep it generous so that a loaded + // machine cannot exhaust it on its own, and keep the consumer stalled for + // longer than the timeout so that the benchmark can only complete when the + // time spent delivering records is excluded from the timeout. + const timeout = common.platformTimeout(500); const completion = runner.bench('slow consumer', { samples: 32, - timeout: common.platformTimeout(20), + timeout, }, recordSample); const stream = runner.run(); const iterator = stream[Symbol.asyncIterator](); await iterator.next(); - await setTimeout(common.platformTimeout(50)); + await setImmediate(); for (;;) { const next = await iterator.next(); if (next.done) break; From fb8a97f46b6928f323ac89c5f7b5b65f03ace198 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Tue, 8 Sep 2026 13:56:03 -0700 Subject: [PATCH 037/217] stream: improve handling of falsy errors in stream/iter Signed-off-by: James M Snell Assisted-by: Opencode PR-URL: https://github.com/nodejs/node/pull/65864 Reviewed-By: Trivikram Kamat Reviewed-By: Tim Perry Reviewed-By: Benjamin Gruenbaum --- doc/api/stream_iter.md | 35 ++- lib/internal/abort_controller.js | 1 + lib/internal/fs/promises.js | 237 +++++++++------ lib/internal/quic/quic.js | 14 +- lib/internal/streams/iter/broadcast.js | 41 +-- lib/internal/streams/iter/classic.js | 206 +++++++++---- lib/internal/streams/iter/pull.js | 13 +- lib/internal/streams/iter/push.js | 53 ++-- lib/internal/streams/iter/share.js | 66 +++-- lib/internal/streams/iter/transform.js | 4 +- lib/internal/streams/iter/utils.js | 13 - .../test-fs-promises-file-handle-pull.js | 4 +- .../test-fs-promises-file-handle-writer.js | 55 +++- test/parallel/test-quic-stream-writer-api.mjs | 12 +- ...test-stream-iter-broadcast-backpressure.js | 17 +- .../test-stream-iter-reason-propagation.js | 271 ++++++++++++++++++ .../test-stream-iter-share-coverage.js | 105 ++++++- test/parallel/test-stream-iter-to-readable.js | 159 ++++++++-- .../test-stream-iter-writable-from.js | 241 +++++++++++----- .../test-stream-iter-writable-interop.js | 65 ++++- 20 files changed, 1230 insertions(+), 382 deletions(-) create mode 100644 test/parallel/test-stream-iter-reason-propagation.js diff --git a/doc/api/stream_iter.md b/doc/api/stream_iter.md index ab91bfdbb1dc..60453d12bf77 100644 --- a/doc/api/stream_iter.md +++ b/doc/api/stream_iter.md @@ -463,14 +463,15 @@ if (result < 0) { } ``` -#### `writer.fail(reason)` +#### `writer.fail([reason])` * `reason` {any} Put the writer into a terminal error state. If the writer is already closed or errored, this is a no-op. Unlike `write()` and `end()`, `fail()` is unconditionally synchronous because failing a writer is a pure state -transition with no async work to perform. +transition with no async work to perform. The reason is stored and propagated +without modification. If omitted, the reason is `undefined`. #### `writer[Symbol.asyncDispose]()` @@ -1295,9 +1296,10 @@ run().catch(console.error); #### `broadcast.cancel([reason])` -* `reason` {Error} +* `reason` {any} -Cancel the broadcast. All consumers receive an error. +Cancel the broadcast. If `reason` is provided, all consumers reject with that +exact reason. If it is omitted, consumers complete normally. #### `broadcast.consumerCount` @@ -1401,9 +1403,10 @@ Create a {Share} from an existing source. #### `share.cancel([reason])` -* `reason` {Error} +* `reason` {any} -Cancel the share. All consumers receive an error. +Cancel the share. If `reason` is provided, all consumers reject with that exact +reason. If it is omitted, consumers complete normally. #### `share.consumerCount` @@ -1471,9 +1474,10 @@ The number of chunks currently buffered. #### `share.cancel([reason])` -* `reason` {Error} +* `reason` {any} -Cancel the share. All consumers receive an error. +Cancel the share. If `reason` is provided, all consumers throw that exact +reason. If it is omitted, consumers complete normally. #### `share.consumerCount` @@ -1593,6 +1597,11 @@ the synchronous Writer methods (`writeSync`, `writevSync`, `endSync`) always return `false` or `-1`, deferring to the async path. The per-write `options.signal` parameter from the Writer interface is also ignored. +If `writer.fail(reason)` receives a non-Error reason, the classic Writable is +destroyed with an `ERR_FALSY_VALUE_REJECTION` or `ERR_OPERATION_FAILED` error. +Its `reason` property contains the original value, which remains the Writer's +stored failure reason. + The result is cached per instance and backpressure policy -- calling `fromWritable()` twice with the same stream and `backpressure` option returns the same Writer. @@ -1649,6 +1658,11 @@ Creates a byte-mode [`stream.Readable`][] from the `source` (the native batch format used by the stream/iter API). Each `Uint8Array` in a yielded batch is pushed as a separate chunk into the Readable. +Classic streams cannot represent arbitrary values as emitted errors. A +non-Error reason is wrapped in an `ERR_FALSY_VALUE_REJECTION` or +`ERR_OPERATION_FAILED` error whose `reason` property contains the original +value. + ```mjs import { createWriteStream } from 'node:fs'; import { from, pull, toReadable } from 'node:stream/iter'; @@ -1729,6 +1743,11 @@ sync path returns `false`. Similarly, `_final()` tries `endSync()` before `end()`. When the sync path succeeds, the callback is deferred via `queueMicrotask` to preserve the async resolution contract. +Classic stream callbacks cannot represent arbitrary values as errors. A +non-Error reason is wrapped in an `ERR_FALSY_VALUE_REJECTION` or +`ERR_OPERATION_FAILED` error before it is passed to the callback. The error's +`reason` property contains the original value. + The Writable's `highWaterMark` is set to `Number.MAX_SAFE_INTEGER` to effectively disable its internal buffering, allowing the underlying Writer to manage backpressure directly. diff --git a/lib/internal/abort_controller.js b/lib/internal/abort_controller.js index 48dcdaafba2f..ef30d9619749 100644 --- a/lib/internal/abort_controller.js +++ b/lib/internal/abort_controller.js @@ -642,6 +642,7 @@ module.exports = { AbortController, AbortSignal, ClonedAbortSignal, + abortSignal, aborted, transferableAbortSignal, transferableAbortController, diff --git a/lib/internal/fs/promises.js b/lib/internal/fs/promises.js index d5a6b9a2c853..8b511f572f0e 100644 --- a/lib/internal/fs/promises.js +++ b/lib/internal/fs/promises.js @@ -15,6 +15,7 @@ const { SafeArrayIterator, SafePromisePrototypeFinally, SafePromiseRace, + SafeSet, Symbol, SymbolAsyncDispose, SymbolAsyncIterator, @@ -537,9 +538,7 @@ if (getOptionValue('--experimental-stream-iter')) { // Signal-aware path while (remaining !== 0) { if (signal.aborted) { - throw signal.reason ?? - lazyDOMException('The operation was aborted', - 'AbortError'); + throw signal.reason; } const toRead = remaining > 0 ? MathMin(readSize, remaining) : readSize; @@ -745,9 +744,13 @@ if (getOptionValue('--experimental-stream-iter')) { let totalBytesWritten = 0; let closed = false; let closing = false; - let pendingEndPromise = null; - let error = null; - let asyncPending = false; + let pendingEnd = null; + let endingCleanup = false; + let errored = false; + let error; + let asyncPending = 0; + let released = false; + const pendingWrites = new SafeSet(); validateBoolean(autoClose, 'options.autoClose'); @@ -766,85 +769,124 @@ if (getOptionValue('--experimental-stream-iter')) { // Write a single buffer with EAGAIN retry (up to 5 retries). async function writeAll(buf, offset, length, position, signal) { - asyncPending = true; - try { - let retries = 0; - while (length > 0) { - const bytesWritten = (await PromisePrototypeThen( - binding.writeBuffer(fd, buf, offset, length, position, - kUsePromises), - undefined, - handleErrorFromBinding, - )) || 0; - - signal?.throwIfAborted(); - - if (bytesWritten === 0) { - if (++retries > 5) { - throw new ERR_OPERATION_FAILED('write failed after retries'); - } - } else { - retries = 0; - } + let retries = 0; + while (length > 0) { + const bytesWritten = (await PromisePrototypeThen( + binding.writeBuffer(fd, buf, offset, length, position, + kUsePromises), + undefined, + handleErrorFromBinding, + )) || 0; + + signal?.throwIfAborted(); - totalBytesWritten += bytesWritten; - offset += bytesWritten; - length -= bytesWritten; - if (position >= 0) position += bytesWritten; + if (bytesWritten === 0) { + if (++retries > 5) { + throw new ERR_OPERATION_FAILED('write failed after retries'); + } + } else { + retries = 0; } - } finally { - asyncPending = false; + + totalBytesWritten += bytesWritten; + offset += bytesWritten; + length -= bytesWritten; + if (position >= 0) position += bytesWritten; } } // Writev with EAGAIN retry. On partial write, concatenates remaining // buffers and falls back to writeAll (same approach as WriteStream). async function writevAll(buffers, position, signal) { - asyncPending = true; - try { - let totalSize = 0; - for (let i = 0; i < buffers.length; i++) { - totalSize += buffers[i].byteLength; - } + let totalSize = 0; + for (let i = 0; i < buffers.length; i++) { + totalSize += buffers[i].byteLength; + } - let retries = 0; - while (totalSize > 0) { - const bytesWritten = (await PromisePrototypeThen( - binding.writeBuffers(fd, buffers, position, kUsePromises), - undefined, - handleErrorFromBinding, - )) || 0; + let retries = 0; + while (totalSize > 0) { + const bytesWritten = (await PromisePrototypeThen( + binding.writeBuffers(fd, buffers, position, kUsePromises), + undefined, + handleErrorFromBinding, + )) || 0; - signal?.throwIfAborted(); + signal?.throwIfAborted(); - if (bytesWritten === 0) { - if (++retries > 5) { - throw new ERR_OPERATION_FAILED('writev failed after retries'); - } - } else { - retries = 0; + if (bytesWritten === 0) { + if (++retries > 5) { + throw new ERR_OPERATION_FAILED('writev failed after retries'); } + } else { + retries = 0; + } - totalBytesWritten += bytesWritten; - totalSize -= bytesWritten; - if (position >= 0) position += bytesWritten; - - if (totalSize > 0) { - // Partial write - concatenate remaining and use writeAll. - const remaining = Buffer.concat(buffers); - const wrote = bytesWritten; - // writeAll is already inside asyncPending = true, but - // writeAll sets it again - that's fine (idempotent). - await writeAll(remaining, wrote, remaining.length - wrote, - position, signal); - return; - } + totalBytesWritten += bytesWritten; + totalSize -= bytesWritten; + if (position >= 0) position += bytesWritten; + + if (totalSize > 0) { + // Partial write - concatenate remaining and use writeAll. + const remaining = Buffer.concat(buffers); + const wrote = bytesWritten; + await writeAll(remaining, wrote, remaining.length - wrote, + position, signal); + return; } - } finally { - asyncPending = false; } } + function releaseAfterFailure() { + if (!errored || asyncPending !== 0 || released) return; + released = true; + handle[kLocked] = false; + handle[kUnref](); + if (autoClose) { + handle[kCloseSync](); + } + } + + function finishEnd() { + if (!closing || errored || asyncPending !== 0 || endingCleanup) return; + endingCleanup = true; + PromisePrototypeThen(cleanup(), () => { + endingCleanup = false; + if (errored) return; + closing = false; + closed = true; + pendingEnd.resolve(totalBytesWritten); + pendingEnd = null; + }, (error) => { + endingCleanup = false; + if (errored) return; + closing = false; + closed = true; + pendingEnd.reject(error); + pendingEnd = null; + }); + } + + function trackOperation(operation) { + const { promise, resolve, reject } = PromiseWithResolvers(); + const pending = { __proto__: null, reject }; + pendingWrites.add(pending); + asyncPending++; + PromisePrototypeThen(operation, (value) => { + const active = pendingWrites.delete(pending); + asyncPending--; + releaseAfterFailure(); + if (active) resolve(value); + finishEnd(); + }, (error) => { + const active = pendingWrites.delete(pending); + asyncPending--; + releaseAfterFailure(); + if (active) reject(error); + finishEnd(); + }); + return promise; + } + // Synchronous write with EAGAIN retry. Throws on I/O error. // Used by writeSync for the full write, and by writevSync for // completing a partial writev. @@ -872,8 +914,8 @@ if (getOptionValue('--experimental-stream-iter')) { } async function cleanup() { - if (closed) return; - closed = true; + if (released) return; + released = true; handle[kLocked] = false; handle[kUnref](); if (autoClose) { @@ -886,13 +928,17 @@ if (getOptionValue('--experimental-stream-iter')) { write(chunk, options = kNullPrototo) { chunk = newStreamsToWriterUint8Array(chunk); const signal = newStreamsGetWriterSignal(options); - if (error) { + if (errored) { return PromiseReject(error); } if (closed) { return PromiseReject( new ERR_INVALID_STATE.TypeError('The writer is closed')); } + if (closing) { + return PromiseReject( + new ERR_INVALID_STATE.TypeError('The writer is closing')); + } if (signal?.aborted) { return PromiseReject(signal.reason); } @@ -904,19 +950,24 @@ if (getOptionValue('--experimental-stream-iter')) { if (bytesRemaining > 0) bytesRemaining -= chunk.byteLength; const position = pos; if (pos >= 0) pos += chunk.byteLength; - return writeAll(chunk, 0, chunk.byteLength, position, signal); + return trackOperation( + writeAll(chunk, 0, chunk.byteLength, position, signal)); }, writev(chunks, options = kNullPrototo) { chunks = newStreamsConvertChunks(chunks); const signal = newStreamsGetWriterSignal(options); - if (error) { + if (errored) { return PromiseReject(error); } if (closed) { return PromiseReject( new ERR_INVALID_STATE.TypeError('The writer is closed')); } + if (closing) { + return PromiseReject( + new ERR_INVALID_STATE.TypeError('The writer is closing')); + } if (signal?.aborted) { return PromiseReject(signal.reason); } @@ -932,12 +983,12 @@ if (getOptionValue('--experimental-stream-iter')) { if (bytesRemaining > 0) bytesRemaining -= totalSize; const position = pos; if (pos >= 0) pos += totalSize; - return writevAll(chunks, position, signal); + return trackOperation(writevAll(chunks, position, signal)); }, writeSync(chunk) { chunk = newStreamsToWriterUint8Array(chunk); - if (error || closed || asyncPending) return false; + if (errored || closed || closing || asyncPending) return false; const length = chunk.byteLength; if (length > syncWriteThreshold) return false; if (length === 0) return true; @@ -968,7 +1019,7 @@ if (getOptionValue('--experimental-stream-iter')) { writevSync(chunks) { chunks = newStreamsConvertChunks(chunks); - if (error || closed || asyncPending) return false; + if (errored || closed || closing || asyncPending) return false; let totalSize = 0; for (let i = 0; i < chunks.length; i++) { totalSize += chunks[i].byteLength; @@ -1004,29 +1055,31 @@ if (getOptionValue('--experimental-stream-iter')) { end(options = kNullPrototo) { const signal = newStreamsGetWriterSignal(options); - if (error) { + if (errored) { return PromiseReject(error); } if (closed) { return PromiseResolve(totalBytesWritten); } if (closing) { - return pendingEndPromise; + return pendingEnd.promise; } if (signal?.aborted) { return PromiseReject(signal.reason); } closing = true; - pendingEndPromise = PromisePrototypeThen( - cleanup(), () => totalBytesWritten); - return pendingEndPromise; + pendingEnd = PromiseWithResolvers(); + finishEnd(); + return pendingEnd.promise; }, endSync() { - if (error) return -1; + if (errored) return -1; if (closed) return totalBytesWritten; + if (closing) return -1; if (asyncPending) return -1; closed = true; + released = true; handle[kLocked] = false; handle[kUnref](); if (autoClose) { @@ -1036,21 +1089,25 @@ if (getOptionValue('--experimental-stream-iter')) { }, fail(reason) { - if (closed || error) return; - error = reason ?? new ERR_INVALID_STATE('Failed'); + if (closed || errored) return; + errored = true; + error = reason; + closing = false; closed = true; - handle[kLocked] = false; - handle[kUnref](); - if (autoClose) { - handle[kCloseSync](); + pendingEnd?.reject(reason); + pendingEnd = null; + for (const pending of pendingWrites) { + pending.reject(reason); } + pendingWrites.clear(); + releaseAfterFailure(); }, [SymbolAsyncDispose]() { if (closing) { - return pendingEndPromise ?? PromiseResolve(); + return pendingEnd?.promise ?? PromiseResolve(); } - if (!closed && !error) { + if (!closed && !errored) { this.fail(); } return PromiseResolve(); diff --git a/lib/internal/quic/quic.js b/lib/internal/quic/quic.js index b6bb31aa97b4..95288775d42a 100644 --- a/lib/internal/quic/quic.js +++ b/lib/internal/quic/quic.js @@ -2250,11 +2250,11 @@ class QuicStream { } async function writeAsync(chunk, signal) { - signal?.throwIfAborted(); if (errored) throw error; if (closed || stream.#inner.state.writeEnded) { throw new ERR_INVALID_STATE('Writer is closed'); } + signal?.throwIfAborted(); // If a drain is already pending, another operation is waiting // for capacity. Under strict policy, reject immediately. // Later, if we add support for other backpressure policies, @@ -2293,12 +2293,11 @@ class QuicStream { } async function writevAsync(chunks, signal) { - signal?.throwIfAborted(); - if (errored) throw error; if (closed || stream.#inner.state.writeEnded) { throw new ERR_INVALID_STATE('Writer is closed'); } + signal?.throwIfAborted(); // If a drain is already pending, another operation is waiting // for capacity. Under strict policy, reject immediately. @@ -2339,6 +2338,8 @@ class QuicStream { } async function endAsync(signal) { + if (errored) throw error; + if (closed) return totalBytesWritten; if (signal !== undefined) { signal.throwIfAborted(); // TODO(@jasnell): The stream/iter spec allows individual sync end @@ -2370,13 +2371,16 @@ class QuicStream { } finally { drainWakeup = null; } - return endSync(); + if (errored) throw error; + const result = endSync(); + if (errored) throw error; + return result; } function fail(reason) { if (closed || errored) return; errored = true; - error = reason ?? new ERR_INVALID_STATE('Failed'); + error = reason; // `writer.fail()` is always an error path, so the wire code on // RESET_STREAM must never be `0n` (which means "no error" in // most application protocols). Resolve the code in priority diff --git a/lib/internal/streams/iter/broadcast.js b/lib/internal/streams/iter/broadcast.js index 16cb4c06533c..ddce0d12d42c 100644 --- a/lib/internal/streams/iter/broadcast.js +++ b/lib/internal/streams/iter/broadcast.js @@ -61,7 +61,6 @@ const { hasProtocol, onSignalAbort, parsePullArgs, - wrapError, toWriterUint8Array, validateBatchEntry, } = require('internal/streams/iter/utils'); @@ -81,6 +80,7 @@ const kCanWrite = Symbol('kCanWrite'); const kOnBufferDrained = Symbol('kOnBufferDrained'); const kOnEndDrained = Symbol('kOnEndDrained'); const kPendingWriteRemoved = Symbol('kPendingWriteRemoved'); +const kNoBroadcastError = Symbol('kNoBroadcastError'); function raceEndWithSignal(promise, signal) { if (!signal) return promise; @@ -107,6 +107,7 @@ class BroadcastImpl { #waiters = []; // Consumers with pending resolve (subset of #consumers) #ended = false; #error; + #errored = false; #cancelled = false; #options; #writer = null; @@ -176,6 +177,7 @@ class BroadcastImpl { reject: null, pending: [], detached: false, + error: kNoBroadcastError, }; this.#consumers.add(state); @@ -211,8 +213,8 @@ class BroadcastImpl { __proto__: null, next() { if (state.detached) { - if (self.#error !== undefined) { - return PromiseReject(self.#error); + if (state.error !== kNoBroadcastError) { + return PromiseReject(state.error); } return kDone; } @@ -231,8 +233,9 @@ class BroadcastImpl { { __proto__: null, done: false, value: chunk }); } - if (self.#error !== undefined) { + if (self.#errored) { state.detached = true; + state.error = self.#error; self.#deleteConsumer(state); return PromiseReject(self.#error); } @@ -272,11 +275,13 @@ class BroadcastImpl { cancel(reason) { if (this.#cancelled) return; + const hasReason = arguments.length > 0; this.#cancelled = true; this.#ended = true; // Prevents [kAbort]() from redundantly iterating consumers - if (reason !== undefined) { + if (hasReason) { this.#error = reason; + this.#errored = true; } // Reject pending writes on the writer so the pump doesn't hang @@ -284,7 +289,7 @@ class BroadcastImpl { for (const consumer of this.#consumers) { if (consumer.resolve) { - if (reason !== undefined) { + if (hasReason) { consumer.reject?.(reason); } else { consumer.resolve({ __proto__: null, done: true, value: undefined }); @@ -292,7 +297,8 @@ class BroadcastImpl { consumer.resolve = null; consumer.reject = null; } - if (reason !== undefined) { + if (hasReason) { + consumer.error = reason; this.#rejectPending(consumer, reason); } else { this.#resolvePendingDone(consumer); @@ -389,7 +395,8 @@ class BroadcastImpl { } [kAbort](reason) { - if (this.#error !== undefined) return; + if (this.#errored) return; + this.#errored = true; this.#error = reason; this.#ended = true; @@ -401,6 +408,7 @@ class BroadcastImpl { consumer.reject = null; } this.#rejectPending(consumer, reason); + consumer.error = reason; consumer.detached = true; } this.#consumers.clear(); @@ -460,7 +468,7 @@ class BroadcastImpl { return validateBatchEntry(entry); } catch (error) { this.#writer.fail(error); - if (this.#error === undefined) this[kAbort](error); + if (!this.#errored) this[kAbort](error); this.#buffer.clear(); this.#bufferedBytes = 0; return null; @@ -706,12 +714,11 @@ class BroadcastWriter { fail(reason) { if (this.#state === 'errored' || this.#state === 'closed') return; this.#state = 'errored'; - const error = reason ?? new ERR_INVALID_STATE.TypeError('Failed'); - this.#error = error; - this.#rejectPendingWrites(error); - this.#rejectPendingDrains(error); - this.#pendingEnd?.reject(error); - this.#broadcast[kAbort](error); + this.#error = reason; + this.#rejectPendingWrites(reason); + this.#rejectPendingDrains(reason); + this.#pendingEnd?.reject(reason); + this.#broadcast[kAbort](reason); } [SymbolAsyncDispose]() { @@ -818,7 +825,7 @@ function wireBroadcastWriteSignal(entry, signal, resolve, reject, self) { const idx = pendingWrites.indexOf(entry); if (idx !== -1) pendingWrites.removeAt(idx); entry.batch = null; - reject(signal.reason ?? lazyDOMException('Aborted', 'AbortError')); + reject(signal.reason); if (idx !== -1) self[kPendingWriteRemoved](); }; entry.resolve = function() { @@ -937,7 +944,7 @@ const Broadcast = { await w.end(signal ? { signal } : undefined); } } catch (error) { - w.fail(wrapError(error)); + w.fail(error); } }; PromisePrototypeThen(pump(), undefined, () => {}); diff --git a/lib/internal/streams/iter/classic.js b/lib/internal/streams/iter/classic.js index 28f6079e6fe2..4ca384cae9ab 100644 --- a/lib/internal/streams/iter/classic.js +++ b/lib/internal/streams/iter/classic.js @@ -13,6 +13,7 @@ const { ArrayPrototypePush, + FunctionPrototypeCall, NumberMAX_SAFE_INTEGER, Promise, PromisePrototypeThen, @@ -32,9 +33,11 @@ const { AbortError, aggregateTwoErrors, codes: { + ERR_FALSY_VALUE_REJECTION, ERR_INVALID_ARG_TYPE, ERR_INVALID_ARG_VALUE, ERR_INVALID_STATE, + ERR_OPERATION_FAILED, ERR_STREAM_WRITE_AFTER_END, }, } = require('internal/errors'); @@ -67,6 +70,25 @@ const { const { Buffer } = require('buffer'); const destroyImpl = require('internal/streams/destroy'); +const { isError } = require('internal/util'); + +// Classic stream error channels require a truthy Error object. +function toClassicError(reason, reasonMap) { + try { + if (isError(reason)) return reason; + } catch { + // Wrap values whose proxy traps make the Error check fail. + } + let error; + if (!reason) { + error = new ERR_FALSY_VALUE_REJECTION.HideStackFramesError(reason); + } else { + error = new ERR_OPERATION_FAILED('Non-Error value'); + error.reason = reason; + } + reasonMap?.set(error, { __proto__: null, reason }); + return error; +} // Lazy-loaded to avoid circular dependencies. Readable and Writable // both require this module's parent, so we defer the require. @@ -299,11 +321,17 @@ function toReadable(source, options = kNullPrototype) { backpressure.resolve(); backpressure = null; } - if (typeof iterator.return === 'function') { - PromisePrototypeThen(iterator.return(), - () => cb(err), (e) => cb(e || err)); - } else { - cb(err); + try { + const returnMethod = iterator.return; + if (typeof returnMethod !== 'function') { + cb(err); + return; + } + const returned = FunctionPrototypeCall(returnMethod, iterator); + PromisePrototypeThen(PromiseResolve(returned), () => cb(err), + (error) => cb(err || toClassicError(error))); + } catch (error) { + cb(err || toClassicError(error)); } }, }); @@ -331,7 +359,7 @@ function toReadable(source, options = kNullPrototype) { } } catch (err) { done = true; - readable.destroy(err); + readable.destroy(toClassicError(err)); } } @@ -372,30 +400,43 @@ function toReadableSync(source, options = kNullPrototype) { __proto__: null, highWaterMark, read() { - for (;;) { - if (hasBatch) { - while (batchIndex < batch.length) { - if (!this.push(batch[batchIndex++])) return; + try { + for (;;) { + if (hasBatch) { + while (batchIndex < batch.length) { + if (!this.push(batch[batchIndex++])) return; + } + batch = undefined; + hasBatch = false; + batchIndex = 0; } - batch = undefined; - hasBatch = false; - batchIndex = 0; - } - const result = iterator.next(); - const { done } = result; - if (done) { - this.push(null); - return; + const result = iterator.next(); + const { done } = result; + if (done) { + this.push(null); + return; + } + batch = result.value; + hasBatch = true; } - batch = result.value; - hasBatch = true; + } catch (error) { + const classicError = toClassicError(error); + throw classicError; } }, destroy(err, cb) { batch = undefined; hasBatch = false; - if (typeof iterator.return === 'function') iterator.return(); + try { + const returnMethod = iterator.return; + if (typeof returnMethod === 'function') { + FunctionPrototypeCall(returnMethod, iterator); + } + } catch (error) { + cb(err || toClassicError(error)); + return; + } cb(err); }, }); @@ -471,6 +512,9 @@ function fromWritable(writable, options = kNullPrototype) { // expose the full stream.Writable property set. const hwm = writable.writableHighWaterMark ?? 16384; let totalBytes = 0; + let errored = false; + let error; + let pendingEnd; // Waiters pending on backpressure resolution (block policy only). // Multiple un-awaited writes can each add a waiter, so this must be @@ -504,11 +548,17 @@ function fromWritable(writable, options = kNullPrototype) { } // Reject all pending waiters and remove the drain/error listeners. - function cleanup(err) { + function cleanup(err, preserveReason = false) { const pending = waiters; waiters = []; for (let i = 0; i < pending.length; i++) { - pending[i].reject(err ?? new AbortError()); + if (!preserveReason && + (err === undefined || err === null) && + pending[i].close !== undefined) { + pending[i].close(); + } else { + pending[i].reject(preserveReason ? err : err ?? new AbortError()); + } } if (!listenersInstalled) return; listenersInstalled = false; @@ -526,7 +576,8 @@ function fromWritable(writable, options = kNullPrototype) { function isWritable() { // Duck-typed streams may not have these properties -- treat missing // as false (i.e., writable is still open). - return !(writable.destroyed ?? false) && + return !errored && + !(writable.destroyed ?? false) && !(writable.writableFinished ?? false) && !(writable.writableEnded ?? false); } @@ -580,6 +631,7 @@ function fromWritable(writable, options = kNullPrototype) { write(chunk, options) { const bytes = toWriterUint8Array(chunk); getWriterSignal(options); + if (errored) return PromiseReject(error); if (!isWritable()) { return PromiseReject(new ERR_STREAM_WRITE_AFTER_END()); } @@ -617,6 +669,7 @@ function fromWritable(writable, options = kNullPrototype) { writev(chunks, options) { chunks = convertChunks(chunks); getWriterSignal(options); + if (errored) return PromiseReject(error); if (!isWritable()) { return PromiseReject(new ERR_STREAM_WRITE_AFTER_END()); } @@ -665,51 +718,71 @@ function fromWritable(writable, options = kNullPrototype) { // write(). end(options) { getWriterSignal(options); + if (errored) return PromiseReject(error); + if (pendingEnd) return pendingEnd.promise; if ((writable.writableFinished ?? false) || (writable.destroyed ?? false)) { cleanup(); return PromiseResolve(totalBytes); } - const { promise, resolve, reject } = PromiseWithResolvers(); + pendingEnd = PromiseWithResolvers(); + const { promise, resolve, reject } = pendingEnd; - if (!(writable.writableEnded ?? false)) { - writable.end(); - } + try { + if (!(writable.writableEnded ?? false)) { + writable.end(); + } - eos(writable, { writable: true, readable: false }, (err) => { - cleanup(err); - if (err) reject(err); - else resolve(totalBytes); - }); + eos(writable, { writable: true, readable: false }, (err) => { + if (errored) return; + pendingEnd = undefined; + cleanup(err); + if (err) reject(err); + else resolve(totalBytes); + }); + } catch (reason) { + pendingEnd = undefined; + errored = true; + error = reason; + cleanup(reason, true); + reject(reason); + try { + writable.destroy?.(toClassicError(reason)); + } catch { + // Preserve the original terminal reason. + } + } return promise; }, fail(reason) { - cleanup(reason); + if (errored || + (writable.writableFinished ?? false) || + (writable.destroyed ?? false)) { + return; + } + errored = true; + error = reason; + pendingEnd?.reject(reason); + pendingEnd = undefined; + cleanup(reason, true); if (typeof writable.destroy === 'function') { - writable.destroy(reason); + writable.destroy(toClassicError(reason)); } }, [SymbolAsyncDispose]() { + if (pendingEnd) return pendingEnd.promise; if (isWritable()) { - cleanup(); - if (typeof writable.destroy === 'function') { - writable.destroy(); - } + this.fail(); } return PromiseResolve(); }, [SymbolDispose]() { - if (isWritable()) { - cleanup(); - if (typeof writable.destroy === 'function') { - writable.destroy(); - } - } + this.fail(); }, }; @@ -719,11 +792,12 @@ function fromWritable(writable, options = kNullPrototype) { if ((writable.writableLength ?? 0) < hwm) { return PromiseResolve(true); } - const { promise, resolve } = PromiseWithResolvers(); + const { promise, resolve, reject } = PromiseWithResolvers(); ArrayPrototypePush(waiters, { __proto__: null, resolve() { resolve(true); }, - reject() { resolve(false); }, + reject, + close() { resolve(false); }, }); installListeners(); return promise; @@ -761,6 +835,7 @@ function toWritable(writer) { const hasEndSync = hasEnd && typeof writer.endSync === 'function'; const hasFail = typeof writer.fail === 'function'; + const classicErrorReasons = new SafeWeakMap(); // Try-sync-first pattern: attempt the synchronous method and fall back to the // async method if it returns false (data not accepted synchronously). // When the sync path succeeds, the callback is deferred via queueMicrotask @@ -778,14 +853,16 @@ function toWritable(writer) { } // WriteSync returned false: not accepted, fall through to async. } catch (err) { - cb(err); + cb(toClassicError(err, classicErrorReasons)); return; } } try { - PromisePrototypeThen(writer.write(bytes), () => cb(), cb); + PromisePrototypeThen( + writer.write(bytes), () => cb(), + (err) => cb(toClassicError(err, classicErrorReasons))); } catch (err) { - cb(err); + cb(toClassicError(err, classicErrorReasons)); } } @@ -804,14 +881,16 @@ function toWritable(writer) { } // WritevSync returned false: not accepted, fall through to async. } catch (err) { - cb(err); + cb(toClassicError(err, classicErrorReasons)); return; } } try { - PromisePrototypeThen(writer.writev(chunks), () => cb(), cb); + PromisePrototypeThen( + writer.writev(chunks), () => cb(), + (err) => cb(toClassicError(err, classicErrorReasons))); } catch (err) { - cb(err); + cb(toClassicError(err, classicErrorReasons)); } } @@ -829,22 +908,31 @@ function toWritable(writer) { } // Result < 0: can't end synchronously, fall through to async. } catch (err) { - cb(err); + cb(toClassicError(err, classicErrorReasons)); return; } } try { - PromisePrototypeThen(writer.end(), () => cb(), cb); + PromisePrototypeThen( + writer.end(), () => cb(), + (err) => cb(toClassicError(err, classicErrorReasons))); } catch (err) { - cb(err); + cb(toClassicError(err, classicErrorReasons)); } } function _destroy(err, cb) { if (err && hasFail) { - writer.fail(err); + const wrapped = classicErrorReasons.get(err); + classicErrorReasons.delete(err); + try { + writer.fail(wrapped === undefined ? err : wrapped.reason); + } catch (error) { + cb(err || toClassicError(error, classicErrorReasons)); + return; + } } - cb(); + cb(err); } const writableOptions = { diff --git a/lib/internal/streams/iter/pull.js b/lib/internal/streams/iter/pull.js index 6b44f5c5e431..01a9504dc871 100644 --- a/lib/internal/streams/iter/pull.js +++ b/lib/internal/streams/iter/pull.js @@ -35,6 +35,7 @@ const { const { AbortController, AbortSignal, + abortSignal, } = require('internal/abort_controller'); const { @@ -54,7 +55,6 @@ const { toUint8Array, validateBatchEntry, validateByteView, - wrapError, yieldAbortable, } = require('internal/streams/iter/utils'); const { @@ -723,8 +723,7 @@ async function* createAsyncPipeline(source, transforms, signal) { let abortHandler; if (signal) { abortHandler = () => { - controller.abort(signal.reason ?? - lazyDOMException('Aborted', 'AbortError')); + abortSignal(controller.signal, signal.reason); }; signal.addEventListener('abort', abortHandler, { __proto__: null, once: true }); } @@ -785,7 +784,7 @@ async function* createAsyncPipeline(source, transforms, signal) { completed = true; } catch (error) { if (!controller.signal.aborted) { - controller.abort(wrapError(error)); + abortSignal(controller.signal, error); } throw error; } finally { @@ -868,7 +867,7 @@ function pull(source, ...args) { return iterator.return(value); }, throw(error) { - controller.abort(error); + abortSignal(controller.signal, error); return iterator.throw(error); }, [SymbolAsyncIterator]() { @@ -1028,7 +1027,7 @@ function pipeToSync(source, ...args) { } } catch (error) { if (!options.preventFail) { - writer.fail?.(wrapError(error)); + writer.fail?.(error); } throw error; } @@ -1053,7 +1052,7 @@ async function pipeTo(source, ...args) { function failWriter(error) { if (!options.preventFail) { - writer.fail?.(wrapError(error)); + writer.fail?.(error); } } diff --git a/lib/internal/streams/iter/push.js b/lib/internal/streams/iter/push.js index 0536d8212c2a..31b437c85b75 100644 --- a/lib/internal/streams/iter/push.js +++ b/lib/internal/streams/iter/push.js @@ -12,7 +12,6 @@ const { PromiseResolve, PromiseWithResolvers, SafeWeakSet, - Symbol, SymbolAsyncDispose, SymbolAsyncIterator, SymbolDispose, @@ -23,7 +22,6 @@ const { ERR_INVALID_STATE, }, } = require('internal/errors'); -const { lazyDOMException } = require('internal/util'); const { validateInteger, } = require('internal/validators'); @@ -55,7 +53,6 @@ const { RingBuffer, } = require('internal/streams/iter/ringbuffer'); -const kNoFailReason = Symbol('kNoFailReason'); const consumerReturnErrors = new SafeWeakSet(); function isConsumerReturnError(error) { @@ -108,8 +105,10 @@ class PushQueue { #writerState = 'open'; /** Consumer state: 'active' | 'returned' | 'thrown' */ #consumerState = 'active'; - /** Error that closed the stream */ - #error = null; + /** Error that closed the writer */ + #writerError; + /** Error supplied by the consumer */ + #consumerError; /** Total bytes written */ #bytesWritten = 0; /** Pending end promise (resolves when consumer drains past end sentinel) */ @@ -255,12 +254,11 @@ class PushQueue { throw new ERR_INVALID_STATE.TypeError('Writer is closing'); } if (this.#writerState === 'errored') { - throw this.#error; + throw this.#writerError; } if (this.#consumerState !== 'active') { - throw this.#consumerState === 'thrown' && this.#error ? - this.#error : - new ERR_INVALID_STATE.TypeError('Stream closed by consumer'); + if (this.#consumerState === 'thrown') throw this.#consumerError; + throw new ERR_INVALID_STATE.TypeError('Stream closed by consumer'); } // Check for pre-aborted signal (after state checks per spec) @@ -307,7 +305,7 @@ class PushQueue { this.#pendingWrites.removeAt(idx); this.#resolvePendingReads(); } - reject(signal.reason ?? lazyDOMException('Aborted', 'AbortError')); + reject(signal.reason); }; // Wrap resolve/reject to clean up signal listener @@ -379,25 +377,23 @@ class PushQueue { * No-op if errored or closed (fully drained). * If closing (draining), short-circuits the drain. */ - fail(reason = kNoFailReason) { + fail(reason) { if (this.#writerState === 'errored' || this.#writerState === 'closed') { return; } const wasClosing = this.#writerState === 'closing'; this.#writerState = 'errored'; - this.#error = reason === kNoFailReason ? - new ERR_INVALID_STATE('Failed') : - reason; + this.#writerError = reason; this.#cleanup(); - this.#rejectPendingReads(this.#error); - this.#rejectPendingDrains(this.#error); - this.#rejectPendingWrites(this.#error); + this.#rejectPendingReads(this.#writerError); + this.#rejectPendingDrains(this.#writerError); + this.#rejectPendingWrites(this.#writerError); if (wasClosing) { // Short-circuit the graceful drain: reject the pending end promise if (this.#pendingEnd) { - this.#pendingEnd.reject(this.#error); + this.#pendingEnd.reject(this.#writerError); this.#pendingEnd = null; } } @@ -408,7 +404,7 @@ class PushQueue { } get error() { - return this.#error; + return this.#writerError; } get backpressurePolicy() { @@ -446,7 +442,7 @@ class PushQueue { return { __proto__: null, done: true, value: undefined }; } if (this.#consumerState === 'thrown') { - throw this.#error; + throw this.#consumerError; } // If there's data in the buffer, return it immediately @@ -467,7 +463,7 @@ class PushQueue { } if (this.#writerState === 'errored') { - throw this.#error; + throw this.#writerError; } const { promise, resolve, reject } = PromiseWithResolvers(); @@ -489,7 +485,7 @@ class PushQueue { consumerThrow(error) { if (this.#consumerState !== 'active') return; this.#consumerState = 'thrown'; - this.#error = error; + this.#consumerError = error; this.#terminateWriterFromConsumer(error); this.#rejectPendingReads(error); // Reject pending drains - the consumer errored @@ -531,7 +527,7 @@ class PushQueue { this.#bufferedBytes = 0; if (this.#writerState === 'open' || this.#writerState === 'closing') { this.#writerState = 'errored'; - this.#error = error; + this.#writerError = error; } this.#cleanup(); this.#rejectPendingWrites(error); @@ -548,7 +544,7 @@ class PushQueue { pending.resolve({ __proto__: null, done: true, value: undefined }); } else if (this.#consumerState === 'thrown') { const pending = this.#pendingReads.shift(); - pending.reject(this.#error); + pending.reject(this.#consumerError); } else if (this.#slots.length > 0) { const pending = this.#pendingReads.shift(); try { @@ -569,7 +565,7 @@ class PushQueue { pending.resolve({ __proto__: null, done: true, value: undefined }); } else if (this.#writerState === 'errored') { const pending = this.#pendingReads.shift(); - pending.reject(this.#error); + pending.reject(this.#writerError); } else { break; } @@ -685,6 +681,11 @@ class PushWriter { end(options) { const signal = getWriterSignal(options); + const state = this.#queue.writerState; + if (state === 'errored') return PromiseReject(this.#queue.error); + if (state === 'closed') { + return PromiseResolve(this.#queue.totalBytesWritten); + } if (signal?.aborted) return PromiseReject(signal.reason); const result = this.#queue.end(); @@ -707,7 +708,7 @@ class PushWriter { } fail(reason) { - this.#queue.fail(arguments.length === 0 ? kNoFailReason : reason); + this.#queue.fail(reason); } [SymbolAsyncDispose]() { diff --git a/lib/internal/streams/iter/share.js b/lib/internal/streams/iter/share.js index 70179dffd3c1..6154509a5b64 100644 --- a/lib/internal/streams/iter/share.js +++ b/lib/internal/streams/iter/share.js @@ -7,6 +7,7 @@ const { ArrayPrototypePush, + FunctionPrototypeCall, PromisePrototypeThen, PromiseResolve, PromiseWithResolvers, @@ -41,7 +42,6 @@ const { getMinCursor, hasProtocol, onSignalAbort, - wrapError, parsePullArgs, validateBatchEntry, } = require('internal/streams/iter/utils'); @@ -79,7 +79,7 @@ class ShareImpl { #consumers = new SafeSet(); #sourceIterator = null; #sourceExhausted = false; - #sourceError; + #sourceError = kNoShareError; #cancelled = false; #pulling = false; #pullWaiters = []; @@ -192,7 +192,7 @@ class ShareImpl { if (self.#sourceExhausted) { state.detached = true; self.#deleteConsumer(state); - if (self.#sourceError !== undefined) { + if (self.#sourceError !== kNoShareError) { state.error = self.#sourceError; throw state.error; } @@ -254,28 +254,33 @@ class ShareImpl { cancel(reason) { if (this.#cancelled) return; + const hasReason = arguments.length > 0; this.#cancelled = true; - if (reason !== undefined) { + if (hasReason) { this.#cancelError = reason; } this.#resolveCancel(kShareCancelled); this.#resolveCancel = null; - if (this.#sourceIterator?.return) { - try { + try { + const returnMethod = this.#sourceIterator?.return; + if (typeof returnMethod === 'function') { PromisePrototypeThen( - PromiseResolve(this.#sourceIterator.return()), undefined, () => {}); - } catch { - // Cancellation has precedence over source cleanup errors. + PromiseResolve(FunctionPrototypeCall( + returnMethod, this.#sourceIterator)), + undefined, + () => {}); } + } catch { + // Cancellation has precedence over source cleanup errors. } for (const consumer of this.#consumers) { consumer.error = this.#cancelError; if (consumer.resolve) { - if (reason !== undefined) { + if (hasReason) { consumer.reject?.(reason); } else { consumer.resolve({ __proto__: null, done: true, value: undefined }); @@ -304,7 +309,7 @@ class ShareImpl { async #waitForBufferSpace() { while (this.#bufferedBytes >= this.#options.budget) { if (this.#cancelled || - this.#sourceError !== undefined || + this.#sourceError !== kNoShareError || this.#sourceExhausted) { return this.#cancelled ? null : true; } @@ -345,7 +350,7 @@ class ShareImpl { async #waitForBufferSpaceAfterDrop() { while (this.#bufferedBytes >= this.#options.budget && !this.#cancelled && - this.#sourceError === undefined && + this.#sourceError === kNoShareError && !this.#sourceExhausted) { const { promise, resolve } = PromiseWithResolvers(); ArrayPrototypePush(this.#pullWaiters, resolve); @@ -406,7 +411,7 @@ class ShareImpl { this.#bufferedBytes += entry.byteLength; } } catch (error) { - this.#sourceError = wrapError(error); + this.#sourceError = error; this.#sourceExhausted = true; } finally { this.#pulling = false; @@ -483,7 +488,7 @@ class SyncShareImpl { #consumers = new SafeSet(); #sourceIterator = null; #sourceExhausted = false; - #sourceError; + #sourceError = kNoShareError; #cancelled = false; #cachedMinCursor = 0; #cachedMinCursorConsumers = 0; @@ -513,6 +518,7 @@ class SyncShareImpl { __proto__: null, cursor: this.#bufferStart, detached: false, + error: kNoShareError, }; this.#consumers.add(state); @@ -532,14 +538,16 @@ class SyncShareImpl { return { __proto__: null, next() { - if (self.#sourceError !== undefined) { - state.detached = true; - self.#deleteConsumer(state); - throw self.#sourceError; - } if (state.detached) { + if (state.error !== kNoShareError) throw state.error; return { __proto__: null, done: true, value: undefined }; } + if (self.#sourceError !== kNoShareError) { + state.detached = true; + state.error = self.#sourceError; + self.#deleteConsumer(state); + throw state.error; + } if (self.#cancelled) { state.detached = true; self.#deleteConsumer(state); @@ -600,10 +608,11 @@ class SyncShareImpl { self.#pullFromSource(); - if (self.#sourceError !== undefined) { + if (self.#sourceError !== kNoShareError) { state.detached = true; + state.error = self.#sourceError; self.#deleteConsumer(state); - throw self.#sourceError; + throw state.error; } const newBufferIndex = state.cursor - self.#bufferStart; @@ -649,17 +658,24 @@ class SyncShareImpl { cancel(reason) { if (this.#cancelled) return; + const hasReason = arguments.length > 0; this.#cancelled = true; - if (reason !== undefined) { + if (hasReason) { this.#sourceError = reason; } - if (this.#sourceIterator?.return) { - this.#sourceIterator.return(); + try { + const returnMethod = this.#sourceIterator?.return; + if (typeof returnMethod === 'function') { + FunctionPrototypeCall(returnMethod, this.#sourceIterator); + } + } catch { + // Cancellation has precedence over source cleanup errors. } for (const consumer of this.#consumers) { + if (hasReason) consumer.error = reason; consumer.detached = true; } this.#consumers.clear(); @@ -685,7 +701,7 @@ class SyncShareImpl { this.#bufferedBytes += entry.byteLength; } } catch (error) { - this.#sourceError = wrapError(error); + this.#sourceError = error; this.#sourceExhausted = true; } } diff --git a/lib/internal/streams/iter/transform.js b/lib/internal/streams/iter/transform.js index cb35906ded02..c4f08b8cd32b 100644 --- a/lib/internal/streams/iter/transform.js +++ b/lib/internal/streams/iter/transform.js @@ -35,7 +35,6 @@ const { }, genericNodeError, } = require('internal/errors'); -const { lazyDOMException } = require('internal/util'); const { isArrayBufferView, isAnyArrayBuffer } = require('internal/util/types'); const { kValidatedTransform } = require('internal/streams/iter/types'); const { @@ -399,8 +398,7 @@ function makeZlibTransform(createHandleFn, processFlag, finishFlag) { resolveWrite = undefined; rejectWrite = undefined; if (reject) { - reject(signal.reason ?? - lazyDOMException('The operation was aborted', 'AbortError')); + reject(signal.reason); } }; signal.addEventListener('abort', onAbort, { __proto__: null, once: true }); diff --git a/lib/internal/streams/iter/utils.js b/lib/internal/streams/iter/utils.js index 08337a5536f2..95d2d914eb1a 100644 --- a/lib/internal/streams/iter/utils.js +++ b/lib/internal/streams/iter/utils.js @@ -9,7 +9,6 @@ const { PromiseWithResolvers, SafePromisePrototypeFinally, SafePromiseRace, - String, SymbolAsyncIterator, TypedArrayPrototypeGetBuffer, TypedArrayPrototypeGetByteLength, @@ -27,10 +26,8 @@ const { codes: { ERR_INVALID_ARG_TYPE, ERR_INVALID_STATE, - ERR_OPERATION_FAILED, }, } = require('internal/errors'); -const { isError } = require('internal/util'); const { isSharedArrayBuffer, isUint8Array } = require('internal/util/types'); @@ -337,15 +334,6 @@ function toWriterUint8Array(chunk) { })); } -/** - * Wrap a caught value as an Error, converting non-Error values. - * @param {unknown} error - * @returns {Error} - */ -function wrapError(error) { - return isError(error) ? error : new ERR_OPERATION_FAILED(String(error)); -} - /** * Check if a value implements a Symbol-keyed protocol (has a function * at the given symbol key). @@ -445,6 +433,5 @@ module.exports = { validateBackpressure, validateBatchEntry, validateByteView, - wrapError, yieldAbortable, }; diff --git a/test/parallel/test-fs-promises-file-handle-pull.js b/test/parallel/test-fs-promises-file-handle-pull.js index 3fc531baf713..cdfaf273f933 100644 --- a/test/parallel/test-fs-promises-file-handle-pull.js +++ b/test/parallel/test-fs-promises-file-handle-pull.js @@ -198,7 +198,7 @@ async function testPullAbortSignal() { const ac = new AbortController(); const fh = await open(filePath, 'r'); try { - ac.abort(); + ac.abort(null); const readable = fh.pull({ signal: ac.signal }); await assert.rejects( @@ -208,7 +208,7 @@ async function testPullAbortSignal() { assert.fail('Should not reach here'); } }, - (err) => err.name === 'AbortError', + (err) => err === null, ); } finally { await fh.close(); diff --git a/test/parallel/test-fs-promises-file-handle-writer.js b/test/parallel/test-fs-promises-file-handle-writer.js index d43bababad1e..a636844f8e57 100644 --- a/test/parallel/test-fs-promises-file-handle-writer.js +++ b/test/parallel/test-fs-promises-file-handle-writer.js @@ -764,8 +764,12 @@ async function testEndSyncReturnsFalseDuringAsync() { const p = w.write(Buffer.from('data')); assert.strictEqual(w.endSync(), -1); + const ending = w.end(); + assert.strictEqual(w.writeSync(Buffer.from('more')), false); + assert.strictEqual(w.endSync(), -1); + await assert.rejects(w.write('more'), { code: 'ERR_INVALID_STATE' }); await p; - const totalBytes = await w.end(); + const totalBytes = await ending; await fh.close(); assert.strictEqual(totalBytes, 4); @@ -850,6 +854,50 @@ async function testEndRejectsOnErrored() { await fh.close(); } +async function testFailPreservesReason() { + for (const reason of [undefined, null, false, 0, '', 'failure']) { + const suffix = String(reason).replaceAll(' ', '-'); + const filePath = path.join(tmpDir, `writer-fail-${suffix}.txt`); + const fh = await open(filePath, 'w'); + const w = fh.writer(); + + w.fail(reason); + + await assert.rejects(w.write('data'), (error) => error === reason); + await assert.rejects(w.end(), (error) => error === reason); + await fh.close(); + } +} + +async function testFailRejectsPendingWriteWithReason() { + const filePath = path.join(tmpDir, 'writer-fail-pending.txt'); + const fh = await open(filePath, 'w'); + const w = fh.writer(); + const reason = null; + const pending = w.write(Buffer.alloc(1024 * 1024)); + + w.fail(reason); + + await assert.rejects(pending, (error) => error === reason); + await fh.close(); +} + +async function testFailWhileClosingPreservesReason() { + const filePath = path.join(tmpDir, 'writer-fail-closing.txt'); + const fh = await open(filePath, 'w'); + const w = fh.writer(); + const reason = false; + const pendingWrite = w.write(Buffer.alloc(1024 * 1024)); + const pendingEnd = w.end(); + + w.fail(reason); + + for (const promise of [pendingWrite, pendingEnd]) { + await assert.rejects(promise, (error) => error === reason); + } + await fh.close(); +} + // ============================================================================= // end() is idempotent when closing/closed // ============================================================================= @@ -914,7 +962,7 @@ async function testAsyncDisposeCallsFail() { // Writer should be in errored state - write should reject await assert.rejects( w.write(Buffer.from('more')), - (err) => err instanceof Error, + (reason) => reason === undefined, ); // Handle should be unlocked and reusable @@ -1125,6 +1173,9 @@ Promise.all([ testEndSyncAutoClose(), testFullSyncPipeline(), testEndRejectsOnErrored(), + testFailPreservesReason(), + testFailRejectsPendingWriteWithReason(), + testFailWhileClosingPreservesReason(), testEndIdempotent(), testAsyncDisposeWhileClosing(), testAsyncDisposeCallsFail(), diff --git a/test/parallel/test-quic-stream-writer-api.mjs b/test/parallel/test-quic-stream-writer-api.mjs index 4f19502d8579..6ccd52b046a4 100644 --- a/test/parallel/test-quic-stream-writer-api.mjs +++ b/test/parallel/test-quic-stream-writer-api.mjs @@ -130,8 +130,8 @@ await clientSession.opened; { const stream = await clientSession.createBidirectionalStream(); const w = stream.writer; - const testError = new Error('writer fail test'); - w.fail(testError); + const reason = null; + w.fail(reason); // After fail, canWrite is null. assert.strictEqual(w.canWrite, null); // drainableProtocol returns null when errored. @@ -141,8 +141,12 @@ await clientSession.opened; assert.strictEqual(w.endSync(), -1); // WriteSync after fail returns false. assert.strictEqual(w.writeSync(encoder.encode('x')), false); - // Write after fail throws with the original error. - await assert.rejects(w.write(encoder.encode('x')), testError); + // Stored failure takes precedence over per-operation cancellation. + const signal = AbortSignal.abort('operation cancelled'); + await assert.rejects( + w.write(encoder.encode('x'), { signal }), + (error) => error === reason); + await assert.rejects(w.end({ signal }), (error) => error === reason); // Don't await stream.closed here — the reset stream may not trigger // server onstream (no data was sent before fail), so the server // won't count it. The stream is cleaned up when the session closes. diff --git a/test/parallel/test-stream-iter-broadcast-backpressure.js b/test/parallel/test-stream-iter-broadcast-backpressure.js index 6efa7eb54c73..e42bb93d5f1f 100644 --- a/test/parallel/test-stream-iter-broadcast-backpressure.js +++ b/test/parallel/test-stream-iter-broadcast-backpressure.js @@ -4,6 +4,7 @@ const common = require('../common'); const assert = require('assert'); const { broadcast, ondrain, text } = require('stream/iter'); +const { setImmediate } = require('timers/promises'); // ============================================================================= // Backpressure policies @@ -67,7 +68,7 @@ async function testDropPoliciesReportPhysicalCapacity() { // Drop policies still accept writes despite having no physical capacity. assert.strictEqual(writer.writeSync(chunk), true); assert.strictEqual(writer.canWrite, false); - await new Promise(setImmediate); + await setImmediate(); assert.strictEqual(drained, false); assert.strictEqual((await iterator.next()).done, false); @@ -93,14 +94,14 @@ async function testBlockBackpressure() { // Next write should block let writeResolved = false; const writePromise = writer.write(kChunk).then(() => { writeResolved = true; }); - await new Promise(setImmediate); + await setImmediate(); assert.strictEqual(writeResolved, false); // Drain consumer to unblock the pending write const iter = consumer[Symbol.asyncIterator](); const first = await iter.next(); assert.strictEqual(first.done, false); - await new Promise(setImmediate); + await setImmediate(); assert.strictEqual(writeResolved, true); writer.endSync(); @@ -122,7 +123,7 @@ async function testBlockBackpressureContent() { writer.writeSync(chunk1); const writePromise = writer.write(chunk2); - await new Promise(setImmediate); + await setImmediate(); // Read all and verify content const iter = consumer[Symbol.asyncIterator](); @@ -158,11 +159,7 @@ async function testStrictBackpressureOverflow() { }); writer.fail(); - await assert.rejects(pending, { - name: 'TypeError', - code: 'ERR_INVALID_STATE', - message: 'Invalid state: Failed', - }); + await assert.rejects(pending, (reason) => reason === undefined); } async function testEndDrainsPendingWrite() { @@ -195,7 +192,7 @@ async function testEndDrainsPendingWrite() { let endResolved = false; endPromise.then(common.mustCall(() => { endResolved = true; })); - await new Promise(setImmediate); + await setImmediate(); assert.strictEqual(endResolved, false); assert.strictEqual((await iter.next()).done, true); diff --git a/test/parallel/test-stream-iter-reason-propagation.js b/test/parallel/test-stream-iter-reason-propagation.js new file mode 100644 index 000000000000..7511dbb2c410 --- /dev/null +++ b/test/parallel/test-stream-iter-reason-propagation.js @@ -0,0 +1,271 @@ +// Flags: --experimental-stream-iter +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { + Broadcast, + broadcast, + from, + fromSync, + pipeTo, + pipeToSync, + pull, + push, + share, + shareSync, +} = require('stream/iter'); + +const reasons = [undefined, null, false, 0, '', 'failure']; + +async function rejectsWith(promise, expected) { + await assert.rejects(promise, (reason) => reason === expected); +} + +function throwsWith(fn, expected) { + assert.throws(fn, (reason) => reason === expected); +} + +function asyncThrowingSource(reason) { + return { + __proto__: null, + [Symbol.asyncIterator]() { + return { + __proto__: null, + next() { return Promise.reject(reason); }, + }; + }, + }; +} + +function syncThrowingSource(reason) { + return { + __proto__: null, + [Symbol.iterator]() { + return { + __proto__: null, + next() { throw reason; }, + }; + }, + }; +} + +async function testPipeReasons() { + for (const reason of reasons) { + let asyncFailCalled = false; + let asyncFailReason; + const asyncWriter = { + __proto__: null, + write() {}, + fail(error) { asyncFailCalled = true; asyncFailReason = error; }, + }; + const asyncSource = asyncThrowingSource(reason); + + await rejectsWith(pipeTo(asyncSource, asyncWriter), reason); + assert.strictEqual(asyncFailCalled, true); + assert.strictEqual(asyncFailReason, reason); + + let syncFailCalled = false; + let syncFailReason; + const syncWriter = { + __proto__: null, + writeSync() { return true; }, + endSync() { return 0; }, + fail(error) { syncFailCalled = true; syncFailReason = error; }, + }; + const syncSource = syncThrowingSource(reason); + + throwsWith(() => pipeToSync(syncSource, syncWriter), reason); + assert.strictEqual(syncFailCalled, true); + assert.strictEqual(syncFailReason, reason); + } +} + +async function testSharedSourceReasons() { + for (const reason of reasons) { + const asyncSource = asyncThrowingSource(reason); + const asyncIterator = share(asyncSource).pull()[Symbol.asyncIterator](); + await rejectsWith(asyncIterator.next(), reason); + + const syncSource = syncThrowingSource(reason); + const syncIterator = shareSync(syncSource).pull()[Symbol.iterator](); + throwsWith(() => syncIterator.next(), reason); + } +} + +async function testBroadcastFromReason() { + const reason = 'source failure'; + const source = asyncThrowingSource(reason); + const { broadcast: channel } = Broadcast.from(source); + await rejectsWith(channel.push()[Symbol.asyncIterator]().next(), reason); +} + +async function testWriterFailReasons() { + for (const reason of reasons) { + const pushed = push(); + pushed.writer.fail(reason); + await rejectsWith( + pushed.readable[Symbol.asyncIterator]().next(), reason); + await rejectsWith(pushed.writer.write('data'), reason); + await rejectsWith(pushed.writer.end(), reason); + + const broadcasted = broadcast(); + const iterator = broadcasted.broadcast.push()[Symbol.asyncIterator](); + const pendingRead = iterator.next(); + broadcasted.writer.fail(reason); + await rejectsWith(pendingRead, reason); + await rejectsWith(broadcasted.writer.write('data'), reason); + await rejectsWith(broadcasted.writer.end(), reason); + await rejectsWith( + broadcasted.broadcast.push()[Symbol.asyncIterator]().next(), reason); + } +} + +async function testDisposeFailsWithUndefined() { + const pushed = push(); + pushed.writer[Symbol.dispose](); + await rejectsWith( + pushed.readable[Symbol.asyncIterator]().next(), undefined); + + const broadcasted = broadcast(); + const next = broadcasted.broadcast + .push()[Symbol.asyncIterator]().next(); + broadcasted.writer[Symbol.dispose](); + await rejectsWith(next, undefined); +} + +async function testExplicitUndefinedCancellation() { + const broadcasted = broadcast(); + const broadcastNext = broadcasted.broadcast + .push()[Symbol.asyncIterator]().next(); + broadcasted.broadcast.cancel(undefined); + await rejectsWith(broadcastNext, undefined); + + const shared = share(from('data')); + const shareIterator = shared.pull()[Symbol.asyncIterator](); + shared.cancel(undefined); + await rejectsWith(shareIterator.next(), undefined); + + const syncShared = shareSync(fromSync('data')); + const syncIterator = syncShared.pull()[Symbol.iterator](); + syncShared.cancel(undefined); + throwsWith(() => syncIterator.next(), undefined); +} + +async function testPendingWriteAbortReasons() { + const chunk = new Uint8Array(16384); + + const pushed = push({ budget: chunk.byteLength }); + pushed.writer.writeSync(chunk); + const pushController = new AbortController(); + const pushWrite = pushed.writer.write('data', { + signal: pushController.signal, + }); + pushController.abort(null); + await rejectsWith(pushWrite, null); + + const broadcasted = broadcast({ budget: chunk.byteLength }); + broadcasted.broadcast.push(); + broadcasted.writer.writeSync(chunk); + const broadcastController = new AbortController(); + const broadcastWrite = broadcasted.writer.write('data', { + signal: broadcastController.signal, + }); + broadcastController.abort(null); + await rejectsWith(broadcastWrite, null); + broadcasted.broadcast.cancel(); +} + +async function testTransformReasons() { + for (const thrownReason of [undefined, 'transform failure']) { + let observedReason; + const watchSignal = (batch, { signal }) => { + signal.addEventListener('abort', () => { + observedReason = signal.reason; + }, { once: true }); + return batch; + }; + const throwReason = () => { throw thrownReason; }; + const transformed = pull(from('data'), watchSignal, throwReason); + const iterator = transformed[Symbol.asyncIterator](); + await rejectsWith(iterator.next(), thrownReason); + assert.strictEqual(observedReason, thrownReason); + } + + const controller = new AbortController(); + const started = Promise.withResolvers(); + const waitForAbort = (batch, { signal }) => { + started.resolve(); + const { promise, reject } = Promise.withResolvers(); + signal.addEventListener('abort', () => reject(signal.reason), { + once: true, + }); + return promise; + }; + const abortedIterator = pull(from('data'), waitForAbort, { + signal: controller.signal, + })[Symbol.asyncIterator](); + const next = abortedIterator.next(); + await started.promise; + controller.abort(null); + await rejectsWith(next, null); +} + +async function testIteratorThrowReasonReachesTransforms() { + const reason = undefined; + let observedReason; + const transformed = pull(from('data'), (batch, { signal }) => { + signal.addEventListener('abort', () => { + observedReason = signal.reason; + }, { once: true }); + return batch; + }); + const iterator = transformed[Symbol.asyncIterator](); + + await iterator.next(); + await rejectsWith(iterator.throw(reason), reason); + assert.strictEqual(observedReason, reason); +} + +async function testErroredWriterPrecedesOperationSignal() { + const failure = 0; + const consumerReason = 'consumer failure'; + const signal = AbortSignal.abort(null); + const { writer, readable } = push(); + const iterator = readable[Symbol.asyncIterator](); + + writer.fail(failure); + + await rejectsWith(writer.write('data', { signal }), failure); + await rejectsWith(writer.end({ signal }), failure); + await rejectsWith(iterator.throw(consumerReason), consumerReason); + await rejectsWith(writer.write('data'), failure); +} + +async function testCompletedBroadcastConsumerStaysCompleted() { + const { writer, broadcast: channel } = broadcast(); + const iterator = channel.push()[Symbol.asyncIterator](); + + await iterator.return(); + writer.fail(undefined); + + assert.deepStrictEqual(await iterator.next(), { + __proto__: null, + done: true, + value: undefined, + }); +} + +Promise.all([ + testPipeReasons(), + testSharedSourceReasons(), + testBroadcastFromReason(), + testWriterFailReasons(), + testDisposeFailsWithUndefined(), + testExplicitUndefinedCancellation(), + testPendingWriteAbortReasons(), + testTransformReasons(), + testIteratorThrowReasonReachesTransforms(), + testErroredWriterPrecedesOperationSignal(), + testCompletedBroadcastConsumerStaysCompleted(), +]).then(common.mustCall()); diff --git a/test/parallel/test-stream-iter-share-coverage.js b/test/parallel/test-stream-iter-share-coverage.js index 48866e61deab..8b0cc1967f2f 100644 --- a/test/parallel/test-stream-iter-share-coverage.js +++ b/test/parallel/test-stream-iter-share-coverage.js @@ -88,32 +88,122 @@ async function testSyncIteratorThrow() { assert.strictEqual(shared.consumerCount, 0); } -// Async source throws non-Error value → wrapError +async function testCompletedSyncConsumerStaysCompleted() { + const reason = undefined; + const source = { + __proto__: null, + [Symbol.iterator]() { + return { + __proto__: null, + next() { throw reason; }, + }; + }, + }; + const shared = shareSync(source); + const completed = shared.pull()[Symbol.iterator](); + const active = shared.pull()[Symbol.iterator](); + + completed.return(); + let caught = false; + try { + active.next(); + } catch (error) { + caught = true; + assert.strictEqual(error, reason); + } + assert.strictEqual(caught, true); + assert.deepStrictEqual(completed.next(), { + __proto__: null, + done: true, + value: undefined, + }); +} + +async function testSyncCancelIgnoresCleanupError() { + const reason = null; + const source = { + __proto__: null, + [Symbol.iterator]() { + let done = false; + return { + __proto__: null, + next() { + if (done) return { done: true, value: undefined }; + done = true; + return { done: false, value: [Buffer.from('data')] }; + }, + get return() { throw new Error('cleanup failed'); }, + }; + }, + }; + const shared = shareSync(source); + const iterator = shared.pull()[Symbol.iterator](); + + iterator.next(); + shared.cancel(reason); + + assert.strictEqual(shared.consumerCount, 0); + assert.throws(() => iterator.next(), (error) => error === reason); +} + +async function testAsyncCancelIgnoresCleanupGetterError() { + const reason = null; + const source = { + __proto__: null, + [Symbol.asyncIterator]() { + let done = false; + return { + __proto__: null, + next() { + if (done) return Promise.resolve({ done: true, value: undefined }); + done = true; + return Promise.resolve({ + done: false, + value: [Buffer.from('data')], + }); + }, + get return() { throw new Error('cleanup failed'); }, + }; + }, + }; + const shared = share(source); + const iterator = shared.pull()[Symbol.asyncIterator](); + + await iterator.next(); + shared.cancel(reason); + + assert.strictEqual(shared.consumerCount, 0); + await assert.rejects(iterator.next(), (error) => error === reason); +} + +// Async source preserves a non-Error thrown value. async function testShareSourceThrowsNonError() { + const reason = 'not an error'; async function* source() { yield [new TextEncoder().encode('ok')]; - throw 'not an error'; // eslint-disable-line no-throw-literal + throw reason; } const shared = share(source()); const consumer = shared.pull(); await assert.rejects(async () => { // eslint-disable-next-line no-unused-vars for await (const batch of consumer) { /* consume */ } - }, { code: 'ERR_OPERATION_FAILED' }); + }, (error) => error === reason); } -// Sync source throws non-Error value → wrapError +// Sync source preserves a non-Error thrown value. async function testSyncShareSourceThrowsNonError() { + const reason = 42; function* source() { yield [new TextEncoder().encode('ok')]; - throw 42; // eslint-disable-line no-throw-literal + throw reason; } const shared = shareSync(source()); const consumer = shared.pull(); assert.throws(() => { // eslint-disable-next-line no-unused-vars for (const batch of consumer) { /* consume */ } - }, { code: 'ERR_OPERATION_FAILED' }); + }, (error) => error === reason); } Promise.all([ @@ -123,6 +213,9 @@ Promise.all([ testSyncShareDispose(), testAsyncIteratorThrow(), testSyncIteratorThrow(), + testCompletedSyncConsumerStaysCompleted(), + testSyncCancelIgnoresCleanupError(), + testAsyncCancelIgnoresCleanupGetterError(), testShareSourceThrowsNonError(), testSyncShareSourceThrowsNonError(), ]).then(common.mustCall()); diff --git a/test/parallel/test-stream-iter-to-readable.js b/test/parallel/test-stream-iter-to-readable.js index d8287036e7e7..a58bf0087df5 100644 --- a/test/parallel/test-stream-iter-to-readable.js +++ b/test/parallel/test-stream-iter-to-readable.js @@ -16,13 +16,15 @@ const { toReadableSync, } = require('stream/iter'); +const kNeverResolves = new Promise(() => { }); + function collect(readable) { - return new Promise((resolve, reject) => { - const chunks = []; - readable.on('data', (chunk) => chunks.push(chunk)); - readable.on('end', () => resolve(Buffer.concat(chunks))); - readable.on('error', reject); - }); + const { promise, resolve, reject } = Promise.withResolvers(); + const chunks = []; + readable.on('data', (chunk) => chunks.push(chunk)); + readable.on('end', () => resolve(Buffer.concat(chunks))); + readable.on('error', reject); + return promise; } // ============================================================================= @@ -105,6 +107,84 @@ async function testErrorAsync() { }, { message: 'source failed' }); } +async function testFalsyErrorAsync() { + for (const [reason, code] of [ + [null, 'ERR_FALSY_VALUE_REJECTION'], + [{ __proto__: null }, 'ERR_OPERATION_FAILED'], + ]) { + const source = { + __proto__: null, + [Symbol.asyncIterator]() { + return { + __proto__: null, + next() { return Promise.reject(reason); }, + return() { return Promise.resolve({ done: true }); }, + }; + }, + }; + + await assert.rejects(collect(toReadable(source)), (error) => { + return error.code === code && error.reason === reason; + }); + } +} + +async function testFalsyThenableCleanupError() { + const reason = null; + const source = { + __proto__: null, + [Symbol.asyncIterator]() { + return { + __proto__: null, + next() { return kNeverResolves; }, + return() { + return { + __proto__: null, + then(resolve, reject) { reject(reason); }, + }; + }, + }; + }, + }; + const readable = toReadable(source); + const { promise, resolve } = Promise.withResolvers(); + readable.once('error', resolve); + + readable.destroy(); + + const result = await promise; + assert.strictEqual(result.code, 'ERR_FALSY_VALUE_REJECTION'); + assert.strictEqual(result.reason, reason); +} + +async function testFalsyCleanupGetterErrors() { + for (const [symbol, create] of [ + [Symbol.asyncIterator, toReadable], + [Symbol.iterator, toReadableSync], + ]) { + const reason = false; + const source = { + __proto__: null, + [symbol]() { + return { + __proto__: null, + next() { return kNeverResolves; }, + get return() { throw reason; }, + }; + }, + }; + const readable = create(source); + const { promise, resolve } = Promise.withResolvers(); + readable.once('error', resolve); + + readable.destroy(); + + const result = await promise; + assert.strictEqual(result.code, 'ERR_FALSY_VALUE_REJECTION'); + assert.strictEqual(result.reason, reason); + } +} + // ============================================================================= // fromStreamIter: empty source // ============================================================================= @@ -155,16 +235,16 @@ async function testDestroyAsync() { // Read a couple chunks then destroy const chunks = []; - await new Promise((resolve, reject) => { - readable.on('data', (chunk) => { - chunks.push(chunk); - if (chunks.length >= 3) { - readable.destroy(); - } - }); - readable.on('close', resolve); - readable.on('error', reject); + const { promise, resolve, reject } = Promise.withResolvers(); + readable.on('data', (chunk) => { + chunks.push(chunk); + if (chunks.length >= 3) { + readable.destroy(); + } }); + readable.on('close', resolve); + readable.on('error', reject); + await promise; assert.ok(chunks.length >= 3); assert.ok(returnCalled, 'iterator.return() should have been called'); @@ -190,15 +270,20 @@ async function testDestroyDuringBackpressure() { const readable = toReadable(gen(), { highWaterMark: 1 }); // Read one chunk to start the pump, then destroy while it's waiting - const chunk = await new Promise((resolve) => { + { + const { promise, resolve } = Promise.withResolvers(); readable.once('readable', () => resolve(readable.read())); - }); - assert.ok(chunk); + assert.ok(await promise); + } // The pump should be waiting on backpressure now. Destroy the stream. readable.destroy(); - await new Promise((resolve) => readable.on('close', resolve)); + { + const { promise, resolve } = Promise.withResolvers(); + readable.on('close', resolve); + await promise; + } assert.ok(readable.destroyed); assert.ok(returnCalled, 'iterator.return() should have been called'); } @@ -243,11 +328,11 @@ async function testPipeAsync() { }, }); - await new Promise((resolve, reject) => { - readable.pipe(writable); - writable.on('finish', resolve); - writable.on('error', reject); - }); + const { promise, resolve, reject } = Promise.withResolvers(); + readable.pipe(writable); + writable.on('finish', resolve); + writable.on('error', reject); + await promise; assert.strictEqual(Buffer.concat(chunks).toString(), 'pipe test data'); } @@ -472,6 +557,24 @@ async function testErrorSync() { }, { message: 'sync source failed' }); } +async function testFalsyErrorSync() { + const reason = false; + const source = { + __proto__: null, + [Symbol.iterator]() { + return { + __proto__: null, + next() { throw reason; }, + }; + }, + }; + + await assert.rejects(collect(toReadableSync(source)), (error) => { + return error.code === 'ERR_FALSY_VALUE_REJECTION' && + error.reason === reason; + }); +} + // ============================================================================= // fromStreamIterSync: empty source // ============================================================================= @@ -506,7 +609,9 @@ async function testDestroySync() { readable.read(); // Start iteration readable.destroy(); - await new Promise((resolve) => readable.on('close', resolve)); + const { promise, resolve } = Promise.withResolvers(); + readable.on('close', resolve); + await promise; assert.ok(returnCalled, 'iterator.return() should have been called'); } @@ -615,6 +720,9 @@ Promise.all([ testMultiBatchAsync(), testBackpressureAsync(), testErrorAsync(), + testFalsyErrorAsync(), + testFalsyThenableCleanupError(), + testFalsyCleanupGetterErrors(), testEmptyAsync(), testEmptyBatchAsync(), testDestroyAsync(), @@ -628,6 +736,7 @@ Promise.all([ testBackpressureSync(), testBackpressureSyncMultiChunkBatch(), testErrorSync(), + testFalsyErrorSync(), testDestroySync(), testRoundTrip(), testRoundTripWithCompression(), diff --git a/test/parallel/test-stream-iter-writable-from.js b/test/parallel/test-stream-iter-writable-from.js index 46cfb627cb4a..5cf7371caa5a 100644 --- a/test/parallel/test-stream-iter-writable-from.js +++ b/test/parallel/test-stream-iter-writable-from.js @@ -6,6 +6,7 @@ const common = require('../common'); const assert = require('assert'); +const { setImmediate, setTimeout } = require('timers/promises'); const { push, text, @@ -28,6 +29,79 @@ async function testBasicWrite() { assert.strictEqual(result, 'hello world'); } +async function testFalsyWriterRejectionBecomesClassicError() { + const nonCoercible = { __proto__: null }; + const trapped = new Proxy({}, { + getPrototypeOf() { throw new Error('unexpected coercion'); }, + }); + for (const [reason, code] of [ + [null, 'ERR_FALSY_VALUE_REJECTION'], + [nonCoercible, 'ERR_OPERATION_FAILED'], + [trapped, 'ERR_OPERATION_FAILED'], + ]) { + let failed = false; + let failReason; + const writable = toWritable({ + __proto__: null, + write() { return Promise.reject(reason); }, + fail(error) { failed = true; failReason = error; }, + }); + writable.on('error', common.mustCall()); + + const { promise, resolve, reject } = Promise.withResolvers(); + writable.write('data', common.mustCall((error) => { + for (const symbol of Object.getOwnPropertySymbols(error)) { + delete error[symbol]; + } + if (error) reject(error); + else resolve(); + })); + + await assert.rejects(promise, (error) => { + return error.code === code && error.reason === reason; + }); + await setImmediate(); + assert.strictEqual(failed, true); + assert.strictEqual(failReason, reason); + } +} + +async function testClassicWrapperReusePreservesErrorIdentity() { + const first = toWritable({ + __proto__: null, + write() { return Promise.reject(null); }, + fail() {}, + }); + first.on('error', common.mustCall()); + let wrapper; + { + const { promise, resolve } = Promise.withResolvers(); + first.write('first', common.mustCall((error) => { + wrapper = error; + resolve(); + })); + await promise; + } + + let failReason; + const second = toWritable({ + __proto__: null, + write() { return Promise.reject(wrapper); }, + fail(reason) { failReason = reason; }, + }); + second.on('error', common.mustCall()); + { + const { promise, resolve } = Promise.withResolvers(); + second.write('second', common.mustCall((error) => { + assert.strictEqual(error, wrapper); + resolve(); + })); + await promise; + } + await setImmediate(); + assert.strictEqual(failReason, wrapper); +} + // ============================================================================= // _write delegates to writer.write() // ============================================================================= @@ -46,12 +120,12 @@ async function testWriteDelegatesToWriter() { const writable = toWritable(writer); - await new Promise((resolve, reject) => { - writable.write('hello', (err) => { - if (err) reject(err); - else resolve(); - }); - }); + const { promise, resolve, reject } = Promise.withResolvers(); + writable.write('hello', common.mustCall((err) => { + if (err) reject(err); + else resolve(); + })); + await promise; assert.strictEqual(Buffer.concat(chunks).toString(), 'hello'); } @@ -86,7 +160,9 @@ async function testWritevDelegation() { writable.write('c'); writable.uncork(); - await new Promise((resolve) => writable.end(resolve)); + const { promise, resolve } = Promise.withResolvers(); + writable.end(resolve); + await promise; // Writev should have been called with the batched chunks assert.ok(batches.length > 0, 'writev should have been called'); @@ -129,9 +205,9 @@ async function testWriteSyncFirst() { const writable = toWritable(writer); - await new Promise((resolve) => { - writable.write('test', resolve); - }); + const { promise, resolve } = Promise.withResolvers(); + writable.write('test', resolve); + await promise; assert.ok(syncCalled, 'writeSync should have been called'); assert.ok(!asyncCalled, 'write should not have been called'); @@ -160,9 +236,9 @@ async function testWriteSyncFallback() { const writable = toWritable(writer); - await new Promise((resolve) => { - writable.write('test', resolve); - }); + const { promise, resolve } = Promise.withResolvers(); + writable.write('test', resolve); + await promise; assert.ok(syncCalled, 'writeSync should have been called'); assert.ok(asyncCalled, 'write should have been called as fallback'); @@ -191,7 +267,9 @@ async function testEndSyncFirst() { const writable = toWritable(writer); - await new Promise((resolve) => writable.end(resolve)); + const { promise, resolve } = Promise.withResolvers(); + writable.end(resolve); + await promise; assert.ok(endSyncCalled, 'endSync should have been called'); assert.ok(!endAsyncCalled, 'end should not have been called'); @@ -220,7 +298,9 @@ async function testEndSyncFallback() { const writable = toWritable(writer); - await new Promise((resolve) => writable.end(resolve)); + const { promise, resolve } = Promise.withResolvers(); + writable.end(resolve); + await promise; assert.ok(endSyncCalled, 'endSync should have been called'); assert.ok(endAsyncCalled, 'end should have been called as fallback'); @@ -243,7 +323,9 @@ async function testFinalDelegatesToEnd() { const writable = toWritable(writer); - await new Promise((resolve) => writable.end(resolve)); + const { promise, resolve } = Promise.withResolvers(); + writable.end(resolve); + await promise; assert.ok(endCalled, 'writer.end() should have been called'); } @@ -261,13 +343,13 @@ async function testDestroyDelegatesToFail() { }; const writable = toWritable(writer); - writable.on('error', () => {}); // Prevent unhandled + writable.on('error', common.mustCall()); const testErr = new Error('destroy test'); writable.destroy(testErr); // Give a tick for destroy to propagate - await new Promise((resolve) => setTimeout(resolve, 10)); + await setTimeout(10); assert.strictEqual(failReason, testErr); } @@ -286,13 +368,14 @@ async function testWriteErrorPropagation() { }; const writable = toWritable(writer); - - await assert.rejects(new Promise((resolve, reject) => { - writable.write('data', (err) => { - if (err) reject(err); - else resolve(); - }); - }), { message: 'write failed' }); + writable.on('error', common.mustCall()); + + const { promise, resolve, reject } = Promise.withResolvers(); + writable.write('data', common.mustCall((err) => { + if (err) reject(err); + else resolve(); + })); + await assert.rejects(promise, { message: 'write failed' }); } // ============================================================================= @@ -343,12 +426,12 @@ async function testPushWriterBlockBackpressureNoDuplicate() { const { writer, readable } = push({ budget: 16384, backpressure: 'unbounded' }); const writable = toWritable(writer); - await new Promise((resolve, reject) => { - writable.write('a', (err) => { - if (err) reject(err); - else resolve(); - }); - }); + const { promise, resolve, reject } = Promise.withResolvers(); + writable.write('a', common.mustCall((err) => { + if (err) reject(err); + else resolve(); + })); + await promise; writable.write('b'); writable.end(); @@ -365,12 +448,12 @@ async function testPushWriterBlockBackpressureWritevNoDuplicate() { const { writer, readable } = push({ budget: 16384, backpressure: 'unbounded' }); const writable = toWritable(writer); - await new Promise((resolve, reject) => { - writable.write('a', (err) => { - if (err) reject(err); - else resolve(); - }); - }); + const { promise, resolve, reject } = Promise.withResolvers(); + writable.write('a', common.mustCall((err) => { + if (err) reject(err); + else resolve(); + })); + await promise; writable.cork(); writable.write('b'); @@ -423,16 +506,14 @@ async function testSyncCallbackDeferred() { const writable = toWritable(writer); - const p = new Promise((resolve) => { - writable.write('test', () => { - callbackTick = true; - resolve(); - }); - // Callback should NOT have fired synchronously - assert.strictEqual(callbackTick, false); - }); - - await p; + const { promise, resolve } = Promise.withResolvers(); + writable.write('test', common.mustCall(() => { + callbackTick = true; + resolve(); + })); + // Callback should NOT have fired synchronously + assert.strictEqual(callbackTick, false); + await promise; assert.strictEqual(callbackTick, true); } @@ -452,10 +533,10 @@ async function testMinimalWriter() { const writable = toWritable(writer); - await new Promise((resolve) => { - writable.write('minimal'); - writable.end(resolve); - }); + const { promise, resolve } = Promise.withResolvers(); + writable.write('minimal'); + writable.end(resolve); + await promise; assert.strictEqual(Buffer.concat(chunks).toString(), 'minimal'); } @@ -474,7 +555,7 @@ async function testDestroyWithoutError() { const writable = toWritable(writer); writable.destroy(); - await new Promise((resolve) => setTimeout(resolve, 10)); + await setTimeout(10); assert.ok(!failCalled, 'fail should not be called on clean destroy'); } @@ -491,12 +572,12 @@ async function testDestroyWithError() { }; const writable = toWritable(writer); - writable.on('error', () => {}); + writable.on('error', common.mustCall()); const err = new Error('test'); writable.destroy(err); - await new Promise((resolve) => setTimeout(resolve, 10)); + await setTimeout(10); assert.strictEqual(failReason, err); } @@ -512,12 +593,12 @@ async function testDestroyWithoutFail() { }; const writable = toWritable(writer); - writable.on('error', () => {}); + writable.on('error', common.mustCall()); // Should not throw even though writer has no fail() writable.destroy(new Error('test')); - await new Promise((resolve) => setTimeout(resolve, 10)); + await setTimeout(10); assert.ok(writable.destroyed); } @@ -553,13 +634,14 @@ async function testWriteSyncThrowsPropagation() { }; const writable = toWritable(writer); - - await assert.rejects(new Promise((resolve, reject) => { - writable.write('test', (err) => { - if (err) reject(err); - else resolve(); - }); - }), { message: 'sync broken' }); + writable.on('error', common.mustCall()); + + const { promise, resolve, reject } = Promise.withResolvers(); + writable.write('test', common.mustCall((err) => { + if (err) reject(err); + else resolve(); + })); + await assert.rejects(promise, { message: 'sync broken' }); } // ============================================================================= @@ -575,13 +657,15 @@ async function testWriteThrowsSyncPropagation() { }; const writable = toWritable(writer); + writable.on('error', common.mustCall()); - await assert.rejects(new Promise((resolve, reject) => { - writable.write('data', (err) => { - if (err) reject(err); - else resolve(); - }); - }), { message: 'sync throw from write' }); + const { promise, resolve, reject } = Promise.withResolvers(); + writable.write('data', common.mustCall((err) => { + if (err) reject(err); + else resolve(); + })); + + await assert.rejects(promise, { message: 'sync throw from write' }); } // ============================================================================= @@ -598,15 +682,16 @@ async function testEndThrowsSyncPropagation() { }; const writable = toWritable(writer); - writable.on('error', () => {}); + writable.on('error', common.mustCall()); - await new Promise((resolve) => { - writable.end(common.mustCall((err) => { - assert.ok(err); - assert.strictEqual(err.message, 'sync throw from end'); - resolve(); - })); - }); + const { promise, resolve } = Promise.withResolvers(); + writable.end(common.mustCall((err) => { + assert.ok(err); + assert.strictEqual(err.message, 'sync throw from end'); + resolve(); + })); + + await promise; } // ============================================================================= @@ -619,6 +704,8 @@ testHighWaterMarkIsMaxSafeInt(); Promise.all([ testBasicWrite(), + testFalsyWriterRejectionBecomesClassicError(), + testClassicWrapperReusePreservesErrorIdentity(), testWriteDelegatesToWriter(), testWritevDelegation(), testWriteSyncFirst(), diff --git a/test/parallel/test-stream-iter-writable-interop.js b/test/parallel/test-stream-iter-writable-interop.js index a8ae0a99cad7..9940ffc34c93 100644 --- a/test/parallel/test-stream-iter-writable-interop.js +++ b/test/parallel/test-stream-iter-writable-interop.js @@ -7,6 +7,7 @@ const common = require('../common'); const assert = require('assert'); const { Writable } = require('stream'); +const { setImmediate } = require('timers/promises'); const { from, fromWritable, @@ -356,7 +357,7 @@ async function testEndReturnsByteCount() { async function testFail() { const writable = new Writable({ write(chunk, enc, cb) { cb(); } }); - writable.on('error', () => {}); // Prevent unhandled error + writable.on('error', common.mustCall()); const writer = fromWritable(writable); writer.fail(new Error('test fail')); @@ -496,6 +497,7 @@ async function testPipeToWithTransform() { async function testDispose() { const writable = new Writable({ write(chunk, enc, cb) { cb(); } }); + writable.on('error', common.mustCall()); const writer = fromWritable(writable); writer[Symbol.dispose](); @@ -504,6 +506,7 @@ async function testDispose() { async function testAsyncDispose() { const writable = new Writable({ write(chunk, enc, cb) { cb(); } }); + writable.on('error', common.mustCall()); const writer = fromWritable(writable); await writer[Symbol.asyncDispose](); @@ -581,7 +584,7 @@ async function testFailRejectsPendingWaiters() { // Never call cb -- stuck }, }); - writable.on('error', () => {}); // Prevent unhandled error + writable.on('error', common.mustCall()); const writer = fromWritable(writable, { backpressure: 'unbounded' }); @@ -594,6 +597,58 @@ async function testFailRejectsPendingWaiters() { await assert.rejects(writePromise, { message: 'fail reason' }); } +async function testFailPreservesReason() { + let classicError; + const writable = new Writable({ + highWaterMark: 1, + write() {}, + }); + writable.on('error', common.mustCall((error) => { classicError = error; })); + const writer = fromWritable(writable, { backpressure: 'unbounded' }); + const pending = writer.write('blocked data'); + const draining = ondrain(writer); + + writer.fail(null); + + await assert.rejects(pending, (reason) => reason === null); + await assert.rejects(draining, (reason) => reason === null); + await assert.rejects(writer.write('more'), (reason) => reason === null); + await assert.rejects(writer.end(), (reason) => reason === null); + await setImmediate(); + assert.strictEqual(classicError.code, 'ERR_FALSY_VALUE_REJECTION'); + assert.strictEqual(classicError.reason, null); +} + +async function testEndThrowPreservesReason() { + const reason = undefined; + const writable = new Writable({ + write(chunk, encoding, callback) { callback(); }, + }); + writable.on('error', common.mustCall()); + writable.end = () => { throw reason; }; + const writer = fromWritable(writable); + + await assert.rejects(writer.end(), (error) => error === reason); + await assert.rejects(writer.write('more'), (error) => error === reason); +} + +async function testFailWhileClosingPreservesReason() { + let finish; + const writable = new Writable({ + write(chunk, encoding, callback) { callback(); }, + final(callback) { finish = callback; }, + }); + writable.on('error', common.mustCall()); + const writer = fromWritable(writable); + const ending = writer.end(); + + writer.fail(false); + + await assert.rejects(ending, (reason) => reason === false); + await assert.rejects(writer.write('more'), (reason) => reason === false); + finish(); +} + // ============================================================================= // dispose rejects pending block waiters // ============================================================================= @@ -605,6 +660,7 @@ async function testDisposeRejectsPendingWaiters() { // Never call cb -- stuck }, }); + writable.on('error', common.mustCall()); const writer = fromWritable(writable, { backpressure: 'unbounded' }); @@ -613,7 +669,7 @@ async function testDisposeRejectsPendingWaiters() { writer[Symbol.dispose](); - await assert.rejects(writePromise, { name: 'AbortError' }); + await assert.rejects(writePromise, (reason) => reason === undefined); } // ============================================================================= @@ -672,5 +728,8 @@ Promise.all([ testAsyncDispose(), testWriteInvalidChunkType(), testFailRejectsPendingWaiters(), + testFailPreservesReason(), + testFailWhileClosingPreservesReason(), + testEndThrowPreservesReason(), testDisposeRejectsPendingWaiters(), ]).then(common.mustCall()); From 678ff3561a33bbb83209aeecc6c241e285ba893a Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sun, 6 Sep 2026 14:33:46 -0700 Subject: [PATCH 038/217] zlib: improve zstd decoding across chunk boundaries Discovered when testing some other changes that zstd was not handling certain chunk boundaries very well. This fixes it. - Decodes concatenated and skippable zstd frames across all chunk boundaries. - Preserves rejectGarbageAfterEnd semantics. - Reports corruption in subsequent frames. - Handles trailing partial frame identifiers consistently. - Covers classic sync/async and iterable APIs. - Updated documentation and internal typings. Signed-off-by: James M Snell Assisted-by: Opencode PR-URL: https://github.com/nodejs/node/pull/65865 Reviewed-By: Filip Skokan Reviewed-By: Trivikram Kamat --- doc/api/zlib.md | 5 +- lib/zlib.js | 12 +- src/node_zlib.cc | 119 ++++++++++++--- .../test-stream-iter-transform-roundtrip.js | 16 ++ .../test-stream-iter-transform-sync.js | 11 ++ .../test-zlib-reject-garbage-after-end.js | 142 +++++++++++++++++- typings/internalBinding/zlib.d.ts | 4 +- 7 files changed, 282 insertions(+), 27 deletions(-) diff --git a/doc/api/zlib.md b/doc/api/zlib.md index 3daa12b4f78e..23df37f3339f 100644 --- a/doc/api/zlib.md +++ b/doc/api/zlib.md @@ -2222,7 +2222,7 @@ Each Zstd-based class takes an `options` object. All options are optional. to improve compression efficiency when compressing or decompressing data that shares common patterns with the dictionary. * `rejectGarbageAfterEnd` {boolean} If `true`, decompression fails when - input remains after the first complete compressed stream. **Default:** `false` + input remains after a complete sequence of Zstd frames. **Default:** `false` For example: @@ -2258,7 +2258,8 @@ added: - v22.15.0 --> -Decompress data using the Zstd algorithm. +Decompress data using the Zstd algorithm. Concatenated Zstd and skippable frames +are decoded as a single stream. ## `zlib.constants` diff --git a/lib/zlib.js b/lib/zlib.js index 29c61b51fc6f..0d7dca9a0c0e 100644 --- a/lib/zlib.js +++ b/lib/zlib.js @@ -203,9 +203,14 @@ function zlibOnError(message, errno, code) { // There is no way to cleanly recover. // Continuing only obscures problems. - const error = genericNodeError(message, { errno, code }); - error.errno = errno; - error.code = code; + let error; + if (code === 'ERR_TRAILING_JUNK_AFTER_STREAM_END') { + error = new ERR_TRAILING_JUNK_AFTER_STREAM_END(); + } else { + error = genericNodeError(message, { errno, code }); + error.errno = errno; + error.code = code; + } self.destroy(error); self[kError] = error; } @@ -962,6 +967,7 @@ class Zstd extends ZlibBase { writeState, processCallback, dictionary, + opts?.rejectGarbageAfterEnd === true, ); super(opts, mode, handle, zstdDefaultOpts); diff --git a/src/node_zlib.cc b/src/node_zlib.cc index af82aa2ae73b..c74e98c9cd11 100644 --- a/src/node_zlib.cc +++ b/src/node_zlib.cc @@ -336,7 +336,8 @@ class ZstdCompressContext final : public ZstdContext { // Zstd specific: CompressionError Init(uint64_t pledged_src_size, - std::string_view dictionary = {}); + std::string_view dictionary = {}, + bool reject_garbage_after_end = false); CompressionError SetParameter(int key, int value); // Wrap ZSTD_freeCCtx to remove the return type. @@ -365,7 +366,8 @@ class ZstdDecompressContext final : public ZstdContext { // Zstd specific: CompressionError Init(uint64_t pledged_src_size, - std::string_view dictionary = {}); + std::string_view dictionary = {}, + bool reject_garbage_after_end = false); CompressionError SetParameter(int key, int value); @@ -379,6 +381,11 @@ class ZstdDecompressContext final : public ZstdContext { private: DeleteFnPtr dctx_; bool frame_complete_ = false; + bool decoding_frame_after_complete_ = false; + bool reject_garbage_after_end_ = false; + bool ignoring_trailing_input_ = false; + size_t frame_prefix_size_ = 0; + uint8_t possible_frame_types_ = 0; }; class CompressionStreamMemoryOwner { @@ -947,9 +954,9 @@ class ZstdStream final : public CompressionStream { } static void Init(const FunctionCallbackInfo& args) { - CHECK((args.Length() == 4 || args.Length() == 5) && + CHECK((args.Length() >= 4 && args.Length() <= 6) && "init(params, pledgedSrcSize, writeResult, writeCallback[, " - "dictionary])"); + "dictionary[, rejectGarbageAfterEnd]])"); ZstdStream* wrap; ASSIGN_OR_RETURN_UNWRAP(&wrap, args.This()); @@ -986,7 +993,7 @@ class ZstdStream final : public CompressionStream { AllocScope alloc_scope(wrap); std::string_view dictionary; ArrayBufferViewContents contents; - if (args.Length() == 5 && !args[4]->IsUndefined()) { + if (args.Length() >= 5 && !args[4]->IsUndefined()) { if (!args[4]->IsArrayBufferView()) { THROW_ERR_INVALID_ARG_TYPE( wrap->env(), "dictionary must be an ArrayBufferView if provided"); @@ -996,7 +1003,14 @@ class ZstdStream final : public CompressionStream { dictionary = std::string_view(contents.data(), contents.length()); } - CompressionError err = wrap->context()->Init(pledged_src_size, dictionary); + bool reject_garbage_after_end = false; + if (args.Length() == 6) { + CHECK(args[5]->IsBoolean()); + reject_garbage_after_end = args[5]->IsTrue(); + } + + CompressionError err = wrap->context()->Init( + pledged_src_size, dictionary, reject_garbage_after_end); if (err.IsError()) { wrap->EmitError(err); THROW_ERR_ZLIB_INITIALIZATION_FAILED(wrap->env(), err.message); @@ -1661,7 +1675,8 @@ void ZstdCompressContext::Close() { } CompressionError ZstdCompressContext::Init(uint64_t pledged_src_size, - std::string_view dictionary) { + std::string_view dictionary, + bool) { pledged_src_size_ = pledged_src_size; if (pledged_src_size == ZSTD_CONTENTSIZE_UNKNOWN) { consumed_src_size_.reset(); @@ -1745,8 +1760,14 @@ void ZstdDecompressContext::Close() { } CompressionError ZstdDecompressContext::Init(uint64_t pledged_src_size, - std::string_view dictionary) { + std::string_view dictionary, + bool reject_garbage_after_end) { frame_complete_ = false; + decoding_frame_after_complete_ = false; + reject_garbage_after_end_ = reject_garbage_after_end; + ignoring_trailing_input_ = false; + frame_prefix_size_ = 0; + possible_frame_types_ = 0; #ifdef NODE_BUNDLED_ZSTD ZSTD_customMem custom_mem = { @@ -1779,10 +1800,14 @@ CompressionError ZstdDecompressContext::Init(uint64_t pledged_src_size, CompressionError ZstdDecompressContext::ResetStream() { // We pass ZSTD_CONTENTSIZE_UNKNOWN because the argument is ignored for // decompression. - return Init(ZSTD_CONTENTSIZE_UNKNOWN); + return Init(ZSTD_CONTENTSIZE_UNKNOWN, {}, reject_garbage_after_end_); } void ZstdDecompressContext::DoThreadPoolWork() { + if (ignoring_trailing_input_) { + return; + } + // The JavaScript processing loop retries with an empty input buffer when the // previous call filled the output buffer. Avoid interpreting that retry as // the beginning of a new, incomplete frame. @@ -1790,15 +1815,64 @@ void ZstdDecompressContext::DoThreadPoolWork() { return; } - size_t const ret = ZSTD_decompressStream(dctx_.get(), &output_, &input_); - if (ZSTD_isError(ret)) { - frame_complete_ = false; - error_ = ZSTD_getErrorCode(ret); - error_code_string_ = ZstdStrerror(error_); - error_string_ = ZSTD_getErrorString(error_); - } else { + do { + if (frame_complete_) { + decoding_frame_after_complete_ = true; + frame_prefix_size_ = 0; + possible_frame_types_ = 0b11; + } + + if (decoding_frame_after_complete_ && frame_prefix_size_ < 4) { + static constexpr uint8_t zstd_magic[] = {0x28, 0xb5, 0x2f, 0xfd}; + static constexpr uint8_t skippable_magic[] = {0x50, 0x2a, 0x4d, 0x18}; + const auto* data = static_cast(input_.src); + size_t input_prefix_offset = 0; + + while (frame_prefix_size_ < 4 && + input_.pos + input_prefix_offset < input_.size) { + const size_t index = frame_prefix_size_; + const uint8_t byte = data[input_.pos + input_prefix_offset]; + if (byte != zstd_magic[index]) { + possible_frame_types_ &= ~0b01; + } + if ((index == 0 && (byte & 0xf0) != skippable_magic[0]) || + (index != 0 && byte != skippable_magic[index])) { + possible_frame_types_ &= ~0b10; + } + frame_prefix_size_++; + input_prefix_offset++; + } + + if (possible_frame_types_ == 0) { + frame_complete_ = true; + decoding_frame_after_complete_ = false; + if (reject_garbage_after_end_) { + error_ = ZSTD_error_GENERIC; + error_code_string_ = "ERR_TRAILING_JUNK_AFTER_STREAM_END"; + error_string_ = + "Trailing junk found after the end of the compressed stream"; + } else { + ignoring_trailing_input_ = true; + } + return; + } + } + + const size_t ret = ZSTD_decompressStream(dctx_.get(), &output_, &input_); + if (ZSTD_isError(ret)) { + frame_complete_ = false; + error_ = ZSTD_getErrorCode(ret); + error_code_string_ = ZstdStrerror(error_); + error_string_ = ZSTD_getErrorString(error_); + return; + } + frame_complete_ = ret == 0; - } + if (frame_complete_) { + decoding_frame_after_complete_ = false; + } + } while (frame_complete_ && input_.pos < input_.size && + output_.pos < output_.size); } CompressionError ZstdDecompressContext::GetErrorInfo() const { @@ -1809,6 +1883,17 @@ CompressionError ZstdDecompressContext::GetErrorInfo() const { if (flush_ == ZSTD_e_end && !frame_complete_ && input_.pos == input_.size && output_.pos < output_.size) { + if (decoding_frame_after_complete_) { + if (frame_prefix_size_ < 4) { + if (reject_garbage_after_end_) { + return CompressionError( + "Trailing junk found after the end of the compressed stream", + "ERR_TRAILING_JUNK_AFTER_STREAM_END", + -1); + } + return {}; + } + } return CompressionError( "unexpected end of file", "Z_BUF_ERROR", Z_BUF_ERROR); } diff --git a/test/parallel/test-stream-iter-transform-roundtrip.js b/test/parallel/test-stream-iter-transform-roundtrip.js index d9a745593c27..a734b6c3132e 100644 --- a/test/parallel/test-stream-iter-transform-roundtrip.js +++ b/test/parallel/test-stream-iter-transform-roundtrip.js @@ -19,6 +19,7 @@ const { decompressBrotli, decompressZstd, } = require('zlib/iter'); +const zlib = require('zlib'); // ============================================================================= // Helper: compress then decompress, verify round-trip equality @@ -143,6 +144,20 @@ async function testZstdActuallyCompresses() { `Compressed ${compressed.byteLength} should be < original ${inputBuf.byteLength}`); } +async function testZstdConcatenatedFrames() { + const first = zlib.zstdCompressSync('a'); + const second = zlib.zstdCompressSync('b'); + const input = Buffer.concat([first, second]); + const result = await bytes(pull(from(input), decompressZstd())); + assert.strictEqual(Buffer.from(result).toString(), 'ab'); + + const withJunk = await bytes(pull( + from([first, Buffer.from('junk'), second]), + decompressZstd(), + )); + assert.strictEqual(Buffer.from(withJunk).toString(), 'a'); +} + // ============================================================================= // Binary data round-trip - verify no corruption on non-text data // ============================================================================= @@ -280,6 +295,7 @@ async function testGzipWithLevel() { await testZstdRoundTrip(); await testZstdLargeData(); await testZstdActuallyCompresses(); + await testZstdConcatenatedFrames(); // Binary data await testBinaryRoundTripGzip(); diff --git a/test/parallel/test-stream-iter-transform-sync.js b/test/parallel/test-stream-iter-transform-sync.js index d674c26cca10..991dba2748ed 100644 --- a/test/parallel/test-stream-iter-transform-sync.js +++ b/test/parallel/test-stream-iter-transform-sync.js @@ -19,6 +19,7 @@ const { decompressBrotliSync, decompressZstdSync, } = require('zlib/iter'); +const { zstdCompressSync } = require('zlib'); // ============================================================================= // Helper: sync compress then decompress, verify round-trip equality @@ -118,6 +119,15 @@ function testZstdLargeData() { assert.strictEqual(result, input); } +function testZstdConcatenatedFrames() { + const input = Buffer.concat([ + zstdCompressSync('a'), + zstdCompressSync('b'), + ]); + const result = bytesSync(pullSync(fromSync(input), decompressZstdSync())); + assert.strictEqual(Buffer.from(result).toString(), 'ab'); +} + // ============================================================================= // Cross-algorithm: compress async-compatible, decompress sync (and vice versa) // The sync transforms should produce output compatible with the standard format @@ -218,6 +228,7 @@ testBrotliRoundTrip(); testBrotliLargeData(); testZstdRoundTrip(); testZstdLargeData(); +testZstdConcatenatedFrames(); testGzipWithOptions(); testBrotliWithOptions(); testMixedStatelessAndStateful(); diff --git a/test/parallel/test-zlib-reject-garbage-after-end.js b/test/parallel/test-zlib-reject-garbage-after-end.js index 8039865f5f11..5cd36f42c267 100644 --- a/test/parallel/test-zlib-reject-garbage-after-end.js +++ b/test/parallel/test-zlib-reject-garbage-after-end.js @@ -23,10 +23,13 @@ function callAsync(fn, input, options) { }); } -async function collect(stream, input) { +async function collect(stream, ...inputs) { const chunks = []; stream.on('data', (chunk) => chunks.push(chunk)); - stream.end(input); + for (let i = 0; i < inputs.length - 1; i++) { + stream.write(inputs[i]); + } + stream.end(inputs[inputs.length - 1]); await finished(stream); return Buffer.concat(chunks); } @@ -79,6 +82,7 @@ const cases = [ decompressSync: zlib.zstdDecompressSync, createDecompress: zlib.createZstdDecompress, defaultOutput: 'a', + trailingInput: Buffer.from('trailing junk'), }, ]; @@ -89,10 +93,14 @@ for (const { decompressSync, createDecompress, defaultOutput, + trailingInput, } of cases) { test(`rejectGarbageAfterEnd rejects trailing input for ${label}`, async () => { const compressed = compress(Buffer.from('a')); - const withTrailingInput = Buffer.concat([compressed, compressed]); + const withTrailingInput = Buffer.concat([ + compressed, + trailingInput ?? compressed, + ]); assert.strictEqual(decompressSync(withTrailingInput).toString(), defaultOutput); assert.strictEqual( @@ -122,6 +130,134 @@ for (const { }); } +test('zstd decompresses concatenated frames regardless of chunking', async () => { + const first = zlib.zstdCompressSync('a'); + const second = zlib.zstdCompressSync('b'); + const skippable = Buffer.alloc(12); + skippable.writeUInt32LE(0x184d2a50, 0); + skippable.writeUInt32LE(4, 4); + skippable.write('meta', 8); + + for (const input of [ + Buffer.concat([first, second]), + Buffer.concat([first, skippable, second]), + ]) { + for (const rejectGarbageAfterEnd of [false, true]) { + const options = { rejectGarbageAfterEnd }; + assert.strictEqual( + zlib.zstdDecompressSync(input, options).toString(), + 'ab', + ); + assert.strictEqual( + (await callAsync(zlib.zstdDecompress, input, options)).toString(), + 'ab', + ); + + for (let split = 0; split <= input.length; split++) { + assert.strictEqual( + (await collect( + zlib.createZstdDecompress(options), + input.subarray(0, split), + input.subarray(split), + )).toString(), + 'ab', + `split at byte ${split}`, + ); + } + } + } +}); + +test('zstd trailing junk handling is independent of chunking', async () => { + const compressed = zlib.zstdCompressSync('a'); + const laterFrame = zlib.zstdCompressSync('b'); + const junk = Buffer.from('trailing junk'); + + assert.strictEqual( + (await collect( + zlib.createZstdDecompress(), + compressed, + junk, + laterFrame, + )).toString(), + 'a', + ); + await assert.rejects( + collect( + zlib.createZstdDecompress({ rejectGarbageAfterEnd: true }), + compressed, + junk, + ), + trailingJunkError, + ); +}); + +test('zstd handles incomplete trailing frame identifiers as junk', async () => { + const compressed = zlib.zstdCompressSync('a'); + const framePrefix = zlib.zstdCompressSync('b').subarray(0, 3); + + for (let length = 1; length <= framePrefix.length; length++) { + const trailing = framePrefix.subarray(0, length); + assert.strictEqual( + zlib.zstdDecompressSync(Buffer.concat([compressed, trailing])).toString(), + 'a', + ); + assert.throws( + () => zlib.zstdDecompressSync(Buffer.concat([compressed, trailing]), { + rejectGarbageAfterEnd: true, + }), + trailingJunkError, + ); + assert.strictEqual( + (await collect( + zlib.createZstdDecompress(), + compressed, + trailing, + )).toString(), + 'a', + ); + } +}); + +test('zstd reports errors in subsequent frames', async () => { + const first = zlib.zstdCompressSync('a'); + const second = zlib.zstdCompressSync('b', { + params: { + [zlib.constants.ZSTD_c_checksumFlag]: 1, + }, + }); + second[second.length - 1] ^= 1; + const input = Buffer.concat([first, second]); + const checksumError = { code: 'ZSTD_error_checksum_wrong' }; + + assert.throws(() => zlib.zstdDecompressSync(input), checksumError); + await assert.rejects( + collect(zlib.createZstdDecompress(), first, second), + checksumError, + ); +}); + +test('zstd decompresses multiple frames across output buffers', async () => { + const firstInput = Buffer.allocUnsafe(1024); + const secondInput = Buffer.allocUnsafe(1024); + for (let i = 0; i < firstInput.length; i++) { + firstInput[i] = i; + secondInput[i] = i + 1; + } + const input = Buffer.concat([ + zlib.zstdCompressSync(firstInput), + zlib.zstdCompressSync(secondInput), + ]); + const expected = Buffer.concat([firstInput, secondInput]); + const options = { chunkSize: 64 }; + + assert.deepStrictEqual(zlib.zstdDecompressSync(input, options), expected); + assert.deepStrictEqual( + await collect(zlib.createZstdDecompress(options), input), + expected, + ); +}); + test('rejectGarbageAfterEnd must be a boolean', () => { const compressed = zlib.deflateSync(Buffer.from('a')); diff --git a/typings/internalBinding/zlib.d.ts b/typings/internalBinding/zlib.d.ts index 706337eb45a7..84c437350433 100644 --- a/typings/internalBinding/zlib.d.ts +++ b/typings/internalBinding/zlib.d.ts @@ -32,12 +32,12 @@ declare namespace InternalZlibBinding { class ZstdCompress extends ZlibBase { constructor(); - init(initParamsArray: Uint32Array, pledgedSrcSize: number | undefined, writeState: Uint32Array, callback: VoidFunction, dictionary?: ArrayBufferView): void; + init(initParamsArray: Uint32Array, pledgedSrcSize: number | undefined, writeState: Uint32Array, callback: VoidFunction, dictionary?: ArrayBufferView, rejectGarbageAfterEnd?: boolean): void; } class ZstdDecompress extends ZlibBase { constructor(); - init(initParamsArray: Uint32Array, pledgedSrcSize: number | undefined, writeState: Uint32Array, callback: VoidFunction, dictionary?: ArrayBufferView): void; + init(initParamsArray: Uint32Array, pledgedSrcSize: number | undefined, writeState: Uint32Array, callback: VoidFunction, dictionary?: ArrayBufferView, rejectGarbageAfterEnd?: boolean): void; } } From baf112b639b69d838a19cb7d4a5481dc2e9e7e61 Mon Sep 17 00:00:00 2001 From: Trivikram Kamat <16024985+trivikr@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:34:15 -0700 Subject: [PATCH 039/217] sqlite: always copy changeset before applying SQLite can invoke JavaScript through user-defined SQL functions while applying a changeset, even when no filter or conflict callback is set. Copy non-empty changesets unconditionally so JavaScript cannot detach or modify the input while SQLite is still reading it. Signed-off-by: Kamat, Trivikram <16024985+trivikr@users.noreply.github.com> Assisted-by: codex PR-URL: https://github.com/nodejs/node/pull/65870 Reviewed-By: Edy Silva Reviewed-By: James M Snell --- src/node_sqlite.cc | 9 ++++--- test/parallel/test-sqlite-session.js | 36 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/node_sqlite.cc b/src/node_sqlite.cc index a0af89346ecf..c3c97844e867 100644 --- a/src/node_sqlite.cc +++ b/src/node_sqlite.cc @@ -2681,12 +2681,11 @@ void DatabaseSync::ApplyChangeset(const FunctionCallbackInfo& args) { return; } - // A callback may detach/modify the input buffer mid-apply, so copy it. - // With no callbacks, no JS runs during sqlite3changeset_apply(), so no - // copy is needed. + // SQLite may invoke JavaScript through explicit callbacks or user-defined + // SQL functions while applying the changeset. Copy the input so JavaScript + // cannot detach or modify the memory while SQLite is still reading it. std::unique_ptr changeset; - if (buf.length() > 0 && - (context.filterCallback || context.conflictCallback)) { + if (buf.length() > 0) { changeset = ArrayBuffer::NewBackingStore( env->isolate(), buf.length(), diff --git a/test/parallel/test-sqlite-session.js b/test/parallel/test-sqlite-session.js index c36ed84a0858..40da13c4bb7f 100644 --- a/test/parallel/test-sqlite-session.js +++ b/test/parallel/test-sqlite-session.js @@ -462,6 +462,42 @@ test('database.applyChangeset() - changeset detached by filter', (t) => { ]); }); +test('database.applyChangeset() - changeset detached by SQL function', (t) => { + const database1 = new DatabaseSync(':memory:'); + const database2 = new DatabaseSync(':memory:'); + let changeset; + let detached = false; + + database1.exec('CREATE TABLE data(key INTEGER PRIMARY KEY, value TEXT)'); + database2.function('validate_changeset_value', (value) => { + if (!detached) { + detached = true; + const transferred = structuredClone(changeset.buffer, { + transfer: [changeset.buffer], + }); + new Uint8Array(transferred).fill(0); + } + return value.length; + }); + database2.exec(` + CREATE TABLE data( + key INTEGER PRIMARY KEY, + value TEXT CHECK (validate_changeset_value(value)) + ) + `); + + const session = database1.createSession(); + database1.exec("INSERT INTO data VALUES (1, 'hello'), (2, 'world')"); + changeset = session.changeset(); + + t.assert.strictEqual(database2.applyChangeset(changeset), true); + t.assert.strictEqual(changeset.byteLength, 0); + deepStrictEqual(t)(database2.prepare('SELECT * FROM data').all(), [ + { key: 1, value: 'hello' }, + { key: 2, value: 'world' }, + ]); +}); + test('database.createSession() - filter changes', (t) => { const database1 = new DatabaseSync(':memory:'); const database2 = new DatabaseSync(':memory:'); From 69911bf6b5857695d86125e6745a71e1f505d815 Mon Sep 17 00:00:00 2001 From: Trivikram Kamat <16024985+trivikr@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:34:28 -0700 Subject: [PATCH 040/217] doc: qualify directory read ordering for native fs Clarify that the documented operating system directory ordering applies to reads handled by the native file system. Signed-off-by: Kamat, Trivikram <16024985+trivikr@users.noreply.github.com> PR-URL: https://github.com/nodejs/node/pull/65868 Reviewed-By: James M Snell Reviewed-By: Luigi Pinca --- doc/api/fs.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/api/fs.md b/doc/api/fs.md index 315ecdb97c43..69aaf60c5b6a 100644 --- a/doc/api/fs.md +++ b/doc/api/fs.md @@ -7313,8 +7313,9 @@ Asynchronously read the next directory entry via readdir(3) as an A promise is returned that will be fulfilled with an {fs.Dirent}, or `null` if there are no more directory entries to read. -Directory entries returned by this function are in no particular order as -provided by the operating system's underlying directory mechanisms. +For directory reads handled by the native file system, directory entries +returned by this function are in no particular order as provided by the +operating system's underlying directory mechanisms. Entries added or removed while iterating over the directory might not be included in the iteration results. From de552bd0449013added2081d148fd9f75d8609d4 Mon Sep 17 00:00:00 2001 From: Trivikram Kamat <16024985+trivikr@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:34:40 -0700 Subject: [PATCH 041/217] vfs: return FileHandle from fs.promises.open Wrap mounted virtual file descriptors in the public FileHandle interface while delegating operations to the underlying provider handle. Signed-off-by: Kamat, Trivikram <16024985+trivikr@users.noreply.github.com> Assisted-by: codex:gpt-5.6-sol PR-URL: https://github.com/nodejs/node/pull/65730 Fixes: https://github.com/nodejs/node/issues/65729 Reviewed-By: James M Snell --- lib/internal/vfs/fd.js | 39 +++++++++++++++++++++++++++ lib/internal/vfs/setup.js | 5 ++-- test/parallel/test-vfs-fs-promises.js | 3 +++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/lib/internal/vfs/fd.js b/lib/internal/vfs/fd.js index bd36ad218f48..1d1ffaabc78d 100644 --- a/lib/internal/vfs/fd.js +++ b/lib/internal/vfs/fd.js @@ -1,6 +1,8 @@ 'use strict'; const { + FunctionPrototypeBind, + ObjectDefineProperty, SafeMap, Symbol, } = primordials; @@ -46,6 +48,42 @@ class VirtualFD { get entry() { return this[kEntry]; } + + getAsyncId() { + return this[kFd]; + } + + async close() { + await this[kEntry].close(); + closeVirtualFd(this[kFd]); + } + + closeSync() { + this[kEntry].closeSync(); + closeVirtualFd(this[kFd]); + } +} + +const vfsFileHandleMethods = [ + 'appendFile', 'chmod', 'chown', 'datasync', 'sync', 'read', 'readv', + 'readFile', 'stat', 'truncate', 'utimes', 'write', 'writev', 'writeFile', +]; +let FileHandle; + +function createVfsFileHandle(vfd) { + FileHandle ??= require('internal/fs/promises').FileHandle; + const fileHandle = new FileHandle(vfd); + const entry = vfd.entry; + for (let i = 0; i < vfsFileHandleMethods.length; i++) { + const method = vfsFileHandleMethods[i]; + ObjectDefineProperty(fileHandle, method, { + __proto__: null, + configurable: true, + value: FunctionPrototypeBind(entry[method], entry), + writable: true, + }); + } + return fileHandle; } /** @@ -81,6 +119,7 @@ function closeVirtualFd(fd) { module.exports = { VFS_FD_MASK, VirtualFD, + createVfsFileHandle, openVirtualFd, getVirtualFd, closeVirtualFd, diff --git a/lib/internal/vfs/setup.js b/lib/internal/vfs/setup.js index 3e6f246d794a..7231d10825ea 100644 --- a/lib/internal/vfs/setup.js +++ b/lib/internal/vfs/setup.js @@ -30,7 +30,7 @@ const { getLayerIdFromPath, getNormalizedVfsRoot, } = require('internal/vfs/router'); -const { getVirtualFd, closeVirtualFd } = require('internal/vfs/fd'); +const { getVirtualFd, closeVirtualFd, createVfsFileHandle } = require('internal/vfs/fd'); const { assertEncoding, setVfsHandlers } = require('internal/fs/utils'); const permission = require('internal/process/permission'); const { getOptionValue } = require('internal/options'); @@ -786,8 +786,7 @@ function createVfsHandlers() { const r = findVFSForPath(pathStr); if (r !== null) { const fd = r.vfs.openSync(r.path, flags, mode); - const vfd = getVirtualFd(fd); - return PromiseResolve(vfd.entry); + return PromiseResolve(createVfsFileHandle(getVirtualFd(fd))); } } return undefined; diff --git a/test/parallel/test-vfs-fs-promises.js b/test/parallel/test-vfs-fs-promises.js index a5761d4ca5dd..af39fdf92b42 100644 --- a/test/parallel/test-vfs-fs-promises.js +++ b/test/parallel/test-vfs-fs-promises.js @@ -86,6 +86,9 @@ const vfs = require('node:vfs'); // FileHandle via fsp.open const handle = await fsp.open(p('src/hello.txt'), 'r'); + assert.strictEqual(handle.constructor.name, 'FileHandle'); + assert.strictEqual(typeof handle.fd, 'number'); + assert.strictEqual(typeof handle.createReadStream, 'function'); assert.strictEqual(await handle.readFile('utf8'), 'hello'); await handle.close(); From ac7fcf0cef4b4f4a423109d0b05a25a5167aa2e1 Mon Sep 17 00:00:00 2001 From: Richard Lau Date: Wed, 9 Sep 2026 03:51:23 +0100 Subject: [PATCH 042/217] build: sync cargo/rustc version warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The version checks in `configure.py` were not updated when the minimum rustc requirement was bumped to 1.86 for temporal_rs 0.2.3. Signed-off-by: Richard Lau PR-URL: https://github.com/nodejs/node/pull/65912 Refs: https://github.com/nodejs/node/pull/64543 Reviewed-By: René Reviewed-By: Mike McCready <66998419+MikeMcC399@users.noreply.github.com> Reviewed-By: Filip Skokan Reviewed-By: James M Snell Reviewed-By: Luigi Pinca --- configure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.py b/configure.py index 84a688917efe..8b3332a461f4 100755 --- a/configure.py +++ b/configure.py @@ -1669,8 +1669,8 @@ def check_compiler(o): # cargo and rustc are needed for Temporal. if not options.v8_disable_temporal_support and not options.shared_temporal_capi: # Minimum cargo and rustc versions should match values in BUILDING.md. - min_cargo_ver_tuple = (1, 82) - min_rustc_ver_tuple = (1, 82) + min_cargo_ver_tuple = (1, 86) + min_rustc_ver_tuple = (1, 86) cargo = os.environ.get('CARGO', 'cargo') cargo_ver = get_cargo_version(cargo) print_verbose(f'Detected cargo (CARGO={cargo}): {cargo_ver}') From cf5d4a8fe625183ac103f0928921528528c15c69 Mon Sep 17 00:00:00 2001 From: Trivikram Kamat <16024985+trivikr@users.noreply.github.com> Date: Tue, 8 Sep 2026 22:27:35 -0700 Subject: [PATCH 043/217] vfs: reject statfs for missing paths Validate mounted VFS paths before returning synthetic statfs data so fs.statfsSync() and fs.promises.statfs() report ENOENT for missing paths. Forward validation errors asynchronously to fs.statfs() callbacks. Signed-off-by: Kamat, Trivikram <16024985+trivikr@users.noreply.github.com> Assisted-by: codex:gpt-5.6-sol PR-URL: https://github.com/nodejs/node/pull/65693 Fixes: https://github.com/nodejs/node/issues/65692 Reviewed-By: James M Snell Reviewed-By: Matteo Collina --- lib/fs.js | 8 +++++++- lib/internal/vfs/setup.js | 5 +++-- test/parallel/test-vfs-fs-promises.js | 5 +++++ test/parallel/test-vfs-fs-statSync.js | 17 ++++++++++++++++- 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/lib/fs.js b/lib/fs.js index b858902cf44a..d53120af9bb3 100644 --- a/lib/fs.js +++ b/lib/fs.js @@ -2050,7 +2050,13 @@ function statfs(path, options = { __proto__: null, bigint: false }, callback) { const h = vfsState.handlers; if (h !== null) { - const result = h.statfsSync(path, options); + let result; + try { + result = h.statfsSync(path, options); + } catch (err) { + process.nextTick(callback, err); + return; + } if (result !== undefined) { process.nextTick(callback, null, result); return; diff --git a/lib/internal/vfs/setup.js b/lib/internal/vfs/setup.js index 7231d10825ea..3e183e422831 100644 --- a/lib/internal/vfs/setup.js +++ b/lib/internal/vfs/setup.js @@ -320,6 +320,7 @@ function findVFSWith(filename, syscall, fn) { const r = findVFS(filename); if (r === null) return undefined; if (r.vfs.existsSync(filename)) { + if (fn === undefined) return true; return fn(r.vfs, filename); } throw createENOENT(syscall, filename); @@ -432,7 +433,7 @@ function createVfsHandlers() { }, statfsSync(path, options) { const pathStr = toPathStr(path); - if (pathStr !== null && findVFSForPath(pathStr) !== null) { + if (pathStr !== null && findVFSWith(pathStr, 'statfs')) { if (options?.bigint) { return { type: 0n, bsize: 4096n, blocks: 0n, @@ -709,7 +710,7 @@ function createVfsHandlers() { vfsOp(path, (vfs, n) => vfs.promises.lutimes(n, atime, mtime).then(() => true)), statfs(path, options) { const pathStr = toPathStr(path); - if (pathStr !== null && findVFSForPath(pathStr) !== null) { + if (pathStr !== null && findVFSWith(pathStr, 'statfs')) { if (options?.bigint) { return { __proto__: null, diff --git a/test/parallel/test-vfs-fs-promises.js b/test/parallel/test-vfs-fs-promises.js index af39fdf92b42..b720aad37ccc 100644 --- a/test/parallel/test-vfs-fs-promises.js +++ b/test/parallel/test-vfs-fs-promises.js @@ -31,6 +31,11 @@ const vfs = require('node:vfs'); // statfs const sfs = await fsp.statfs(p('src/hello.txt')); assert.strictEqual(typeof sfs.bsize, 'number'); + await assert.rejects(fsp.statfs(p('missing')), { + code: 'ENOENT', + syscall: 'statfs', + path: p('missing'), + }); // Path-based writes await fsp.writeFile(p('src/pw.txt'), 'pdata'); diff --git a/test/parallel/test-vfs-fs-statSync.js b/test/parallel/test-vfs-fs-statSync.js index 5a48c4e426fa..6e5d27c3059e 100644 --- a/test/parallel/test-vfs-fs-statSync.js +++ b/test/parallel/test-vfs-fs-statSync.js @@ -4,7 +4,7 @@ // fs.statSync / fs.lstatSync / fs.statfsSync dispatch through the VFS layer, // including the `throwIfNoEntry: false` option. -require('../common'); +const common = require('../common'); const assert = require('assert'); const fs = require('fs'); const path = require('path'); @@ -57,4 +57,19 @@ assert.strictEqual( assert.strictEqual(typeof s.bsize, 'bigint'); } +// statfsSync on a missing path throws ENOENT +assert.throws(() => fs.statfsSync(path.join(mountPoint, 'missing')), + { + code: 'ENOENT', + syscall: 'statfs', + path: path.join(mountPoint, 'missing'), + }); + +// Statfs on a missing path reports ENOENT through the callback +fs.statfs(path.join(mountPoint, 'missing'), common.mustCall((err) => { + assert.strictEqual(err.code, 'ENOENT'); + assert.strictEqual(err.syscall, 'statfs'); + assert.strictEqual(err.path, path.join(mountPoint, 'missing')); +})); + myVfs.unmount(); From 07ae602005ce77c5f7af5bc476f113e89d65f78e Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Wed, 9 Sep 2026 09:34:30 +0200 Subject: [PATCH 044/217] tools: unlabel author ready on base branch conflicts Add a daily workflow that removes the `[author ready]` label from open pull requests conflicting with their base branch and comments with rebase guidance. Signed-off-by: Filip Skokan PR-URL: https://github.com/nodejs/node/pull/65872 Reviewed-By: Antoine du Hamel Reviewed-By: James M Snell Reviewed-By: Luigi Pinca Reviewed-By: Tierney Cyren --- .github/workflows/author-ready-conflicts.yml | 48 ++++++++++++++++++++ doc/contributing/collaborator-guide.md | 1 + 2 files changed, 49 insertions(+) create mode 100644 .github/workflows/author-ready-conflicts.yml diff --git a/.github/workflows/author-ready-conflicts.yml b/.github/workflows/author-ready-conflicts.yml new file mode 100644 index 000000000000..f447c6b80a2e --- /dev/null +++ b/.github/workflows/author-ready-conflicts.yml @@ -0,0 +1,48 @@ +name: Unlabel not-ready pull requests + +on: + schedule: + - cron: 15 3 * * * + +permissions: + contents: read + +jobs: + remove-label: + name: Remove author ready + if: github.repository == 'nodejs/node' + runs-on: ubuntu-slim + permissions: + pull-requests: write + steps: + - name: Remove the label and comment + env: + GH_TOKEN: ${{ github.token }} + MESSAGE: >- + This pull request has conflicts with its base branch, removing the + `author ready` label. + + Please + [rebase](https://github.com/nodejs/node/blob/HEAD/doc/contributing/pull-requests.md#step-5-rebase) + your branch onto the latest base branch, resolve the conflicts + locally, and force-push. + + Afterwards the pull request needs a fresh collaborator approval, + and a collaborator will add the label back once it is + [author ready](https://github.com/nodejs/node/blob/HEAD/doc/contributing/collaborator-guide.md#author-ready-pull-requests) + again. + run: | + # Requesting the mergeable field makes GitHub compute the merge state, + # pull requests still reported as UNKNOWN are picked up by a later run. + gh pr list \ + --repo "$GITHUB_REPOSITORY" \ + --label 'author ready' \ + --state open \ + --limit 100 \ + --json number,mergeable \ + --jq '.[] | select(.mergeable == "CONFLICTING") | .number' | + while read -r number; do + gh pr edit "$number" --repo "$GITHUB_REPOSITORY" --remove-label 'author ready' + printf '%s\n' "$MESSAGE" | + gh pr comment "$number" --repo "$GITHUB_REPOSITORY" --body-file - + done diff --git a/doc/contributing/collaborator-guide.md b/doc/contributing/collaborator-guide.md index 2966f4dfeec0..49ece70c2bb0 100644 --- a/doc/contributing/collaborator-guide.md +++ b/doc/contributing/collaborator-guide.md @@ -87,6 +87,7 @@ A pull request is _author ready_ when: * There is a CI run in progress or completed. * There is at least one collaborator approval. * There are no outstanding review comments. +* There are no conflicts with the base branch. Please always add the `author ready` label to the pull request in that case. Please always remove it again as soon as the conditions are not met anymore. From 0d14c0e507149a1ae2703a9712d256d416a72931 Mon Sep 17 00:00:00 2001 From: Wiyeong Seo Date: Wed, 9 Sep 2026 17:04:41 +0900 Subject: [PATCH 045/217] path: remove `StringPrototypeCharCodeAt` from some methods of `posix` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove `StringPrototypeCharCodeAt` from `posix.resolve`, `posix.normalize`, `posix.parse`. PR-URL: https://github.com/nodejs/node/pull/54668 Refs: https://github.com/nodejs/node/pull/54546 Reviewed-By: Luigi Pinca Reviewed-By: Jordan Harband Reviewed-By: Gürgün Dayıoğlu Reviewed-By: James M Snell --- lib/path.js | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/path.js b/lib/path.js index 63b037cddfb9..658b5d6901df 100644 --- a/lib/path.js +++ b/lib/path.js @@ -1262,8 +1262,7 @@ const posix = { } resolvedPath = `${path}/${resolvedPath}`; - resolvedAbsolute = - StringPrototypeCharCodeAt(path, 0) === CHAR_FORWARD_SLASH; + resolvedAbsolute = path[0] === '/'; } if (!resolvedAbsolute) { @@ -1296,10 +1295,8 @@ const posix = { if (path.length === 0) return '.'; - const isAbsolute = - StringPrototypeCharCodeAt(path, 0) === CHAR_FORWARD_SLASH; - const trailingSeparator = - StringPrototypeCharCodeAt(path, path.length - 1) === CHAR_FORWARD_SLASH; + const isAbsolute = path[0] === '/'; + const trailingSeparator = path[path.length - 1] === '/'; // Normalize the path path = normalizeString(path, !isAbsolute, '/', isPosixPathSeparator); @@ -1639,8 +1636,8 @@ const posix = { // Get non-dir info for (; i >= start; --i) { - const code = StringPrototypeCharCodeAt(path, i); - if (code === CHAR_FORWARD_SLASH) { + const char = path[i]; + if (char === '/') { // If we reached a path separator that was not part of a set of path // separators at the end of the string, stop now if (!matchedSlash) { @@ -1655,7 +1652,7 @@ const posix = { matchedSlash = false; end = i + 1; } - if (code === CHAR_DOT) { + if (char === '.') { // If this is our first dot, mark it as the start of our extension if (startDot === -1) startDot = i; From e5bbcfdd7881b3ba5e9766abfbcf652054866f37 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Wed, 9 Sep 2026 12:19:49 +0200 Subject: [PATCH 046/217] tools: lint PR commit messages without approval Use pull_request_target to run commit message linting without fork workflow approval. Fetch the first commit through the API and pass its message to a pinned validator as JSON on stdin. Capture validator output and report failures through escaped annotations, using only read access to pull requests. Signed-off-by: Filip Skokan PR-URL: https://github.com/nodejs/node/pull/65875 Reviewed-By: Mike McCready <66998419+MikeMcC399@users.noreply.github.com> Reviewed-By: James M Snell --- .../commit-lint-problem-matcher.json | 13 ----- .github/workflows/commit-lint.yml | 49 +++++++++++++------ 2 files changed, 33 insertions(+), 29 deletions(-) delete mode 100644 .github/workflows/commit-lint-problem-matcher.json diff --git a/.github/workflows/commit-lint-problem-matcher.json b/.github/workflows/commit-lint-problem-matcher.json deleted file mode 100644 index 72dd13b9e092..000000000000 --- a/.github/workflows/commit-lint-problem-matcher.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "problemMatcher": [ - { - "owner": "core-validate-commit", - "pattern": [ - { - "regexp": "^not ok \\d+ (.*)$", - "message": 1 - } - ] - } - ] -} diff --git a/.github/workflows/commit-lint.yml b/.github/workflows/commit-lint.yml index 5537d1e19c65..037ea7e810e0 100644 --- a/.github/workflows/commit-lint.yml +++ b/.github/workflows/commit-lint.yml @@ -1,7 +1,7 @@ name: First commit message adheres to guidelines on: - pull_request: + pull_request_target: branches: - main @@ -9,27 +9,44 @@ env: NODE_VERSION: lts/* permissions: - contents: read + pull-requests: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true jobs: lint-commit-message: + name: lint-commit-message runs-on: ubuntu-slim steps: - - name: Compute number of commits in the PR - id: nb-of-commits - run: | - echo "plusOne=$((${{ github.event.pull_request.commits }} + 1))" >> $GITHUB_OUTPUT - echo "minusOne=$((${{ github.event.pull_request.commits }} - 1))" >> $GITHUB_OUTPUT - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: ${{ steps.nb-of-commits.outputs.plusOne }} - persist-credentials: false - - run: git reset HEAD^2 - name: Install Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: ${{ env.NODE_VERSION }} - - name: Validate commit message - run: | - echo "::add-matcher::.github/workflows/commit-lint-problem-matcher.json" - git rev-parse HEAD~${{ steps.nb-of-commits.outputs.minusOne }} | xargs npx -q core-validate-commit --no-validate-metadata --tap + - name: Validate first commit message + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { data: [commit] } = await github.rest.pulls.listCommits({ + ...context.repo, + pull_number: context.issue.number, + per_page: 1, + }); + if (!commit) { + throw new Error('No commits found in pull request'); + } + const { exitCode, stdout, stderr } = await exec.getExecOutput('npx', [ + '-q', '--yes', '--ignore-scripts', 'core-validate-commit@6.0.0', + '--no-validate-metadata', '--tap', '-', + ], { + cwd: process.env.RUNNER_TEMP, + input: Buffer.from(JSON.stringify([{ id: commit.sha, message: commit.commit.message }])), + silent: true, + ignoreReturnCode: true, + }); + if (exitCode !== 0) { + core.setFailed(stdout + stderr || 'Commit message validation failed'); + } else { + core.info('First commit message passes validation'); + } From 7f514d33105da2db61e8eecc05d8a7f3b857171d Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Wed, 9 Sep 2026 15:57:36 +0200 Subject: [PATCH 047/217] lib: fix shared buffer growability validation Use the intrinsic growable getter instead of buffer.growable so shadowed properties cannot bypass validation or reject fixed buffers. Signed-off-by: Filip Skokan Assisted-by: GitHub Copilot PR-URL: https://github.com/nodejs/node/pull/65845 Reviewed-By: James M Snell Reviewed-By: Yagiz Nizipli --- lib/internal/webidl.js | 10 +-- src/node_util.cc | 19 ++++ .../test-internal-webidl-buffer-source.js | 88 ++++++++++++++++++- typings/internalBinding/util.d.ts | 1 + 4 files changed, 111 insertions(+), 7 deletions(-) diff --git a/lib/internal/webidl.js b/lib/internal/webidl.js index 128db9b16c74..ee0388b70e97 100644 --- a/lib/internal/webidl.js +++ b/lib/internal/webidl.js @@ -37,6 +37,8 @@ const { isTypedArray, } = require('internal/util/types'); +const { getSharedArrayBufferGrowable } = internalBinding('util'); + const BIGINT_2_63 = 1n << 63n; const BIGINT_2_64 = 1n << 64n; @@ -946,12 +948,8 @@ function validateBufferSourceBacking(buffer, options) { function validateAllowGrowableSharedArrayBuffer(buffer, options) { // SharedArrayBuffer and ArrayBufferView conversion step 3: // IsFixedLengthArrayBuffer(buffer) must be true without [AllowResizable]. - // Do not use a primordial getter here. When this module is included in the - // startup snapshot, an early-captured SharedArrayBuffer.prototype.growable - // getter does not detect growable buffers created after deserialization. - // Lazily capturing the getter would work, but it would observe the runtime - // prototype at first comparison, so it would not be an actual primordial. - if (!options.allowResizable && buffer.growable) { + if (!options.allowResizable && + FunctionPrototypeCall(getSharedArrayBufferGrowable, buffer)) { throw makeException( 'is backed by a growable SharedArrayBuffer, which is not allowed.', options); diff --git a/src/node_util.cc b/src/node_util.cc index 6d3373caae6c..578c2bd1153a 100644 --- a/src/node_util.cc +++ b/src/node_util.cc @@ -497,6 +497,25 @@ void Initialize(Local target, Environment* env = Environment::GetCurrent(context); Isolate* isolate = env->isolate(); + { + const Local prototype = + SharedArrayBuffer::New(isolate, 0)->GetPrototypeV2().As(); + const Local descriptor = + prototype + ->GetOwnPropertyDescriptor( + context, FIXED_ONE_BYTE_STRING(isolate, "growable")) + .ToLocalChecked() + .As(); + const Local getter = + descriptor->Get(context, env->get_string()).ToLocalChecked(); + CHECK(getter->IsFunction()); + target + ->Set(context, + FIXED_ONE_BYTE_STRING(isolate, "getSharedArrayBufferGrowable"), + getter) + .Check(); + } + { Local tmpl = ObjectTemplate::New(isolate); #define V(PropertyName, _) \ diff --git a/test/parallel/test-internal-webidl-buffer-source.js b/test/parallel/test-internal-webidl-buffer-source.js index 9e522d7d7b8a..8e81f42a9456 100644 --- a/test/parallel/test-internal-webidl-buffer-source.js +++ b/test/parallel/test-internal-webidl-buffer-source.js @@ -1,7 +1,7 @@ // Flags: --expose-internals 'use strict'; -require('../common'); +const common = require('../common'); const assert = require('assert'); const { test } = require('node:test'); const vm = require('vm'); @@ -272,6 +272,92 @@ test('AllowSharedBufferSource handles growable shared buffers with explicit ' + } }); +test('Shared buffer growability checks do not read JavaScript properties', () => { + for (const [buffer, growable] of [ + [new SharedArrayBuffer(8), false], + [new SharedArrayBuffer(8, { maxByteLength: 8 }), true], + [new SharedArrayBuffer(8, { maxByteLength: 16 }), true], + [vm.runInNewContext('new SharedArrayBuffer(8)'), false], + [vm.runInNewContext('new SharedArrayBuffer(8, { maxByteLength: 16 })'), true], + ]) { + const view = new Uint8Array(buffer); + const dataView = new DataView(buffer); + for (const mode of ['shadow', 'getter', 'prototype']) { + if (mode === 'shadow') { + Object.defineProperty(buffer, 'growable', { + value: !growable, + configurable: true, + }); + } else if (mode === 'getter') { + Object.defineProperty(buffer, 'growable', { + get: common.mustNotCall('Unexpected growable getter'), + configurable: true, + }); + } else { + delete buffer.growable; + Object.setPrototypeOf(buffer, null); + } + + for (const value of [buffer, view, dataView]) { + if (growable) { + assert.throws(() => converters.AllowSharedBufferSource(value), { + code: 'ERR_INVALID_ARG_TYPE', + }); + } else { + assert.strictEqual(converters.AllowSharedBufferSource(value), value); + } + assert.strictEqual(converters.AllowSharedBufferSource(value, { + allowResizable: true, + }), value); + } + + if (growable) { + assert.throws(() => converters.Uint8Array(view, { allowShared: true }), { + code: 'ERR_INVALID_ARG_TYPE', + }); + } else { + assert.strictEqual(converters.Uint8Array(view, { allowShared: true }), view); + } + assert.strictEqual(converters.Uint8Array(view, { + allowShared: true, + allowResizable: true, + }), view); + } + } +}); + +test('Shared WebAssembly buffer growability is checked per buffer', { + skip: typeof WebAssembly === 'undefined', +}, () => { + const memory = new WebAssembly.Memory({ initial: 1, maximum: 2, shared: true }); + for (const [buffer, growable] of [ + [memory.buffer, false], + [memory.toResizableBuffer(), true], + [memory.toFixedLengthBuffer(), false], + ]) { + for (const value of [buffer, new Uint8Array(buffer), new DataView(buffer)]) { + if (growable) { + assert.throws(() => converters.AllowSharedBufferSource(value), { + code: 'ERR_INVALID_ARG_TYPE', + }); + } else { + assert.strictEqual(converters.AllowSharedBufferSource(value), value); + } + assert.strictEqual(converters.AllowSharedBufferSource(value, { + allowResizable: true, + }), value); + } + const view = new Uint8Array(buffer); + if (growable) { + assert.throws(() => converters.Uint8Array(view, { allowShared: true }), { + code: 'ERR_INVALID_ARG_TYPE', + }); + } else { + assert.strictEqual(converters.Uint8Array(view, { allowShared: true }), view); + } + } +}); + test('BufferSource rejects objects with a forged @@toStringTag', () => { const fake = { [Symbol.toStringTag]: 'Uint8Array' }; assert.throws( diff --git a/typings/internalBinding/util.d.ts b/typings/internalBinding/util.d.ts index a3026b5a0305..4c3eeca8ed82 100644 --- a/typings/internalBinding/util.d.ts +++ b/typings/internalBinding/util.d.ts @@ -47,6 +47,7 @@ export interface UtilBinding { styleText(format: Array | string, text: string): string; isInsideNodeModules(frameLimit?: number): boolean; constructSharedArrayBuffer(length?: number): SharedArrayBuffer; + getSharedArrayBufferGrowable(this: SharedArrayBuffer): boolean; constants: { kPending: 0; From 9a3697c5c77efb8a7cff3155ba7f9e61887bde08 Mon Sep 17 00:00:00 2001 From: Christopher Buss Date: Wed, 9 Sep 2026 21:24:46 +0100 Subject: [PATCH 048/217] doc: document windowsHide for child_process.fork Describe the existing windowsHide option and its false default alongside the other fork options, using the wording already documented for spawn. Refs: https://github.com/DefinitelyTyped/DefinitelyTyped/pull/75525 Signed-off-by: christopher-buss Assisted-by: Codex Co-authored-by: Codex PR-URL: https://github.com/nodejs/node/pull/65887 Reviewed-By: Xuguang Mei Reviewed-By: Stefan Stojanovic Reviewed-By: Luigi Pinca --- doc/api/child_process.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/api/child_process.md b/doc/api/child_process.md index bdf7d1f98b40..e06f88af3a5f 100644 --- a/doc/api/child_process.md +++ b/doc/api/child_process.md @@ -561,6 +561,8 @@ changes: is used, it must contain exactly one item with value `'ipc'` or an error will be thrown. For instance `[0, 1, 2, 'ipc']`. * `uid` {number} Sets the user identity of the process (see setuid(2)). + * `windowsHide` {boolean} Hide the subprocess console window that would + normally be created on Windows systems. **Default:** `false`. * `windowsVerbatimArguments` {boolean} No quoting or escaping of arguments is done on Windows. Ignored on Unix. **Default:** `false`. * `timeout` {number} In milliseconds the maximum amount of time the process From b9b0e339231354de4fa5d6c6f14124659396324c Mon Sep 17 00:00:00 2001 From: Donghoon Kang Date: Thu, 10 Sep 2026 07:54:13 +0900 Subject: [PATCH 049/217] lib: avoid unsafe array iteration in cli table Avoid unsafe array iteration because for...of relies on the user-mutable Symbol.iterator. Use an index-based loop instead. Signed-off-by: HoonDongKang PR-URL: https://github.com/nodejs/node/pull/65838 Reviewed-By: James M Snell Reviewed-By: Daeyeon Jeong Reviewed-By: Jordan Harband --- lib/internal/cli_table.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/internal/cli_table.js b/lib/internal/cli_table.js index be0bd44bbc36..13b3c86c5e40 100644 --- a/lib/internal/cli_table.js +++ b/lib/internal/cli_table.js @@ -81,8 +81,10 @@ const table = (head, columns) => { ArrayPrototypeJoin(divider, tableChars.rowMiddle) + tableChars.rightMiddle + '\n'; - for (const row of rows) + for (let i = 0; i < rows.length; i++) { + const row = rows[i]; result += `${renderRow(row, columnWidths)}\n`; + } result += tableChars.bottomLeft + ArrayPrototypeJoin(divider, tableChars.bottomMiddle) + From 6f5fb76d0e3e79570fd52ea37b1b7ee0db9e0767 Mon Sep 17 00:00:00 2001 From: Trivikram Kamat <16024985+trivikr@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:03:34 -0700 Subject: [PATCH 050/217] sqlite: track registered user-defined functions Track scalar and aggregate/window functions on each DatabaseSync instance. Remove registrations through SQLite destruction callbacks and clear tracking when the database closes. Signed-off-by: Kamat, Trivikram <16024985+trivikr@users.noreply.github.com> Assisted-by: codex PR-URL: https://github.com/nodejs/node/pull/65896 Fixes: https://github.com/nodejs/node/issues/65880 Reviewed-By: James M Snell Reviewed-By: Xuguang Mei --- src/node_sqlite.cc | 20 +++++++++++++++++--- src/node_sqlite.h | 9 +++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/node_sqlite.cc b/src/node_sqlite.cc index c3c97844e867..57d1da28f076 100644 --- a/src/node_sqlite.cc +++ b/src/node_sqlite.cc @@ -404,7 +404,13 @@ class CustomAggregate { start_(env->isolate(), start), step_fn_(env->isolate(), step_fn), inverse_fn_(env->isolate(), inverse_fn), - result_fn_(env->isolate(), result_fn) {} + result_fn_(env->isolate(), result_fn) { + db_->user_defined_functions_.insert(this); + } + + ~CustomAggregate() { + if (db_) db_->user_defined_functions_.erase(this); + } static void xStep(sqlite3_context* ctx, int argc, sqlite3_value** argv) { xStepBase(ctx, argc, argv, &CustomAggregate::step_fn_); @@ -772,9 +778,13 @@ UserDefinedFunction::UserDefinedFunction(Environment* env, : env_(env), fn_(env->isolate(), fn), db_(std::move(db)), - use_bigint_args_(use_bigint_args) {} + use_bigint_args_(use_bigint_args) { + db_->user_defined_functions_.insert(this); +} -UserDefinedFunction::~UserDefinedFunction() {} +UserDefinedFunction::~UserDefinedFunction() { + if (db_) db_->user_defined_functions_.erase(this); +} void UserDefinedFunction::xFunc(sqlite3_context* ctx, int argc, @@ -1071,6 +1081,8 @@ DatabaseSync::~DatabaseSync() { } void DatabaseSync::MemoryInfo(MemoryTracker* tracker) const { + tracker->TrackFieldWithSize("user_defined_functions", + user_defined_functions_.size() * sizeof(void*)); // TODO(tniessen): more accurately track the size of all fields tracker->TrackFieldWithSize( "open_config", sizeof(open_config_), "DatabaseOpenConfiguration"); @@ -1615,6 +1627,8 @@ void DatabaseSync::Close(const FunctionCallbackInfo& args) { int r = sqlite3_close_v2(db->connection_.get()); CHECK_ERROR_OR_THROW(env->isolate(), db, r, SQLITE_OK, void()); db->connection_.release(); + // Backups can defer SQLite destruction until after the connection is closed. + db->user_defined_functions_.clear(); } void DatabaseSync::Dispose(const v8::FunctionCallbackInfo& args) { diff --git a/src/node_sqlite.h b/src/node_sqlite.h index 306a47f6c47f..5b91e27d5736 100644 --- a/src/node_sqlite.h +++ b/src/node_sqlite.h @@ -266,6 +266,10 @@ class DatabaseSync : public BaseObject { void FinalizeBackups(); void UntrackStatement(StatementSync* statement); bool IsOpen(); + // SQL functions are one of several paths by which SQLite can invoke JS. + size_t GetUserDefinedFunctionCount() const { + return user_defined_functions_.size(); + } bool use_big_ints() const { return open_config_.get_use_big_ints(); } bool return_arrays() const { return open_config_.get_return_arrays(); } bool allow_bare_named_params() const { @@ -340,11 +344,16 @@ class DatabaseSync : public BaseObject { int trace_suppression_depth_ = 0; std::vector stepping_statements_; + // SQLite owns these scalar and aggregate/window function registrations. Its + // destroy callbacks untrack replaced functions and failed registrations. + std::unordered_set user_defined_functions_; std::set backups_; std::unordered_set sessions_; std::unordered_set statements_; BaseObjectPtr trace_channel_; + friend class UserDefinedFunction; + friend class CustomAggregate; friend class DatabaseSyncLimits; friend class Session; friend class SQLTagStore; From f245e53b29f99baa1845233d20e0e6796ad1f9a4 Mon Sep 17 00:00:00 2001 From: Colin McDonnell <3084745+colinhacks@users.noreply.github.com> Date: Wed, 9 Sep 2026 22:11:24 -0700 Subject: [PATCH 051/217] src: seed V8 from the OS CSPRNG instead of OpenSSL's DRBG InitializeOncePerProcessInternal() calls CSPRNG(nullptr, 0) to confirm OpenSSL's random source is seeded and installs a V8 entropy source that goes through CSPRNG() as well. The first RAND_status() of the process therefore runs before V8 starts, instantiates the DRBG, and with it constructs the default provider's algorithm and name tables (ossl_method_construct, ossl_namemap_stored): 3.7% of the samples of `node -e 0` on Linux x64, all of it before v8Start. V8 uses the entropy for hash seeds, address space layout randomization and Math.random(), none of which are cryptographic, so read the OS CSPRNG directly through uv_random(). AIX is the exception: uv_random() reads the blocking /dev/random there, so it stays on OpenSSL's DRBG, which seeds from /dev/urandom. Keep activating the default provider at startup, which the eager check did as a side effect and --openssl-legacy-provider depends on. Its explicit OSSL_PROVIDER_load() disables OpenSSL's provider fallback, so without a prior activation the default provider never loads. Run the seeding check itself only when that provider is unavailable or FIPS is in effect, the cases where an OpenSSL configuration from any source can leave the process without a DRBG and an early abort beats a hang at the first crypto call. Every crypto consumer stays on OpenSSL, and a system without a usable CSPRNG still aborts at startup, now from uv_random() failing. Two other behaviors change. A configuration whose [random] section names a DRBG that cannot be fetched used to abort at startup; it now starts and the first crypto call fails on the fetch. With --secure-heap the process DRBGs are instantiated after the secure heap exists, so they are allocated from it, and a Worker's isolate setup no longer aborts the process from the entropy callback when the heap cannot hold another per-thread DRBG. Tests cover both, and the default provider staying active under --openssl-legacy-provider. Measured on Linux x64 against an unpatched build of the same tree, both binaries interleaved, min of 300 runs: `node -e 0` 29.18 -> 27.82 ms, nodeStart to v8Start 2.91 -> 2.11 ms. RAND_status and the provider's table construction leave the startup profile (2.8% of samples before); the provider activation that remains is 0.05%. The first crypto.randomBytes() instantiates the DRBG in 0.19 ms. The `parallel`, `sequential`, `message`, `es-module` and `addons` suites show no failure the unpatched build does not have. Refs: https://github.com/nodejs/node/commit/5cc36c39d2 Refs: https://github.com/nodejs/node/pull/44493 Refs: https://github.com/nodejs/node/pull/46237 Signed-off-by: Colin McDonnell <3084745+colinhacks@users.noreply.github.com> PR-URL: https://github.com/nodejs/node/pull/65796 Reviewed-By: Filip Skokan Reviewed-By: James M Snell --- src/node.cc | 36 +++++++++++++++---- .../test-legacy-provider-option.js | 3 ++ .../openssl3-conf/random_unavailable.cnf | 7 ++++ test/parallel/test-crypto-no-algorithm.js | 18 ++++++++++ test/parallel/test-crypto-secure-heap.js | 35 ++++++++++++++++++ 5 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 test/fixtures/openssl3-conf/random_unavailable.cnf diff --git a/src/node.cc b/src/node.cc index fcf572d71e06..959e6552cb7e 100644 --- a/src/node.cc +++ b/src/node.cc @@ -49,6 +49,9 @@ #if HAVE_OPENSSL #include "ncrypto.h" +#if OPENSSL_VERSION_MAJOR >= 3 +#include +#endif #include "node_crypto.h" #if OPENSSL_VERSION_MAJOR >= 3 && !defined(CONF_MFLAGS_IGNORE_MISSING_FILE) // OpenSSL hides this deprecated macro under OPENSSL_NO_DEPRECATED, but the @@ -1287,15 +1290,36 @@ InitializeOncePerProcessInternal(const std::vector& args, } crypto::InstallFipsIndicatorCallback(); - // Ensure CSPRNG is properly seeded. - CHECK(ncrypto::CSPRNG(nullptr, 0)); + // Activating the default provider here keeps --openssl-legacy-provider + // working. Its explicit load disables OpenSSL's fallback, and the eager + // CSPRNG check used to activate the provider as a side effect. Only + // check the seeding when that provider is missing or FIPS is on, so a + // configuration without a DRBG still aborts at startup instead of + // hanging at the first crypto call. Otherwise the DRBG is instantiated + // on first use. +#if OPENSSL_VERSION_MAJOR >= 3 + const bool check_csprng = ncrypto::isFipsEnabled() || + !OSSL_PROVIDER_available(nullptr, "default"); +#else + const bool check_csprng = true; +#endif + if (check_csprng) { + CHECK(ncrypto::CSPRNG(nullptr, 0)); + } + // V8 uses the entropy for hash seeds, ASLR and Math.random(), none of + // it cryptographic. Going through OpenSSL would instantiate the DRBG + // and build the default provider's algorithm tables on every startup. + // V8 falls back to very weak entropy when the source fails, so abort + // instead. V8::SetEntropySource([](unsigned char* buffer, size_t length) { - // V8 falls back to very weak entropy when this function fails - // and /dev/urandom isn't available. That wouldn't be so bad if - // the entropy was only used for Math.random() but it's also used for - // hash table and address space layout randomization. Better to abort. +#ifdef _AIX + // uv_random() reads /dev/random on AIX, which blocks. OpenSSL seeds + // from /dev/urandom there. CHECK(ncrypto::CSPRNG(buffer, length)); +#else + CHECK_EQ(uv_random(nullptr, nullptr, buffer, length, 0, nullptr), 0); +#endif return true; }); #endif // !defined(OPENSSL_IS_BORINGSSL) diff --git a/test/addons/openssl-providers/test-legacy-provider-option.js b/test/addons/openssl-providers/test-legacy-provider-option.js index 5ad60dac9b86..1f01ce55a8f2 100644 --- a/test/addons/openssl-providers/test-legacy-provider-option.js +++ b/test/addons/openssl-providers/test-legacy-provider-option.js @@ -22,3 +22,6 @@ if (getFips()) { common.skip('this test cannot be run in FIPS mode'); } providers.testProviderPresent('legacy'); +// The explicit legacy load disables OpenSSL's provider fallback, so the +// default provider has to be active before it runs. +providers.testProviderPresent('default'); diff --git a/test/fixtures/openssl3-conf/random_unavailable.cnf b/test/fixtures/openssl3-conf/random_unavailable.cnf new file mode 100644 index 000000000000..a2dc8d2c9ffa --- /dev/null +++ b/test/fixtures/openssl3-conf/random_unavailable.cnf @@ -0,0 +1,7 @@ +nodejs_conf = nodejs_init + +[nodejs_init] +random = random_sect + +[random_sect] +random = NO-SUCH-DRBG diff --git a/test/parallel/test-crypto-no-algorithm.js b/test/parallel/test-crypto-no-algorithm.js index 90d19ff97fcb..2b5851a1d8c5 100644 --- a/test/parallel/test-crypto-no-algorithm.js +++ b/test/parallel/test-crypto-no-algorithm.js @@ -57,3 +57,21 @@ if (isMainThread) { assert(common.nodeProcessAborted(cp.status, cp.signal), `process did not abort, code:${cp.status} signal:${cp.signal}`); } + +// AIX keeps OpenSSL as V8's entropy source, so a DRBG that cannot be +// fetched still aborts at startup there. +if (!common.isAIX) { + // A configuration whose random section names a DRBG that cannot be + // fetched starts normally; the first crypto call fails, without a hang. + const fixtures = require('../common/fixtures'); + const { spawnSync } = require('node:child_process'); + const randomConf = fixtures.path('openssl3-conf', 'random_unavailable.cnf'); + const cp = spawnSync(process.execPath, + [ `--openssl-config=${randomConf}`, '-e', + 'require("node:crypto").randomBytes(8)' ], + { encoding: 'utf8' }); + assert(!common.nodeProcessAborted(cp.status, cp.signal), + `process aborted, code:${cp.status} signal:${cp.signal}`); + assert.strictEqual(cp.status, 1); + assert.match(cp.stderr, /unable to fetch drbg/); +} diff --git a/test/parallel/test-crypto-secure-heap.js b/test/parallel/test-crypto-secure-heap.js index 8bd93c5281da..638ab49c82ca 100644 --- a/test/parallel/test-crypto-secure-heap.js +++ b/test/parallel/test-crypto-secure-heap.js @@ -61,6 +61,28 @@ if (process.argv[2] === 'child') { return; } +if (process.argv[2] === 'workers') { + // Eight Workers held alive at once. A 1 KiB secure heap has room for a + // few DRBGs only, so an isolate setup that drew its entropy through + // OpenSSL would fail for the later Workers and abort the process. + const { Worker } = require('worker_threads'); + const i32 = new Int32Array(new SharedArrayBuffer(4)); + let online = 0; + for (let i = 0; i < 8; i++) { + const worker = new Worker( + 'const { workerData } = require("worker_threads");' + + 'Atomics.wait(workerData.i32, 0, 0);', + { eval: true, workerData: { i32 } }); + worker.on('online', () => { + if (++online === 8) { + Atomics.store(i32, 0, 1); + Atomics.notify(i32, 0); + } + }); + } + return; +} + const child = fork( process.argv[1], ['child'], @@ -70,6 +92,19 @@ child.on('exit', common.mustCall((code) => { assert.strictEqual(code, 0); })); +// AIX keeps OpenSSL as V8's entropy source, so a Worker's isolate setup +// still draws on the secure heap there. +if (!common.isAIX) { + const child = fork( + process.argv[1], + ['workers'], + { execArgv: ['--secure-heap=1024', '--secure-heap-min=4'] }); + child.on('exit', common.mustCall((code, signal) => { + assert.strictEqual(signal, null); + assert.strictEqual(code, 0); + })); +} + { const child = fork(fixtures.path('a.js'), { execArgv: ['--secure-heap=3', '--secure-heap-min=3'], From 89bae46e68bfaa94666f628ea69282bf897b3f89 Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Thu, 10 Sep 2026 09:21:44 +0200 Subject: [PATCH 052/217] buffer: fix unaligned UTF-16LE decoding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For odd-length input, the destination only has room for complete code units. Copy those units and ignore the trailing byte. This matches the aligned and big-endian paths. Assisted-by: pi Signed-off-by: Matteo Collina PR-URL: https://github.com/nodejs/node/pull/65905 Reviewed-By: Gürgün Dayıoğlu Reviewed-By: Filip Skokan Reviewed-By: Robert Nagy --- src/string_bytes.cc | 4 ++-- test/parallel/test-buffer-tostring.js | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/string_bytes.cc b/src/string_bytes.cc index 6e628b907e66..94b579b6984d 100644 --- a/src/string_bytes.cc +++ b/src/string_bytes.cc @@ -751,8 +751,8 @@ MaybeLocal StringBytes::Encode(Isolate* isolate, } if (reinterpret_cast(buf) % 2 != 0) { return EncodeTwoByteString( - isolate, str_len, [buf, buflen](uint16_t* dst) { - memcpy(dst, buf, buflen); + isolate, str_len, [buf, str_len](uint16_t* dst) { + memcpy(dst, buf, str_len * sizeof(*dst)); }); } return ExternTwoByteString::NewFromCopy( diff --git a/test/parallel/test-buffer-tostring.js b/test/parallel/test-buffer-tostring.js index a3dad0146d75..676d2f85f567 100644 --- a/test/parallel/test-buffer-tostring.js +++ b/test/parallel/test-buffer-tostring.js @@ -9,6 +9,13 @@ for (const encoding of ['utf8', 'utf-8', 'ucs2', 'ucs-2', 'ascii', 'latin1', assert.strictEqual(Buffer.from('foo', encoding).toString(encoding), 'foo'); } +// Ignore an incomplete trailing code unit when decoding unaligned UTF-16LE. +for (const size of [514, 516]) { + const buffer = Buffer.alloc(size, 0x61); + assert.strictEqual(buffer.toString('utf16le', 1), + '\u6161'.repeat((size - 1) >>> 1)); +} + // base64 ['base64', 'BASE64'].forEach((encoding) => { assert.strictEqual(Buffer.from('Zm9v', encoding).toString(encoding), 'Zm9v'); From 8d1ce4d477a74c33199a013e626306410bac4f19 Mon Sep 17 00:00:00 2001 From: Yuya Inoue <65857152+inoway46@users.noreply.github.com> Date: Thu, 10 Sep 2026 22:34:57 +0900 Subject: [PATCH 053/217] doc: add inoway46 as triager Signed-off-by: inoway46 PR-URL: https://github.com/nodejs/node/pull/65565 Reviewed-By: Luigi Pinca Reviewed-By: Daeyeon Jeong Reviewed-By: Trivikram Kamat Reviewed-By: Filip Skokan Reviewed-By: Darshan Sen --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 0376c0e9814c..24a76d872f2a 100644 --- a/README.md +++ b/README.md @@ -761,6 +761,8 @@ maintaining the Node.js project. **Wiyeong Seo** <> * [iam-frankqiu](https://github.com/iam-frankqiu) - **Frank Qiu** <> (he/him) +* [inoway46](https://github.com/inoway46) - + **Yuya Inoue** <> (he/him) * [milesguicent](https://github.com/milesguicent) - **Miles Guicent** <> (he/him) * [preveen-stack](https://github.com/preveen-stack) - From 472215ff9b0beb8d8c3c4ee8d53a9a2a0b2282c8 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Thu, 10 Sep 2026 08:19:13 -0700 Subject: [PATCH 054/217] test: try fixing windows build replacing WMIC Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/65949 Reviewed-By: Filip Skokan Reviewed-By: Stefan Stojanovic --- test/common/child_process.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/common/child_process.js b/test/common/child_process.js index c74154bb084f..59505e5824dd 100644 --- a/test/common/child_process.js +++ b/test/common/child_process.js @@ -15,12 +15,12 @@ function cleanupStaleProcess(filename) { process.once('beforeExit', () => { const basename = filename.replace(/.*[/\\]/g, ''); try { - execFileSync(`${process.env.SystemRoot}\\System32\\wbem\\WMIC.exe`, [ - 'process', - 'where', - `commandline like '%${basename}%child'`, - 'delete', - '/nointeractive', + execFileSync(`${process.env.SystemRoot}\\System32\\WindowsPowerShell\\v1.0\\powershell.exe`, [ + '-NoProfile', + '-NonInteractive', + '-Command', + `Get-CimInstance Win32_Process -Filter "CommandLine LIKE '%${basename}%child'" | ` + + 'ForEach-Object { Stop-Process -Id $_.ProcessId -Force }', ]); } catch { // Ignore failures, there might not be any stale process to clean up. From 81eba460b5e17d4ded27ab7a2e993ff570bda1e6 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Thu, 10 Sep 2026 17:19:25 +0200 Subject: [PATCH 055/217] tools: fix commit queue error summary matching Recognize the error symbol emitted by core-validate-commit so validation failures appear outside the collapsed output. Signed-off-by: Filip Skokan PR-URL: https://github.com/nodejs/node/pull/65913 Reviewed-By: James M Snell Reviewed-By: Luigi Pinca Reviewed-By: Xuguang Mei Reviewed-By: Trivikram Kamat --- tools/actions/commit-queue.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/actions/commit-queue.sh b/tools/actions/commit-queue.sh index e260773d14a5..0c3c70da89f0 100755 --- a/tools/actions/commit-queue.sh +++ b/tools/actions/commit-queue.sh @@ -72,7 +72,7 @@ commit_queue_failed() { Add https://github.com/nodejs/node/labels/commit-queue-squash to land it as one commit, or https://github.com/nodejs/node/labels/commit-queue-rebase to land the commits separately.' else if [ -z "$reported_failure" ]; then - reported_failure=$(grep -e '✘' -e '⚠' output | tail -n 10) + reported_failure=$(grep -e '✘' -e '✖' -e '⚠' output | tail -n 10) fi if [ -z "$reported_failure" ]; then reported_failure=$(tail -n 10 output) From 28acafe6a007ab14f6d6827128dd7588f949a94a Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Thu, 10 Sep 2026 20:46:24 +0200 Subject: [PATCH 056/217] lib: avoid repeat internal receiver checks Let internal helpers access the private cache directly to avoid repeating receiver checks. Signed-off-by: Filip Skokan PR-URL: https://github.com/nodejs/node/pull/65910 Refs: https://github.com/nodejs/node/pull/65846 Reviewed-By: Xuguang Mei Reviewed-By: Yagiz Nizipli --- lib/internal/crypto/keys.js | 19 +++- .../test-webcrypto-cryptokey-brand-check.js | 104 +++++++++--------- ...test-webcrypto-cryptokey-clone-transfer.js | 12 ++ 3 files changed, 78 insertions(+), 57 deletions(-) diff --git a/lib/internal/crypto/keys.js b/lib/internal/crypto/keys.js index e82e48ad653f..44aece77d3eb 100644 --- a/lib/internal/crypto/keys.js +++ b/lib/internal/crypto/keys.js @@ -1144,6 +1144,9 @@ const { if (depth < 0) return this; + if (!isCryptoKey(this)) + throw new ERR_INVALID_THIS('CryptoKey'); + const opts = { ...options, depth: options.depth == null ? null : options.depth - 1, @@ -1158,14 +1161,20 @@ const { } get type() { + if (!isCryptoKey(this)) + throw new ERR_INVALID_THIS('CryptoKey'); return getCryptoKeyType(this); } get extractable() { + if (!isCryptoKey(this)) + throw new ERR_INVALID_THIS('CryptoKey'); return getCryptoKeyExtractable(this); } get algorithm() { + if (!isCryptoKey(this)) + throw new ERR_INVALID_THIS('CryptoKey'); const slots = getSlots(this); let cached = slots[kSlotClonedAlgorithm]; if (cached === undefined) { @@ -1176,6 +1185,8 @@ const { } get usages() { + if (!isCryptoKey(this)) + throw new ERR_INVALID_THIS('CryptoKey'); const slots = getSlots(this); let cached = slots[kSlotClonedUsages]; if (cached === undefined) { @@ -1210,12 +1221,8 @@ const { return #slots in key || isNativeCryptoKey(key); }; getSlots = (key) => { - if (!key || typeof key !== 'object') - throw new ERR_INVALID_THIS('CryptoKey'); - if (#slots in key) { - const cached = key.#slots; - if (cached !== undefined) return cached; - } + const cached = key.#slots; + if (cached !== undefined) return cached; const slots = nativeGetCryptoKeySlots(key); slots[kSlotAlgorithm] = cloneInternalAlgorithm(slots[kSlotAlgorithm]); key.#slots = slots; diff --git a/test/parallel/test-webcrypto-cryptokey-brand-check.js b/test/parallel/test-webcrypto-cryptokey-brand-check.js index 9dd115f00721..9174aebe8f05 100644 --- a/test/parallel/test-webcrypto-cryptokey-brand-check.js +++ b/test/parallel/test-webcrypto-cryptokey-brand-check.js @@ -1,13 +1,10 @@ 'use strict'; -// The four CryptoKey prototype getters (`type`, `extractable`, -// `algorithm`, `usages`) are user-configurable per Web IDL, so they -// can be invoked with an arbitrary `this`. The native callbacks that -// implement them must brand-check their receiver and throw cleanly -// (ERR_INVALID_THIS) rather than crashing the process or returning -// garbage. This test exercises four progressively more hostile -// receiver shapes, including subverting `instanceof` via -// `Symbol.hasInstance`, to make sure the C++ brand check holds. +// CryptoKey prototype getters and methods can be invoked with an +// arbitrary `this`. They must brand-check their receiver and throw +// cleanly (ERR_INVALID_THIS) rather than crashing the process or +// returning garbage. This test exercises invalid receiver shapes, +// including subverting `instanceof` via `Symbol.hasInstance`. // // It also verifies that `util.types.isCryptoKey()` cannot be fooled // by prototype spoofing. @@ -17,7 +14,7 @@ if (!common.hasCrypto) common.skip('missing crypto'); const assert = require('node:assert'); -const { types: { isCryptoKey } } = require('node:util'); +const { inspect, types: { isCryptoKey } } = require('node:util'); const { subtle } = globalThis.crypto; (async () => { @@ -29,22 +26,16 @@ const { subtle } = globalThis.crypto; const CryptoKey = key.constructor; - // Capture the underlying prototype getters once, so that subsequent + // Capture the underlying prototype members once, so that subsequent // tampering with `CryptoKey.prototype` cannot affect what we call. - const getters = { - type: Object.getOwnPropertyDescriptor(CryptoKey.prototype, 'type').get, - extractable: - Object.getOwnPropertyDescriptor(CryptoKey.prototype, 'extractable').get, - algorithm: - Object.getOwnPropertyDescriptor(CryptoKey.prototype, 'algorithm').get, - usages: - Object.getOwnPropertyDescriptor(CryptoKey.prototype, 'usages').get, - }; + const descriptors = Object.getOwnPropertyDescriptors(CryptoKey.prototype); // Sanity: each getter works on a real CryptoKey. - Object.entries(getters).forEach(([name, getter]) => { - assert.notStrictEqual(getter.call(key), undefined, `baseline ${name}`); - }); + for (const name of Reflect.ownKeys(descriptors)) { + const { get } = descriptors[name]; + if (get !== undefined) + Reflect.apply(get, key, []); + } assert.strictEqual(isCryptoKey(key), true); assert.strictEqual(Object.hasOwn(CryptoKey, 'getSlots'), false); const internalProto = Object.getPrototypeOf(key); @@ -56,36 +47,51 @@ const { subtle } = globalThis.crypto; const invalidThis = { code: 'ERR_INVALID_THIS', name: 'TypeError' }; const invalidArgType = { code: 'ERR_INVALID_ARG_TYPE', name: 'TypeError' }; + async function assertInvalidReceiver(receiver) { + for (const name of Reflect.ownKeys(descriptors)) { + if (name === 'constructor') continue; + const descriptor = descriptors[name]; + const args = name === inspect.custom ? [0, {}] : []; + for (const kind of ['get', 'set', 'value']) { + const member = descriptor[kind]; + if (typeof member !== 'function') continue; + await assert.rejects( + async () => Reflect.apply(member, receiver, args), + invalidThis, + `CryptoKey.${String(name)} (${kind})`, + ); + } + } + } + // Plain object receiver. - Object.entries(getters).forEach(([, getter]) => { - assert.throws(() => getter.call({}), invalidThis); - }); + await assertInvalidReceiver({}); // Null-prototype object receiver. - Object.entries(getters).forEach(([, getter]) => { - assert.throws(() => getter.call({ __proto__: null }), invalidThis); - }); + await assertInvalidReceiver({ __proto__: null }); // Primitive receiver. - Object.entries(getters).forEach(([, getter]) => { - assert.throws(() => getter.call(1), invalidThis); - }); + await assertInvalidReceiver(1); // Null. - Object.entries(getters).forEach(([, getter]) => { - // eslint-disable-next-line no-useless-call - assert.throws(() => getter.call(null), invalidThis); - }); + await assertInvalidReceiver(null); // Undefined. - Object.entries(getters).forEach(([, getter]) => { - assert.throws(() => getter.call(), invalidThis); - }); + await assertInvalidReceiver(undefined); // Function - Object.entries(getters).forEach(([, getter]) => { - assert.throws(() => getter.call(function() {}), invalidThis); - }); + await assertInvalidReceiver(function() {}); + + const revoked = Proxy.revocable(key, {}); + revoked.revoke(); + for (const receiver of [ + { __proto__: CryptoKey.prototype }, + { __proto__: key }, + new Proxy(key, {}), + revoked.proxy, + ]) { + await assertInvalidReceiver(receiver); + } // Prototype spoofing with InternalCryptoKey.prototype must not pass // util.types.isCryptoKey(). @@ -111,9 +117,7 @@ const { subtle } = globalThis.crypto; const fake = { foo: 'bar' }; assert.strictEqual(fake instanceof CryptoKey, true); assert.strictEqual(isCryptoKey(fake), false); - Object.entries(getters).forEach(([, getter]) => { - assert.throws(() => getter.call(fake), invalidThis); - }); + await assertInvalidReceiver(fake); // Subverted `instanceof` plus a real BaseObject of a different // kind (a Buffer) as the receiver. Without the C++ tag check @@ -121,13 +125,11 @@ const { subtle } = globalThis.crypto; const buf = Buffer.alloc(16); assert.strictEqual(buf instanceof CryptoKey, true); assert.strictEqual(isCryptoKey(buf), false); - Object.entries(getters).forEach(([, getter]) => { - assert.throws(() => getter.call(buf), invalidThis); - }); + await assertInvalidReceiver(buf); // The real CryptoKey continues to work after all of the above. - assert.strictEqual(getters.type.call(key), 'secret'); - assert.strictEqual(getters.extractable.call(key), true); - assert.strictEqual(getters.algorithm.call(key).name, 'HMAC'); - assert.deepStrictEqual(getters.usages.call(key), ['sign']); + assert.strictEqual(descriptors.type.get.call(key), 'secret'); + assert.strictEqual(descriptors.extractable.get.call(key), true); + assert.strictEqual(descriptors.algorithm.get.call(key).name, 'HMAC'); + assert.deepStrictEqual(descriptors.usages.get.call(key), ['sign']); })().then(common.mustCall()); diff --git a/test/parallel/test-webcrypto-cryptokey-clone-transfer.js b/test/parallel/test-webcrypto-cryptokey-clone-transfer.js index 4983e1c0bda3..2567ac69454d 100644 --- a/test/parallel/test-webcrypto-cryptokey-clone-transfer.js +++ b/test/parallel/test-webcrypto-cryptokey-clone-transfer.js @@ -17,6 +17,7 @@ if (!common.hasCrypto) common.skip('missing crypto'); const assert = require('node:assert'); +const { KeyObject } = require('node:crypto'); const { inspect } = require('node:util'); const { once } = require('node:events'); const { Worker, MessageChannel } = require('node:worker_threads'); @@ -320,6 +321,17 @@ async function checkRsaPssTransferToWorker({ publicKey, privateKey }) { { name: 'AES-GCM', iv }, k, ciphertext); assert.deepStrictEqual(Buffer.from(decrypted), plaintext); } + + const bytes = new Uint8Array(await subtle.exportKey('raw', key)); + const nullPrototypeClone = structuredClone(key); + Object.setPrototypeOf(nullPrototypeClone, null); + assert.deepStrictEqual( + new Uint8Array(await subtle.exportKey('raw', nullPrototypeClone)), bytes); + const typeGetter = Object.getOwnPropertyDescriptor(key.constructor.prototype, 'type').get; + const customInspect = key[inspect.custom]; + assert.strictEqual(typeGetter.call(nullPrototypeClone), 'secret'); + assert.strictEqual(typeof customInspect.call(nullPrototypeClone, 0, {}), 'string'); + assert.deepStrictEqual(KeyObject.from(structuredClone(key)).export(), Buffer.from(bytes)); } // ECDSA keypair (public extractable, private non-extractable) From 775173a5db3cf5c915d29555e5b87796676e9187 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Mon, 7 Sep 2026 23:42:13 +0200 Subject: [PATCH 057/217] crypto: optimize private EC JWK import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avoid creating temporary EVP objects and repeating key validation while retaining the private scalar range and public/private consistency checks. Assisted-by: GitHub Copilot Signed-off-by: Filip Skokan PR-URL: https://github.com/nodejs/node/pull/65908 Reviewed-By: Yagiz Nizipli Reviewed-By: James M Snell Reviewed-By: Tobias Nießen --- deps/ncrypto/ncrypto.cc | 18 +++++ deps/ncrypto/ncrypto.h | 1 + src/crypto/crypto_ec.cc | 7 +- .../test-crypto-key-objects-ec-jwk-private.js | 79 +++++++++++++++++++ 4 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 test/parallel/test-crypto-key-objects-ec-jwk-private.js diff --git a/deps/ncrypto/ncrypto.cc b/deps/ncrypto/ncrypto.cc index 20f22e614525..acc4ec0bb712 100644 --- a/deps/ncrypto/ncrypto.cc +++ b/deps/ncrypto/ncrypto.cc @@ -5348,6 +5348,24 @@ bool ECPointPointer::mul(const EC_GROUP* group, const BIGNUM* priv_key) { // ============================================================================ +bool ECKeyPointer::checkPrivateKey() const { + const auto group = getGroup(); + const auto priv = getPrivateKey(); + const auto pub = getPublicKey(); + if (group == nullptr || priv == nullptr || pub == nullptr) return false; + + auto order = BignumPointer::New(); + if (!order || !EC_GROUP_get_order(group, order.get(), nullptr) || + BN_is_zero(priv) || BN_is_negative(priv) || + BN_cmp(priv, order.get()) >= 0) { + return false; + } + + auto expected = ECPointPointer::New(group); + return expected && expected.mul(group, priv) && + EC_POINT_cmp(group, expected.get(), pub, nullptr) == 0; +} + #if NCRYPTO_USE_LEGACY_KEY_TYPES ECKeyPointer::ECKeyPointer() : key_(nullptr) {} diff --git a/deps/ncrypto/ncrypto.h b/deps/ncrypto/ncrypto.h index 6b1edceed061..633c2c4202a3 100644 --- a/deps/ncrypto/ncrypto.h +++ b/deps/ncrypto/ncrypto.h @@ -1764,6 +1764,7 @@ class ECKeyPointer final { bool setPublicKeyRaw(const BignumPointer& x, const BignumPointer& y); bool generate(); bool checkKey() const; + bool checkPrivateKey() const; DataPointer computeSecret(const ECPointPointer& peer) const; const EC_GROUP* getGroup() const; diff --git a/src/crypto/crypto_ec.cc b/src/crypto/crypto_ec.cc index d6920c8b30e2..d9b88e264a7f 100644 --- a/src/crypto/crypto_ec.cc +++ b/src/crypto/crypto_ec.cc @@ -756,7 +756,7 @@ KeyObjectData ImportJWKEcKey(Environment* env, Local jwk) { return {}; } // Verify that the public point matches the private scalar (d*G == (x,y)). - if (!ec.checkKey()) { + if (!ec.checkPrivateKey()) { THROW_ERR_CRYPTO_INVALID_JWK(env, "Invalid JWK EC key"); return {}; } @@ -764,7 +764,10 @@ KeyObjectData ImportJWKEcKey(Environment* env, Local jwk) { auto pkey = EVPKeyPointer::New(); if (!pkey) return {}; - CHECK(pkey.set(ec)); + if (!pkey.set(ec)) { + THROW_ERR_CRYPTO_INVALID_JWK(env, "Invalid JWK EC key"); + return {}; + } return KeyObjectData::CreateAsymmetric(type, std::move(pkey)); } diff --git a/test/parallel/test-crypto-key-objects-ec-jwk-private.js b/test/parallel/test-crypto-key-objects-ec-jwk-private.js new file mode 100644 index 000000000000..bede4e2f31c0 --- /dev/null +++ b/test/parallel/test-crypto-key-objects-ec-jwk-private.js @@ -0,0 +1,79 @@ +'use strict'; + +const common = require('../common'); +if (!common.hasCrypto) + common.skip('missing crypto'); + +const assert = require('assert'); +const { + createECDH, + createPrivateKey, + createPublicKey, + getCurves, + getFips, + sign, + verify, +} = require('crypto'); + +const curves = [ + ['prime256v1', 'P-256', 32, + 'ffffffff00000000ffffffffffffffffbce6faada7179e84f3b9cac2fc632551'], + ['secp384r1', 'P-384', 48, + 'ffffffffffffffffffffffffffffffffffffffffffffffffc7634d81f4372ddf' + + '581a0db248b0a77aecec196accc52973'], + ['secp521r1', 'P-521', 66, + '01ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff' + + 'fa51868783bf2f966b7fcc0148f709a5d03bb5c9b8899c47aebb6fb71e91386409'], +]; +if (!getFips() && getCurves().includes('secp256k1')) { + curves.push(['secp256k1', 'secp256k1', 32, + 'fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141']); +} + +for (const [namedCurve, crv, width, orderHex] of curves) { + const order = BigInt(`0x${orderHex}`); + const encode = (scalar) => Buffer.from( + scalar.toString(16).padStart(width * 2, '0'), 'hex'); + const makeJwk = (scalar) => { + const ecdh = createECDH(namedCurve); + ecdh.setPrivateKey(encode(scalar)); + const point = ecdh.getPublicKey(); + return { + kty: 'EC', + crv, + x: point.subarray(1, 1 + width).toString('base64url'), + y: point.subarray(1 + width).toString('base64url'), + d: encode(scalar).toString('base64url'), + }; + }; + const generator = makeJwk(1n); + const other = makeJwk(2n); + const message = Buffer.from('EC JWK private key consistency'); + + for (const jwk of [generator, other, makeJwk(order - 1n)]) { + const key = createPrivateKey({ format: 'jwk', key: jwk }); + assert.deepStrictEqual(key.export({ format: 'jwk' }), jwk); + const publicJwk = { kty: jwk.kty, crv, x: jwk.x, y: jwk.y }; + const publicKey = createPublicKey({ format: 'jwk', key: publicJwk }); + assert(verify('sha256', message, publicKey, sign('sha256', message, key))); + } + + const invalid = [ + { ...generator, d: other.d }, + { ...generator, x: other.x, y: other.y }, + ...[0n, order, order + 1n].map((scalar) => ({ + ...generator, d: encode(scalar).toString('base64url'), + })), + { ...generator, d: '' }, + { + ...generator, + x: Buffer.alloc(width).toString('base64url'), + y: Buffer.alloc(width).toString('base64url'), + }, + ]; + for (const jwk of invalid) { + assert.throws(() => createPrivateKey({ format: 'jwk', key: jwk }), { + code: 'ERR_CRYPTO_INVALID_JWK', + }); + } +} From f1295e11db52c607b011a3895be74264c41a2bc1 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Tue, 8 Sep 2026 00:20:36 +0200 Subject: [PATCH 058/217] crypto: read EC curve metadata directly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avoid reconstructing EC keys for key details and TLS ephemeral-key curve reporting. Assisted-by: GitHub Copilot Signed-off-by: Filip Skokan PR-URL: https://github.com/nodejs/node/pull/65908 Reviewed-By: Yagiz Nizipli Reviewed-By: James M Snell Reviewed-By: Tobias Nießen --- benchmark/crypto/ec-key-details.js | 23 +++++++++++++++++++++++ deps/ncrypto/ncrypto.cc | 18 ++++++++++++++++++ deps/ncrypto/ncrypto.h | 1 + src/crypto/crypto_common.cc | 5 +---- src/crypto/crypto_ec.cc | 6 +----- 5 files changed, 44 insertions(+), 9 deletions(-) create mode 100644 benchmark/crypto/ec-key-details.js diff --git a/benchmark/crypto/ec-key-details.js b/benchmark/crypto/ec-key-details.js new file mode 100644 index 000000000000..e7266e9ad19b --- /dev/null +++ b/benchmark/crypto/ec-key-details.js @@ -0,0 +1,23 @@ +'use strict'; + +const common = require('../common.js'); +const { KeyObject } = require('crypto'); + +const bench = common.createBenchmark(main, { + namedCurve: ['P-256', 'P-384', 'P-521'], + type: ['public', 'private'], + n: [10000], +}); + +async function main({ namedCurve, type, n }) { + const pair = await crypto.subtle.generateKey({ + name: 'ECDSA', namedCurve, + }, true, ['sign', 'verify']); + const cryptoKey = pair[`${type}Key`]; + bench.start(); + for (let index = 0; index < n; index++) { + if (!KeyObject.from(cryptoKey).asymmetricKeyDetails.namedCurve) + throw new Error('Missing named curve'); + } + bench.end(n); +} diff --git a/deps/ncrypto/ncrypto.cc b/deps/ncrypto/ncrypto.cc index acc4ec0bb712..6309febf5a17 100644 --- a/deps/ncrypto/ncrypto.cc +++ b/deps/ncrypto/ncrypto.cc @@ -6933,6 +6933,24 @@ int Ec::getCurve() const { return EC_GROUP_get_curve_name(getGroup()); } +int Ec::GetCurveId(const EVPKeyPointer& key) { +#if NCRYPTO_USE_OPENSSL3_PROVIDER + char name[80]; + size_t length = 0; + if (EVP_PKEY_get_utf8_string_param( + key.get(), OSSL_PKEY_PARAM_GROUP_NAME, name, sizeof(name), &length) != + 1) { + return NID_undef; + } + return GetCurveIdFromName(name); +#else + const EC_KEY* ec = key; + if (ec == nullptr) return NID_undef; + const EC_GROUP* group = EC_KEY_get0_group(ec); + return group == nullptr ? NID_undef : EC_GROUP_get_curve_name(group); +#endif +} + int Ec::GetCurveIdFromName(const char* name) { int nid = EC_curve_nist2nid(name); if (nid == NID_undef) { diff --git a/deps/ncrypto/ncrypto.h b/deps/ncrypto/ncrypto.h index 633c2c4202a3..798d8a6130ca 100644 --- a/deps/ncrypto/ncrypto.h +++ b/deps/ncrypto/ncrypto.h @@ -812,6 +812,7 @@ class Ec final { #endif static int GetCurveIdFromName(const char* name); + static int GetCurveId(const EVPKeyPointer& key); using GetCurveCallback = std::function; static bool GetCurves(GetCurveCallback callback); diff --git a/src/crypto/crypto_common.cc b/src/crypto/crypto_common.cc index fde12953860c..b1b1c48e4cc7 100644 --- a/src/crypto/crypto_common.cc +++ b/src/crypto/crypto_common.cc @@ -28,7 +28,6 @@ namespace node { using ncrypto::ClearErrorOnReturn; -using ncrypto::ECKeyPointer; using ncrypto::EVPKeyPointer; using ncrypto::SSLPointer; using ncrypto::SSLSessionPointer; @@ -231,9 +230,7 @@ MaybeLocal GetEphemeralKey(Environment* env, const SSLPointer& ssl) { case EVP_PKEY_X448: { const char* curve_name; if (kid == EVP_PKEY_EC) { - ECKeyPointer ec(key); - if (!ec) break; - int nid = EC_GROUP_get_curve_name(ec.getGroup()); + int nid = ncrypto::Ec::GetCurveId(key); if (nid == NID_undef) break; curve_name = OBJ_nid2sn(nid); } else { diff --git a/src/crypto/crypto_ec.cc b/src/crypto/crypto_ec.cc index d9b88e264a7f..17f7999f22bd 100644 --- a/src/crypto/crypto_ec.cc +++ b/src/crypto/crypto_ec.cc @@ -779,11 +779,7 @@ bool GetEcKeyDetail(Environment* env, const auto& m_pkey = key.GetAsymmetricKey(); CHECK_EQ(m_pkey.id(), EVP_PKEY_EC); - ECKeyPointer ec(m_pkey); - if (!ec) return true; - - const auto group = ec.getGroup(); - int nid = EC_GROUP_get_curve_name(group); + int nid = Ec::GetCurveId(m_pkey); if (nid == NID_undef) return true; return target From 220a49961490c6a0ba5a2ea9cf5d32d606615c03 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Tue, 8 Sep 2026 00:27:41 +0200 Subject: [PATCH 059/217] crypto: export EC JWK coordinates directly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Query provider coordinates together instead of serializing and decoding the public point. Assisted-by: GitHub Copilot Signed-off-by: Filip Skokan PR-URL: https://github.com/nodejs/node/pull/65908 Reviewed-By: Yagiz Nizipli Reviewed-By: James M Snell Reviewed-By: Tobias Nießen --- benchmark/crypto/ec-jwk-export.js | 19 ++++++++++ deps/ncrypto/ncrypto.cc | 59 +++++++++++++++++++++++++++++++ deps/ncrypto/ncrypto.h | 5 +++ src/crypto/crypto_ec.cc | 36 +++++++------------ 4 files changed, 96 insertions(+), 23 deletions(-) create mode 100644 benchmark/crypto/ec-jwk-export.js diff --git a/benchmark/crypto/ec-jwk-export.js b/benchmark/crypto/ec-jwk-export.js new file mode 100644 index 000000000000..0539fc6468a0 --- /dev/null +++ b/benchmark/crypto/ec-jwk-export.js @@ -0,0 +1,19 @@ +'use strict'; + +const common = require('../common.js'); +const { generateKeyPairSync } = require('crypto'); + +const bench = common.createBenchmark(main, { + namedCurve: ['prime256v1', 'secp384r1', 'secp521r1', 'secp256k1'], + type: ['public', 'private'], + n: [10000], +}); + +function main({ namedCurve, type, n }) { + const key = generateKeyPairSync('ec', { namedCurve })[`${type}Key`]; + const options = { format: 'jwk' }; + bench.start(); + for (let index = 0; index < n; index++) + key.export(options); + bench.end(n); +} diff --git a/deps/ncrypto/ncrypto.cc b/deps/ncrypto/ncrypto.cc index 6309febf5a17..f7b2898d94a5 100644 --- a/deps/ncrypto/ncrypto.cc +++ b/deps/ncrypto/ncrypto.cc @@ -6933,6 +6933,65 @@ int Ec::getCurve() const { return EC_GROUP_get_curve_name(getGroup()); } +bool Ec::GetKeyComponents(const EVPKeyPointer& key, + BignumPointer* x, + BignumPointer* y, + BignumPointer* priv, + int* degree) { +#if NCRYPTO_USE_OPENSSL3_PROVIDER + const int nid = GetCurveId(key); + switch (nid) { + case NID_X9_62_prime256v1: + case NID_secp256k1: + *degree = 256; + break; + case NID_secp384r1: + *degree = 384; + break; + case NID_secp521r1: + *degree = 521; + break; + default: + *degree = 0; + } + if (*degree != 0) { + MarkPopErrorOnReturn pop_errors; + unsigned char x_bytes[66]{}; + unsigned char y_bytes[66]{}; + const size_t width = (*degree + 7) / 8; + OSSL_PARAM params[] = { + OSSL_PARAM_construct_BN(OSSL_PKEY_PARAM_EC_PUB_X, x_bytes, width), + OSSL_PARAM_construct_BN(OSSL_PKEY_PARAM_EC_PUB_Y, y_bytes, width), + OSSL_PARAM_construct_end(), + }; + if (EVP_PKEY_get_params(key.get(), params) == 1 && + OSSL_PARAM_modified(¶ms[0]) && OSSL_PARAM_modified(¶ms[1])) { + x->reset(BN_native2bn(x_bytes, width, nullptr)); + y->reset(BN_native2bn(y_bytes, width, nullptr)); + return *x && *y && + (priv == nullptr || + GetPKeyBnParam(key.get(), OSSL_PKEY_PARAM_PRIV_KEY, priv)); + } + } +#endif + ECKeyPointer ec(key); + if (!ec || ec.getPublicKey() == nullptr) return false; + *degree = EC_GROUP_get_degree(ec.getGroup()); + x->reset(BN_new()); + y->reset(BN_new()); + if (!*x || !*y || + EC_POINT_get_affine_coordinates( + ec.getGroup(), ec.getPublicKey(), x->get(), y->get(), nullptr) != 1) { + return false; + } + if (priv != nullptr) { + if (ec.getPrivateKey() == nullptr) return false; + priv->reset(BN_dup(ec.getPrivateKey())); + if (!*priv) return false; + } + return true; +} + int Ec::GetCurveId(const EVPKeyPointer& key) { #if NCRYPTO_USE_OPENSSL3_PROVIDER char name[80]; diff --git a/deps/ncrypto/ncrypto.h b/deps/ncrypto/ncrypto.h index 798d8a6130ca..397d68c731cb 100644 --- a/deps/ncrypto/ncrypto.h +++ b/deps/ncrypto/ncrypto.h @@ -813,6 +813,11 @@ class Ec final { static int GetCurveIdFromName(const char* name); static int GetCurveId(const EVPKeyPointer& key); + static bool GetKeyComponents(const EVPKeyPointer& key, + BignumPointer* x, + BignumPointer* y, + BignumPointer* priv, + int* degree); using GetCurveCallback = std::function; static bool GetCurves(GetCurveCallback callback); diff --git a/src/crypto/crypto_ec.cc b/src/crypto/crypto_ec.cc index 17f7999f22bd..f9263635d0b0 100644 --- a/src/crypto/crypto_ec.cc +++ b/src/crypto/crypto_ec.cc @@ -488,30 +488,21 @@ bool ExportJWKEcKey(Environment* env, const auto& m_pkey = key.GetAsymmetricKey(); CHECK_EQ(m_pkey.id(), EVP_PKEY_EC); - ECKeyPointer ec(m_pkey); - if (!ec) { - THROW_ERR_CRYPTO_INVALID_JWK(env, "Invalid JWK EC key"); + BignumPointer x; + BignumPointer y; + BignumPointer priv; + int degree_bits; + if (!Ec::GetKeyComponents( + m_pkey, + &x, + &y, + key.GetKeyType() == kKeyTypePrivate ? &priv : nullptr, + °ree_bits)) { return false; } - // A provider-backed key need not expose its public point. - if (ec.getPublicKey() == nullptr) return false; - - const auto pub = ec.getPublicKey(); - const auto group = ec.getGroup(); - - int degree_bits = EC_GROUP_get_degree(group); int degree_bytes = (degree_bits / CHAR_BIT) + (7 + (degree_bits % CHAR_BIT)) / 8; - auto x = BignumPointer::New(); - auto y = BignumPointer::New(); - - if (!EC_POINT_get_affine_coordinates(group, pub, x.get(), y.get(), nullptr)) { - ThrowCryptoError(env, ERR_get_error(), - "Failed to get elliptic-curve point coordinates"); - return false; - } - if (!target ->DefineOwnProperty( env->context(), env->jwk_kty_string(), env->jwk_ec_string()) @@ -535,7 +526,7 @@ bool ExportJWKEcKey(Environment* env, } Local crv_name; - const int nid = EC_GROUP_get_curve_name(group); + const int nid = Ec::GetCurveId(m_pkey); switch (nid) { case NID_X9_62_prime256v1: crv_name = env->p256_string(); @@ -562,9 +553,8 @@ bool ExportJWKEcKey(Environment* env, } if (key.GetKeyType() == kKeyTypePrivate) { - auto pvt = ec.getPrivateKey(); - if (pvt == nullptr) return false; - return SetEncodedValue(env, target, env->jwk_d_string(), pvt, degree_bytes) + return SetEncodedValue( + env, target, env->jwk_d_string(), priv.get(), degree_bytes) .IsJust(); } From fab5dddd77255857bf6d0786ac60402d2a880966 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Tue, 8 Sep 2026 00:34:22 +0200 Subject: [PATCH 060/217] crypto: avoid EC raw export reconstruction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read private scalars and matching uncompressed provider encodings directly. Assisted-by: GitHub Copilot Signed-off-by: Filip Skokan PR-URL: https://github.com/nodejs/node/pull/65908 Reviewed-By: Yagiz Nizipli Reviewed-By: James M Snell Reviewed-By: Tobias Nießen --- benchmark/crypto/ec-raw-export.js | 23 ++++++++++ deps/ncrypto/ncrypto.cc | 45 ++++++++++++++++++ deps/ncrypto/ncrypto.h | 3 ++ src/crypto/crypto_keys.cc | 76 ++++++++++--------------------- 4 files changed, 95 insertions(+), 52 deletions(-) create mode 100644 benchmark/crypto/ec-raw-export.js diff --git a/benchmark/crypto/ec-raw-export.js b/benchmark/crypto/ec-raw-export.js new file mode 100644 index 000000000000..15c1d9333bb4 --- /dev/null +++ b/benchmark/crypto/ec-raw-export.js @@ -0,0 +1,23 @@ +'use strict'; + +const common = require('../common.js'); +const { generateKeyPairSync } = require('crypto'); + +const bench = common.createBenchmark(main, { + namedCurve: ['prime256v1', 'secp384r1', 'secp521r1'], + format: ['raw-private', 'raw-public'], + type: ['uncompressed', 'compressed'], + n: [10000], +}, { + combinationFilter: ({ format, type }) => format === 'raw-public' || type === 'uncompressed', +}); + +function main({ namedCurve, format, type, n }) { + const pair = generateKeyPairSync('ec', { namedCurve }); + const key = format === 'raw-private' ? pair.privateKey : pair.publicKey; + const options = format === 'raw-public' ? { format, type } : { format }; + bench.start(); + for (let index = 0; index < n; index++) + key.export(options); + bench.end(n); +} diff --git a/deps/ncrypto/ncrypto.cc b/deps/ncrypto/ncrypto.cc index f7b2898d94a5..9cf724c1560c 100644 --- a/deps/ncrypto/ncrypto.cc +++ b/deps/ncrypto/ncrypto.cc @@ -6933,6 +6933,51 @@ int Ec::getCurve() const { return EC_GROUP_get_curve_name(getGroup()); } +DataPointer Ec::TryExportPublic(const EVPKeyPointer& key, + point_conversion_form_t form) { + if (form != POINT_CONVERSION_UNCOMPRESSED) return {}; +#if NCRYPTO_USE_OPENSSL3_PROVIDER + { + MarkPopErrorOnReturn pop_errors; + size_t length = 0; + if (EVP_PKEY_get_octet_string_param( + key.get(), OSSL_PKEY_PARAM_PUB_KEY, nullptr, 0, &length) == 1) { + auto bytes = DataPointer::Alloc(length); + if (bytes && length != 0 && + EVP_PKEY_get_octet_string_param(key.get(), + OSSL_PKEY_PARAM_PUB_KEY, + bytes.get(), + length, + &length) == 1 && + (bytes.get()[0] & ~1) == form) { + return bytes.resize(length); + } + } + } +#endif + return {}; +} + +DataPointer Ec::ExportPrivate(const EVPKeyPointer& key) { +#if NCRYPTO_USE_OPENSSL3_PROVIDER + { + MarkPopErrorOnReturn pop_errors; + BignumPointer priv; + BignumPointer order; + if (GetPKeyBnParam(key.get(), OSSL_PKEY_PARAM_PRIV_KEY, &priv) && + GetPKeyBnParam(key.get(), OSSL_PKEY_PARAM_EC_ORDER, &order)) { + return priv.encodePadded(order.byteLength()); + } + } +#endif + ECKeyPointer ec(key); + if (!ec || ec.getPrivateKey() == nullptr) return {}; + auto order = BignumPointer::New(); + if (!order || !EC_GROUP_get_order(ec.getGroup(), order.get(), nullptr)) + return {}; + return BignumPointer::EncodePadded(ec.getPrivateKey(), order.byteLength()); +} + bool Ec::GetKeyComponents(const EVPKeyPointer& key, BignumPointer* x, BignumPointer* y, diff --git a/deps/ncrypto/ncrypto.h b/deps/ncrypto/ncrypto.h index 397d68c731cb..944c42490d6c 100644 --- a/deps/ncrypto/ncrypto.h +++ b/deps/ncrypto/ncrypto.h @@ -813,6 +813,9 @@ class Ec final { static int GetCurveIdFromName(const char* name); static int GetCurveId(const EVPKeyPointer& key); + static DataPointer TryExportPublic(const EVPKeyPointer& key, + point_conversion_form_t form); + static DataPointer ExportPrivate(const EVPKeyPointer& key); static bool GetKeyComponents(const EVPKeyPointer& key, BignumPointer* x, BignumPointer* y, diff --git a/src/crypto/crypto_keys.cc b/src/crypto/crypto_keys.cc index 2d80caf76661..4623f4726bd0 100644 --- a/src/crypto/crypto_keys.cc +++ b/src/crypto/crypto_keys.cc @@ -386,21 +386,24 @@ bool KeyObjectData::ToEncodedPublicKey( Mutex::ScopedLock lock(mutex()); const auto& pkey = GetAsymmetricKey(); if (pkey.id() == EVP_PKEY_EC) { + auto form = static_cast(config.ec_point_form); + auto bytes = ncrypto::Ec::TryExportPublic(pkey, form); + if (bytes) + return Buffer::Copy(env, bytes.get(), bytes.size()) + .ToLocal(out); ECKeyPointer ec_key(pkey); if (!ec_key) { THROW_ERR_CRYPTO_INCOMPATIBLE_KEY_OPTIONS(env); return false; } - // A provider-backed key need not expose its public point. if (ec_key.getPublicKey() == nullptr) { THROW_ERR_CRYPTO_OPERATION_FAILED(env, "Failed to export EC public key"); return false; } - auto form = static_cast(config.ec_point_form); - const auto group = ec_key.getGroup(); - const auto point = ec_key.getPublicKey(); - return ECPointToBuffer(env, group, point, form).ToLocal(out); + return ECPointToBuffer( + env, ec_key.getGroup(), ec_key.getPublicKey(), form) + .ToLocal(out); } const int id = pkey.id(); bool is_raw_supported = id == EVP_PKEY_ED25519 || id == EVP_PKEY_ED448 || @@ -441,25 +444,7 @@ bool KeyObjectData::ToEncodedPrivateKey( Mutex::ScopedLock lock(mutex()); const auto& pkey = GetAsymmetricKey(); if (pkey.id() == EVP_PKEY_EC) { - ECKeyPointer ec_key(pkey); - if (!ec_key) { - THROW_ERR_CRYPTO_INCOMPATIBLE_KEY_OPTIONS(env); - return false; - } - const BIGNUM* private_key = ec_key.getPrivateKey(); - if (private_key == nullptr) { - THROW_ERR_CRYPTO_OPERATION_FAILED(env, - "Failed to export EC private key"); - return false; - } - const auto group = ec_key.getGroup(); - auto order = BignumPointer::New(); - if (!order || !EC_GROUP_get_order(group, order.get(), nullptr)) { - THROW_ERR_CRYPTO_OPERATION_FAILED(env, - "Failed to export EC private key"); - return false; - } - auto buf = BignumPointer::EncodePadded(private_key, order.byteLength()); + auto buf = ncrypto::Ec::ExportPrivate(pkey); if (!buf) { THROW_ERR_CRYPTO_OPERATION_FAILED(env, "Failed to export EC private key"); @@ -1581,24 +1566,27 @@ void KeyObjectHandle::ExportECPublicRaw( return THROW_ERR_CRYPTO_INCOMPATIBLE_KEY_OPTIONS(env); } + CHECK(args[0]->IsInt32()); + auto form = + static_cast(args[0].As()->Value()); + + auto bytes = ncrypto::Ec::TryExportPublic(m_pkey, form); + if (bytes) { + args.GetReturnValue().Set( + Buffer::Copy(env, bytes.get(), bytes.size()) + .FromMaybe(Local())); + return; + } ECKeyPointer ec_key(m_pkey); if (!ec_key) return THROW_ERR_CRYPTO_INCOMPATIBLE_KEY_OPTIONS(env); - // A provider-backed key need not expose its public point. if (ec_key.getPublicKey() == nullptr) { return THROW_ERR_CRYPTO_OPERATION_FAILED(env, "Failed to export EC public key"); } - - CHECK(args[0]->IsInt32()); - auto form = - static_cast(args[0].As()->Value()); - - const auto group = ec_key.getGroup(); - const auto point = ec_key.getPublicKey(); - Local buf; - if (!ECPointToBuffer(env, group, point, form).ToLocal(&buf)) return; - + if (!ECPointToBuffer(env, ec_key.getGroup(), ec_key.getPublicKey(), form) + .ToLocal(&buf)) + return; args.GetReturnValue().Set(buf); } @@ -1617,23 +1605,7 @@ void KeyObjectHandle::ExportECPrivateRaw( return THROW_ERR_CRYPTO_INCOMPATIBLE_KEY_OPTIONS(env); } - ECKeyPointer ec_key(m_pkey); - if (!ec_key) return THROW_ERR_CRYPTO_INCOMPATIBLE_KEY_OPTIONS(env); - - const BIGNUM* private_key = ec_key.getPrivateKey(); - if (private_key == nullptr) { - return THROW_ERR_CRYPTO_OPERATION_FAILED(env, - "Failed to export EC private key"); - } - - const auto group = ec_key.getGroup(); - auto order = BignumPointer::New(); - if (!order || !EC_GROUP_get_order(group, order.get(), nullptr)) { - return THROW_ERR_CRYPTO_OPERATION_FAILED(env, - "Failed to export EC private key"); - } - - auto buf = BignumPointer::EncodePadded(private_key, order.byteLength()); + auto buf = ncrypto::Ec::ExportPrivate(m_pkey); if (!buf) { return THROW_ERR_CRYPTO_OPERATION_FAILED(env, "Failed to export EC private key"); From 70edf90851298918f0f81776185b37b2a2f088e1 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Mon, 7 Sep 2026 22:36:38 +0200 Subject: [PATCH 061/217] crypto: avoid EC reconstruction for signature sizing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use EVP_PKEY_bits() to determine the width of ECDSA signature components on OpenSSL 3. This avoids reconstructing the EC group and public point just to read the group order size. Signed-off-by: Filip Skokan Assisted-by: GitHub Copilot PR-URL: https://github.com/nodejs/node/pull/65908 Reviewed-By: Yagiz Nizipli Reviewed-By: James M Snell Reviewed-By: Tobias Nießen --- deps/ncrypto/ncrypto.cc | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/deps/ncrypto/ncrypto.cc b/deps/ncrypto/ncrypto.cc index 9cf724c1560c..9c88d919e9ad 100644 --- a/deps/ncrypto/ncrypto.cc +++ b/deps/ncrypto/ncrypto.cc @@ -4130,11 +4130,7 @@ std::optional EVPKeyPointer::getBytesOfRS() const { #endif } else if (id == EVP_PKEY_EC) { #if NCRYPTO_USE_OPENSSL3_PROVIDER - Ec ec(get()); - if (!ec) return std::nullopt; - const EC_GROUP* group = ec.getGroup(); - if (group == nullptr) return std::nullopt; - bits = EC_GROUP_order_bits(group); + bits = EVP_PKEY_bits(get()); #else const EC_KEY* ec_key = EVP_PKEY_get0_EC_KEY(get()); if (ec_key == nullptr) return std::nullopt; From 858702e51c4eecea9d7aa68b20dfdec2c57f56af Mon Sep 17 00:00:00 2001 From: Philipp Dunkel Date: Thu, 10 Sep 2026 23:21:29 +0200 Subject: [PATCH 062/217] test: skip C++ symbols in tick-processor-arguments The test only checks that a CLI flag is passed through to the V8 tick processor, but processing a --prof log makes the tick processor resolve the C++ symbols of every shared library listed in it by shelling out to nm (plus c++filt on macOS) once per library. On a --shared build that links around a hundred dylibs this takes longer than the test timeout on the macOS x86_64 GitHub Actions runner, and the outcome depends on the host toolchain rather than on node. Drop the shared-library entries from the log before processing it so the test exercises argument handling only. C++ symbol resolution is covered by test/tick-processor. Signed-off-by: Philipp Dunkel PR-URL: https://github.com/nodejs/node/pull/65906 Refs: https://github.com/nodejs/node/issues/50050 Reviewed-By: James M Snell Reviewed-By: Matteo Collina --- test/parallel/test-tick-processor-arguments.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/parallel/test-tick-processor-arguments.js b/test/parallel/test-tick-processor-arguments.js index 406b13b676d7..a2d99192f979 100644 --- a/test/parallel/test-tick-processor-arguments.js +++ b/test/parallel/test-tick-processor-arguments.js @@ -19,6 +19,17 @@ const files = fs.readdirSync(tmpdir.path); const logfile = files.find((name) => /\.log$/.test(name)); assert(logfile); +// Drop the shared-library entries: the tick processor resolves the C++ +// symbols of every listed library through nm (and c++filt on macOS), which is +// slow on builds that link many shared libraries and depends on the host +// toolchain. This test only checks that CLI arguments reach the tick +// processor; C++ symbol resolution is covered by test/tick-processor. +const logpath = tmpdir.resolve(logfile); +fs.writeFileSync(logpath, fs.readFileSync(logpath, 'utf8') + .split('\n') + .filter((line) => !line.startsWith('shared-library,')) + .join('\n')); + // Make sure that the --preprocess argument is passed through correctly, // as an example flag listed in deps/v8/tools/tickprocessor.js. // Any of the other flags there should work for this test too, if --preprocess From afc3e559d21251eb979c7fdb396b59e059325636 Mon Sep 17 00:00:00 2001 From: Shelley Vohr Date: Wed, 2 Sep 2026 21:57:09 +0000 Subject: [PATCH 063/217] src: fix Stop() terminating the next Environment on the isolate After `Stop(env)`, freeing the Environment and creating another one on the same isolate failed whenever no JavaScript ran in between: the new Environment's first script was terminated before it started. That is the normal case when `Stop()` is called from the process exit handler for an uncaught exception, or by an embedder while the loop is idle. `Stop()` calls `isolate->TerminateExecution()` unless `kDoNotTerminateIsolate` is set, and V8 only clears that request the next time JavaScript runs, so it outlived the Environment it was meant for. Cancel a pending termination when the Environment it was meant for is freed. `Worker::Run()` already did this by hand before freeing its Environment, with a TODO asking why V8 hit a DCHECK without it; this is why, and that call now happens in `FreeEnvironment()`. Refs: https://github.com/nodejs/node/pull/33347 Signed-off-by: Shelley Vohr PR-URL: https://github.com/nodejs/node/pull/65819 Reviewed-By: Yagiz Nizipli Reviewed-By: Chemi Atlow --- src/api/environment.cc | 3 +++ src/node_worker.cc | 5 ----- test/cctest/test_environment.cc | 21 +++++++++++++++++++++ 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/api/environment.cc b/src/api/environment.cc index 7e4c1c341f11..3824cfdf3f26 100644 --- a/src/api/environment.cc +++ b/src/api/environment.cc @@ -536,6 +536,9 @@ void FreeEnvironment(Environment* env) { Isolate* isolate = env->isolate(); Isolate::DisallowJavascriptExecutionScope disallow_js(isolate, Isolate::DisallowJavascriptExecutionScope::THROW_ON_FAILURE); + // A termination requested by Stop() targets this Environment; if no JS ran + // since, it is still pending and must not hit the isolate's next user. + isolate->CancelTerminateExecution(); { HandleScope handle_scope(isolate); // For env->context(). Context::Scope context_scope(env->context()); diff --git a/src/node_worker.cc b/src/node_worker.cc index 5617fd34b3f9..9de1a9e25760 100644 --- a/src/node_worker.cc +++ b/src/node_worker.cc @@ -326,11 +326,6 @@ void Worker::Run() { DeleteFnPtr env_; auto cleanup_env = OnScopeLeave([&]() { - // TODO(addaleax): This call is harmless but should not be necessary. - // Figure out why V8 is raising a DCHECK() here without it - // (in test/parallel/test-async-hooks-worker-asyncfn-terminate-4.js). - isolate_->CancelTerminateExecution(); - if (!env_) return; env_->set_can_call_into_js(false); diff --git a/test/cctest/test_environment.cc b/test/cctest/test_environment.cc index c6c479a3a609..3b2e571025ea 100644 --- a/test/cctest/test_environment.cc +++ b/test/cctest/test_environment.cc @@ -367,6 +367,27 @@ TEST_F(EnvironmentTest, WorkerInEnvironmentWithoutSnapshot) { EXPECT_EQ(node::SpinEventLoop(*env).FromJust(), 0); } +TEST_F(EnvironmentTest, StopFromExitHandlerDoesNotLeakIntoNextEnvironment) { + const v8::HandleScope handle_scope(isolate_); + const Argv argv; + { + Env env{handle_scope, argv}; + node::SetProcessExitHandler( + *env, [](node::Environment* env_, int) { node::Stop(env_); }); + // The uncaught exception runs the exit handler from C++ and does not + // re-enter JS afterwards, so nothing consumes the termination request. + EXPECT_TRUE( + node::LoadEnvironment(*env, "throw new Error('uncaught')").IsEmpty()); + EXPECT_TRUE(node::SpinEventLoop(*env).IsNothing()); + } + { + Env env{handle_scope, argv, node::EnvironmentFlags::kNoCreateInspector}; + v8::Local result = + node::LoadEnvironment(*env, "return 42;").ToLocalChecked(); + EXPECT_EQ(result->Int32Value(env.context()).FromJust(), 42); + } +} + TEST_F(EnvironmentTest, NoEnvironmentSanity) { const v8::HandleScope handle_scope(isolate_); v8::Local context = v8::Context::New(isolate_); From 0e449d02a664628a2c716165a6f1349f7bdd92a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2026 07:44:09 +0000 Subject: [PATCH 064/217] tools: bump js-yaml from 4.3.1 to 4.3.2 in /tools/lint-md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [js-yaml](https://github.com/nodeca/js-yaml) from 4.3.1 to 4.3.2. - [Changelog](https://github.com/nodeca/js-yaml/blob/4.3.2/CHANGELOG.md) - [Commits](https://github.com/nodeca/js-yaml/compare/4.3.1...4.3.2) --- updated-dependencies: - dependency-name: js-yaml dependency-version: 4.3.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] PR-URL: https://github.com/nodejs/node/pull/65932 Reviewed-By: Marco Ippolito Reviewed-By: Filip Skokan Reviewed-By: Colin Ihrig Reviewed-By: Luigi Pinca Reviewed-By: Gürgün Dayıoğlu --- tools/lint-md/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/lint-md/package-lock.json b/tools/lint-md/package-lock.json index 1ab4268328b2..ea6663dc369b 100644 --- a/tools/lint-md/package-lock.json +++ b/tools/lint-md/package-lock.json @@ -326,9 +326,9 @@ } }, "node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "funding": [ { "type": "github", From 15f2549f74077b671dc47f5ce12b7a2a8df3565e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2026 07:44:22 +0000 Subject: [PATCH 065/217] tools: bump js-yaml from 4.3.1 to 4.3.2 in /tools/eslint Bumps [js-yaml](https://github.com/nodeca/js-yaml) from 4.3.1 to 4.3.2. - [Changelog](https://github.com/nodeca/js-yaml/blob/4.3.2/CHANGELOG.md) - [Commits](https://github.com/nodeca/js-yaml/compare/4.3.1...4.3.2) --- updated-dependencies: - dependency-name: js-yaml dependency-version: 4.3.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] PR-URL: https://github.com/nodejs/node/pull/65931 Reviewed-By: Filip Skokan Reviewed-By: Colin Ihrig Reviewed-By: Luigi Pinca --- tools/eslint/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/eslint/package-lock.json b/tools/eslint/package-lock.json index d9f1b6c5e682..ea1716a3247d 100644 --- a/tools/eslint/package-lock.json +++ b/tools/eslint/package-lock.json @@ -1454,9 +1454,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "funding": [ { "type": "github", From 62e2bf025c3138f8debd475a1079c61afe761af7 Mon Sep 17 00:00:00 2001 From: Shelley Vohr Date: Thu, 3 Sep 2026 04:13:45 +0000 Subject: [PATCH 066/217] src: detach cppgc wrappers from their Realm before it is freed `Realm::RunCleanup()` finalizes the cppgc-managed wrappers it tracks so that none of them touches the Realm once it is gone, but it reaches them through weak persistents, and the GC clears those as soon as it finds a wrapper dead. With lazy and concurrent sweeping the destructor can run much later, so a wrapper collected shortly before `FreeEnvironment()` and swept after it was skipped by the cleanup and kept its `realm_`: `~CppgcMixin()` then wrote `should_purge_empty_cppgc_wrappers_` into the freed Realm, and a subclass destructor calling `Finalize()` as documented would have called `Clean()` with a dangling Realm. A Worker that compiles a few `vm.Script`s, gets a full GC from external memory pressure and calls `process.exit()` is enough to hit the first case. Move the Realm pointer into the list node, which the wrapper now owns and deletes in its destructor. `CppgcWrapperList::Cleanup()` unlinks every node, finalizing the wrappers that are still alive and clearing the Realm pointer for the collected ones, which only their own destructor may still touch. `Realm::PendingCleanup()` accounts for the list so it is always drained. The purge flag, its GC epilogue callback and `PurgeEmpty()` are no longer needed, and removing them also stops the list nodes of wrappers that are alive at `FreeEnvironment()` from leaking. Refs: https://github.com/nodejs/node/pull/56534 Signed-off-by: Shelley Vohr PR-URL: https://github.com/nodejs/node/pull/65778 Reviewed-By: Matteo Collina --- src/cppgc_helpers-inl.h | 22 +++++--- src/cppgc_helpers.cc | 28 +++------- src/cppgc_helpers.h | 16 ++---- src/node_realm-inl.h | 12 ++-- src/node_realm.cc | 19 +------ src/node_realm.h | 29 +++------- test/cctest/test_cppgc.cc | 114 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 156 insertions(+), 84 deletions(-) diff --git a/src/cppgc_helpers-inl.h b/src/cppgc_helpers-inl.h index 26cf107602f3..4db197d694f8 100644 --- a/src/cppgc_helpers-inl.h +++ b/src/cppgc_helpers-inl.h @@ -11,7 +11,6 @@ namespace node { template void CppgcMixin::Wrap(T* ptr, Realm* realm, v8::Local obj) { CHECK_GE(obj->InternalFieldCount(), T::kInternalFieldCount); - ptr->realm_ = realm; v8::Isolate* isolate = realm->isolate(); ptr->traced_reference_ = v8::TracedReference(isolate, obj); // Note that ptr must be of concrete type T in Wrap. @@ -23,7 +22,7 @@ void CppgcMixin::Wrap(T* ptr, Realm* realm, v8::Local obj) { realm->isolate_data()->embedder_id_for_cppgc(), EmbedderDataTag::kEmbedderType); obj->SetAlignedPointerInInternalField(kSlot, ptr, EmbedderDataTag::kDefault); - realm->TrackCppgcWrapper(ptr); + ptr->list_node_ = realm->TrackCppgcWrapper(ptr); } template @@ -49,17 +48,26 @@ T* CppgcMixin::Unwrap(v8::Local obj) { } v8::Local CppgcMixin::object() const { - return traced_reference_.Get(realm_->isolate()); + return traced_reference_.Get(realm()->isolate()); } Environment* CppgcMixin::env() const { - return realm_->env(); + return realm()->env(); +} + +Realm* CppgcMixin::realm() const { + return list_node_ == nullptr ? nullptr : list_node_->realm; +} + +void CppgcMixin::Finalize() { + Realm* current_realm = realm(); + if (current_realm == nullptr) return; + this->Clean(current_realm); + list_node_->realm = nullptr; } CppgcMixin::~CppgcMixin() { - if (realm_ != nullptr) { - realm_->set_should_purge_empty_cppgc_wrappers(true); - } + delete list_node_; } } // namespace node diff --git a/src/cppgc_helpers.cc b/src/cppgc_helpers.cc index 7c557a822d20..9424904c0ea8 100644 --- a/src/cppgc_helpers.cc +++ b/src/cppgc_helpers.cc @@ -1,14 +1,14 @@ -#include "cppgc_helpers.h" -#include "env-inl.h" +#include "cppgc_helpers.h" // NOLINT(build/include_inline) +#include "cppgc_helpers-inl.h" namespace node { void CppgcWrapperList::Cleanup() { - for (auto node : *this) { - CppgcMixin* ptr = node->persistent.Get(); - if (ptr != nullptr) { - ptr->Finalize(); - } + while (!IsEmpty()) { + CppgcWrapperListNode* node = PopFront(); + CppgcMixin* wrapper = node->persistent.Get(); + if (wrapper != nullptr) wrapper->Finalize(); + node->realm = nullptr; } } @@ -23,18 +23,4 @@ void CppgcWrapperList::MemoryInfo(MemoryTracker* tracker) const { } } } - -void CppgcWrapperList::PurgeEmpty() { - for (auto weak_it = begin(); weak_it != end();) { - CppgcWrapperListNode* node = *weak_it; - auto next_it = ++weak_it; - // The underlying cppgc wrapper has already been garbage collected. - // Remove it from the list. - if (!node->persistent) { - node->persistent.Clear(); - delete node; - } - weak_it = next_it; - } -} } // namespace node diff --git a/src/cppgc_helpers.h b/src/cppgc_helpers.h index fe6300a5d5d2..f1363d5da78f 100644 --- a/src/cppgc_helpers.h +++ b/src/cppgc_helpers.h @@ -47,9 +47,9 @@ class CppgcWrapperListNode; * cleanup relies on a living Node.js `Realm`, it should implement a * pattern like this: * - * ~MyWrap() { this->Destroy(); } + * ~MyWrap() { this->Finalize(); } * void Clean(Realm* env) override { - * // Do cleanup that relies on a living Environemnt. + * // Do cleanup that relies on a living Realm. * } */ class CppgcMixin : public cppgc::GarbageCollectedMixin, public MemoryRetainer { @@ -68,7 +68,7 @@ class CppgcMixin : public cppgc::GarbageCollectedMixin, public MemoryRetainer { inline v8::Local object() const; inline Environment* env() const; - inline Realm* realm() const { return realm_; } + inline Realm* realm() const; inline v8::Local object(v8::Isolate* isolate) const { return traced_reference_.Get(isolate); } @@ -95,11 +95,7 @@ class CppgcMixin : public cppgc::GarbageCollectedMixin, public MemoryRetainer { // destructor. Outside of Finalize(), subclasses should avoid calling // into JavaScript or perform any operation that can trigger garbage // collection during the destruction. - void Finalize() { - if (realm_ == nullptr) return; - this->Clean(realm_); - realm_ = nullptr; - } + inline void Finalize(); // The default implementation of Clean() is a no-op. If subclasses wish // to perform cleanup that require a living Realm, they should @@ -110,10 +106,8 @@ class CppgcMixin : public cppgc::GarbageCollectedMixin, public MemoryRetainer { inline ~CppgcMixin(); - friend class CppgcWrapperListNode; - private: - Realm* realm_ = nullptr; + CppgcWrapperListNode* list_node_ = nullptr; v8::TracedReference traced_reference_; }; diff --git a/src/node_realm-inl.h b/src/node_realm-inl.h index 394ece5a8ace..62f77384a387 100644 --- a/src/node_realm-inl.h +++ b/src/node_realm-inl.h @@ -133,11 +133,13 @@ void Realm::TrackBaseObject(BaseObject* bo) { ++base_object_count_; } -CppgcWrapperListNode::CppgcWrapperListNode(CppgcMixin* ptr) : persistent(ptr) {} +CppgcWrapperListNode::CppgcWrapperListNode(Realm* realm, CppgcMixin* wrapper) + : realm(realm), persistent(wrapper) {} -void Realm::TrackCppgcWrapper(CppgcMixin* handle) { - DCHECK_EQ(handle->realm(), this); - cppgc_wrapper_list_.PushFront(new CppgcWrapperListNode(handle)); +CppgcWrapperListNode* Realm::TrackCppgcWrapper(CppgcMixin* handle) { + CppgcWrapperListNode* node = new CppgcWrapperListNode(this, handle); + cppgc_wrapper_list_.PushFront(node); + return node; } void Realm::UntrackBaseObject(BaseObject* bo) { @@ -146,7 +148,7 @@ void Realm::UntrackBaseObject(BaseObject* bo) { } bool Realm::PendingCleanup() const { - return !base_object_list_.IsEmpty(); + return !base_object_list_.IsEmpty() || !cppgc_wrapper_list_.IsEmpty(); } } // namespace node diff --git a/src/node_realm.cc b/src/node_realm.cc index d2459d4eeb33..91bb530dfd94 100644 --- a/src/node_realm.cc +++ b/src/node_realm.cc @@ -10,8 +10,6 @@ namespace node { using v8::Context; using v8::EscapableHandleScope; -using v8::GCCallbackFlags; -using v8::GCType; using v8::HandleScope; using v8::Isolate; using v8::Local; @@ -25,26 +23,11 @@ Realm::Realm(Environment* env, v8::Local context, Kind kind) : env_(env), isolate_(Isolate::GetCurrent()), kind_(kind) { context_.Reset(isolate_, context); env->AssignToContext(context, this, ContextInfo("")); - // The environment can also purge empty wrappers in the check callback, - // though that may be a bit excessive depending on usage patterns. - // For now using the GC epilogue is adequate. - isolate_->AddGCEpilogueCallback(PurgeEmptyCppgcWrappers, this); } Realm::~Realm() { - isolate_->RemoveGCEpilogueCallback(PurgeEmptyCppgcWrappers, this); CHECK_EQ(base_object_count_, 0); -} - -void Realm::PurgeEmptyCppgcWrappers(Isolate* isolate, - GCType type, - GCCallbackFlags flags, - void* data) { - Realm* realm = static_cast(data); - if (realm->should_purge_empty_cppgc_wrappers_) { - realm->cppgc_wrapper_list_.PurgeEmpty(); - realm->should_purge_empty_cppgc_wrappers_ = false; - } + CHECK(cppgc_wrapper_list_.IsEmpty()); } void Realm::MemoryInfo(MemoryTracker* tracker) const { diff --git a/src/node_realm.h b/src/node_realm.h index 690beaf1a1aa..8bba26f50a92 100644 --- a/src/node_realm.h +++ b/src/node_realm.h @@ -27,16 +27,16 @@ using BindingDataStore = static_cast(BindingDataType::kBindingDataTypeCount)>; /** - * This is a wrapper around a weak persistent of CppgcMixin, used in the - * CppgcWrapperList to avoid accessing already garbage collected CppgcMixins. + * Owned by a CppgcMixin and linked into its Realm's list until the Realm + * cleans up and clears `realm`. The Realm only calls into wrappers the GC + * still considers alive (the weak persistent); a collected wrapper whose + * destructor runs later sees `realm == nullptr` instead of a freed Realm. */ class CppgcWrapperListNode { public: - explicit inline CppgcWrapperListNode(CppgcMixin* ptr); - inline explicit operator bool() const { return !persistent; } - inline CppgcMixin* operator->() const { return persistent.Get(); } - inline CppgcMixin* operator*() const { return persistent.Get(); } + inline CppgcWrapperListNode(Realm* realm, CppgcMixin* wrapper); + Realm* realm; cppgc::WeakPersistent persistent; // Used by ContainerOf in the ListNode implementation for fast manipulation of // CppgcWrapperList. @@ -53,7 +53,6 @@ class CppgcWrapperList public MemoryRetainer { public: void Cleanup(); - void PurgeEmpty(); SET_MEMORY_INFO_NAME(CppgcWrapperList) SET_SELF_SIZE(CppgcWrapperList) @@ -148,7 +147,7 @@ class Realm : public MemoryRetainer { // Base object count created after the bootstrap of the realm. inline int64_t base_object_created_after_bootstrap() const; - inline void TrackCppgcWrapper(CppgcMixin* handle); + inline CppgcWrapperListNode* TrackCppgcWrapper(CppgcMixin* handle); inline CppgcWrapperList* cppgc_wrapper_list() { return &cppgc_wrapper_list_; } #define V(PropertyName, TypeName) \ @@ -164,14 +163,6 @@ class Realm : public MemoryRetainer { // it's only used for tests. std::vector builtins_in_snapshot; - // This used during the destruction of cppgc wrappers to inform a GC epilogue - // callback to clean up the weak persistents used to track cppgc wrappers if - // the wrappers are already garbage collected to prevent holding on to - // excessive useless persistents. - inline void set_should_purge_empty_cppgc_wrappers(bool value) { - should_purge_empty_cppgc_wrappers_ = value; - } - protected: ~Realm(); @@ -181,17 +172,11 @@ class Realm : public MemoryRetainer { // Shorthand for isolate pointer. v8::Isolate* isolate_; v8::Global context_; - bool should_purge_empty_cppgc_wrappers_ = false; #define V(PropertyName, TypeName) v8::Global PropertyName##_; PER_REALM_STRONG_PERSISTENT_VALUES(V) #undef V - static void PurgeEmptyCppgcWrappers(v8::Isolate* isolate, - v8::GCType type, - v8::GCCallbackFlags flags, - void* data); - private: void InitializeContext(v8::Local context, const RealmSerializeInfo* realm_info); diff --git a/test/cctest/test_cppgc.cc b/test/cctest/test_cppgc.cc index 2f586617bd6c..478098154665 100644 --- a/test/cctest/test_cppgc.cc +++ b/test/cctest/test_cppgc.cc @@ -3,8 +3,11 @@ #include #include #include +#include #include #include +#include "cppgc_helpers-inl.h" +#include "node_realm-inl.h" #include "node_test_fixture.h" // This tests that Node.js can work with an existing CppHeap. @@ -106,3 +109,114 @@ TEST_F(NodeZeroIsolateTestFixture, ExistingCppHeapTest) { // heap can be reclaimed. So just check at least some of them are traced. EXPECT_GT(CppGCed::kTraceCount, 0); } + +class CppgcTest : public EnvironmentTestFixture { + protected: + // Above the external memory hard limit, so V8 runs a full GC synchronously + // and leaves sweeping (and cppgc destructors) for later. + static constexpr size_t kExternalMemoryPressure = size_t{8} << 30; + + void CollectGarbageLeavingSweepingPending() { + v8::ExternalMemoryAccounter pressure; + pressure.Increase(isolate_, kExternalMemoryPressure); + pressure.Decrease(isolate_, kExternalMemoryPressure); + } + + void FinishSweeping() { + isolate_->LowMemoryNotification(); + platform->DrainTasks(isolate_); + } +}; + +using node::CppgcMixin; + +class RealmBoundWrap final : CPPGC_MIXIN(RealmBoundWrap) { + public: + SET_CPPGC_NAME(RealmBoundWrap) + DEFAULT_CPPGC_TRACE() + SET_NO_MEMORY_INFO() + + static node::Realm* live_realm; + static int clean_count; + static int clean_with_dead_realm_count; + static int destructor_count; + + RealmBoundWrap(node::Environment* env, v8::Local object) { + CppgcMixin::Wrap(this, env, object); + } + ~RealmBoundWrap() { + Finalize(); + destructor_count++; + } + void Clean(node::Realm* realm) override { + clean_count++; + if (realm != live_realm) clean_with_dead_realm_count++; + } +}; + +node::Realm* RealmBoundWrap::live_realm = nullptr; +int RealmBoundWrap::clean_count = 0; +int RealmBoundWrap::clean_with_dead_realm_count = 0; +int RealmBoundWrap::destructor_count = 0; + +TEST_F(CppgcTest, CleanIsNotCalledWithFreedRealm) { + constexpr int kCount = 32; + { + const v8::HandleScope handle_scope(isolate_); + Env env{handle_scope, Argv()}; + RealmBoundWrap::live_realm = (*env)->principal_realm(); + + v8::Local ctor = v8::FunctionTemplate::New(isolate_); + ctor->InstanceTemplate()->SetInternalFieldCount( + node::CppgcMixin::kInternalFieldCount); + v8::Local fn = + ctor->GetFunction(env.context()).ToLocalChecked(); + { + v8::HandleScope inner_scope(isolate_); + for (int i = 0; i <= kCount; i++) { + v8::Local obj = + fn->NewInstance(env.context()).ToLocalChecked(); + cppgc::MakeGarbageCollected( + (*env)->cppgc_allocation_handle(), *env, obj); + if (i < kCount) continue; + env.context() + ->Global() + ->Set(env.context(), + v8::String::NewFromUtf8Literal(isolate_, "kept"), + obj) + .Check(); + } + } + + CollectGarbageLeavingSweepingPending(); + EXPECT_LT(RealmBoundWrap::destructor_count, kCount); + } + RealmBoundWrap::live_realm = nullptr; + FinishSweeping(); + + EXPECT_GE(RealmBoundWrap::clean_count, 1); + EXPECT_EQ(RealmBoundWrap::clean_with_dead_realm_count, 0); +} + +TEST_F(CppgcTest, WrappersAliveAtFreeEnvironmentDoNotLeak) { + const v8::HandleScope handle_scope(isolate_); + Env env{handle_scope, Argv()}; + node::LoadEnvironment(*env, + "globalThis.script = new (require('vm').Script)('1');" + "globalThis.context = require('vm').createContext();") + .ToLocalChecked(); +} + +TEST_F(CppgcTest, VmScriptCollectedBeforeFreeEnvironmentSweptAfter) { + { + const v8::HandleScope handle_scope(isolate_); + Env env{handle_scope, Argv()}; + node::LoadEnvironment(*env, + "const { Script } = require('vm');" + "for (let i = 0; i < 64; i++) new Script('1');" + "undefined;") + .ToLocalChecked(); + CollectGarbageLeavingSweepingPending(); + } + FinishSweeping(); +} From 6eee01fbaeba68a90f877a7284a5f51300d6b5b7 Mon Sep 17 00:00:00 2001 From: Yuya Inoue <65857152+inoway46@users.noreply.github.com> Date: Fri, 11 Sep 2026 20:44:15 +0900 Subject: [PATCH 067/217] test: fix flaky common WPT inspector test Wait for the WPT child to reach its startup wait before sending Runtime.runIfWaitingForDebugger. Signed-off-by: inoway46 Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/65937 Reviewed-By: Filip Skokan Reviewed-By: Luigi Pinca Reviewed-By: Xuguang Mei --- test/parallel/test-common-wpt-inspect.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/parallel/test-common-wpt-inspect.js b/test/parallel/test-common-wpt-inspect.js index ff53902972c5..e614a4cfaa2c 100644 --- a/test/parallel/test-common-wpt-inspect.js +++ b/test/parallel/test-common-wpt-inspect.js @@ -23,11 +23,14 @@ async function main() { parent.on('stderr', (line) => stderr.push(line)); const session = await parent.connectInspectorSession(); + await session.send({ method: 'NodeRuntime.enable' }); + await session.waitForNotification('NodeRuntime.waitingForDebugger'); await session.send([ { method: 'Runtime.enable' }, { method: 'Debugger.enable' }, { method: 'Runtime.runIfWaitingForDebugger' }, ]); + await session.send({ method: 'NodeRuntime.disable' }); await session.waitForNotification('Debugger.paused'); await session.send({ method: 'Debugger.resume' }); await session.disconnect(); From 4cdcf7f6a4160c8180552535d6b78c8dcaad3c9c Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sun, 6 Sep 2026 15:27:21 -0700 Subject: [PATCH 068/217] zlib: fix zstd reset Preserve dictionaries and params when zstd is reset. Update missing documentation. Signed-off-by: James M Snell Assisted-by: Opencode PR-URL: https://github.com/nodejs/node/pull/65867 Reviewed-By: Trivikram Kamat Reviewed-By: Filip Skokan Reviewed-By: Robert Nagy --- doc/api/zlib.md | 10 +++- src/node_zlib.cc | 44 +++++++++++++++-- test/parallel/test-zlib-zstd-reset.js | 68 +++++++++++++++++++++++++++ 3 files changed, 116 insertions(+), 6 deletions(-) create mode 100644 test/parallel/test-zlib-zstd-reset.js diff --git a/doc/api/zlib.md b/doc/api/zlib.md index 23df37f3339f..f4fe51036df1 100644 --- a/doc/api/zlib.md +++ b/doc/api/zlib.md @@ -2186,8 +2186,14 @@ Only applicable to deflate algorithm. added: v0.7.0 --> -Reset the compressor/decompressor to factory defaults. Only applicable to -the inflate and deflate algorithms. +For inflate and deflate streams, reset the compressor/decompressor to factory +defaults. + +For Zstd streams, cancel the current frame and start a new session while +preserving the configured parameters and dictionary. If `pledgedSrcSize` was +configured for a Zstd compressor, it applies again to the next frame. + +Calling `reset()` while a write is in progress throws an `Error`. ## Class: `ZstdOptions` diff --git a/src/node_zlib.cc b/src/node_zlib.cc index c74e98c9cd11..b1c83f3e7a26 100644 --- a/src/node_zlib.cc +++ b/src/node_zlib.cc @@ -1718,7 +1718,29 @@ CompressionError ZstdCompressContext::Init(uint64_t pledged_src_size, } CompressionError ZstdCompressContext::ResetStream() { - return Init(pledged_src_size_); + size_t result = ZSTD_CCtx_reset(cctx_.get(), ZSTD_reset_session_only); + if (ZSTD_isError(result)) { + const ZSTD_ErrorCode error = ZSTD_getErrorCode(result); + return CompressionError( + ZSTD_getErrorString(error), ZstdStrerror(error), error); + } + + result = ZSTD_CCtx_setPledgedSrcSize(cctx_.get(), pledged_src_size_); + if (ZSTD_isError(result)) { + const ZSTD_ErrorCode error = ZSTD_getErrorCode(result); + return CompressionError( + ZSTD_getErrorString(error), ZstdStrerror(error), error); + } + + if (pledged_src_size_ == ZSTD_CONTENTSIZE_UNKNOWN) { + consumed_src_size_.reset(); + } else { + consumed_src_size_ = 0; + } + error_ = ZSTD_error_no_error; + error_string_.clear(); + error_code_string_.clear(); + return {}; } void ZstdCompressContext::DoThreadPoolWork() { @@ -1798,9 +1820,23 @@ CompressionError ZstdDecompressContext::Init(uint64_t pledged_src_size, } CompressionError ZstdDecompressContext::ResetStream() { - // We pass ZSTD_CONTENTSIZE_UNKNOWN because the argument is ignored for - // decompression. - return Init(ZSTD_CONTENTSIZE_UNKNOWN, {}, reject_garbage_after_end_); + const size_t result = + ZSTD_DCtx_reset(dctx_.get(), ZSTD_reset_session_only); + if (ZSTD_isError(result)) { + const ZSTD_ErrorCode error = ZSTD_getErrorCode(result); + return CompressionError( + ZSTD_getErrorString(error), ZstdStrerror(error), error); + } + + frame_complete_ = false; + decoding_frame_after_complete_ = false; + ignoring_trailing_input_ = false; + frame_prefix_size_ = 0; + possible_frame_types_ = 0; + error_ = ZSTD_error_no_error; + error_string_.clear(); + error_code_string_.clear(); + return {}; } void ZstdDecompressContext::DoThreadPoolWork() { diff --git a/test/parallel/test-zlib-zstd-reset.js b/test/parallel/test-zlib-zstd-reset.js new file mode 100644 index 000000000000..839669bc630a --- /dev/null +++ b/test/parallel/test-zlib-zstd-reset.js @@ -0,0 +1,68 @@ +'use strict'; + +require('../common'); +const assert = require('assert'); +const { finished } = require('stream/promises'); +const test = require('node:test'); +const zlib = require('zlib'); + +const dictionary = Buffer.from( + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. ' + + 'Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.', +); +const input = Buffer.from( + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. '.repeat(100), +); + +async function collect(stream, ...data) { + const chunks = []; + stream.on('data', (chunk) => chunks.push(chunk)); + for (let i = 0; i < data.length - 1; i++) { + stream.write(data[i]); + } + stream.end(data[data.length - 1]); + await finished(stream); + return Buffer.concat(chunks); +} + +test('ZstdCompress reset preserves its initial options', async () => { + const options = { + dictionary, + pledgedSrcSize: input.length, + params: { + [zlib.constants.ZSTD_c_compressionLevel]: 19, + [zlib.constants.ZSTD_c_checksumFlag]: 1, + }, + }; + const expected = await collect(zlib.createZstdCompress(options), input); + const reset = zlib.createZstdCompress(options); + reset.reset(); + + assert.deepStrictEqual(await collect(reset, input), expected); +}); + +test('ZstdDecompress reset preserves its dictionary', async () => { + const compressed = zlib.zstdCompressSync(input, { dictionary }); + const decompress = zlib.createZstdDecompress({ dictionary }); + decompress.reset(); + + assert.deepStrictEqual(await collect(decompress, compressed), input); +}); + +test('ZstdDecompress reset preserves its parameters', async () => { + const compressed = await collect(zlib.createZstdCompress({ + params: { + [zlib.constants.ZSTD_c_windowLog]: 11, + }, + }), Buffer.alloc(2048), Buffer.alloc(2048)); + const decompress = zlib.createZstdDecompress({ + params: { + [zlib.constants.ZSTD_d_windowLogMax]: 10, + }, + }); + decompress.reset(); + + await assert.rejects(collect(decompress, compressed), { + code: 'ZSTD_error_frameParameter_windowTooLarge', + }); +}); From 89aa2bd10c20f6e0abb5f36194788bfb90abd357 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sun, 6 Sep 2026 15:34:06 -0700 Subject: [PATCH 069/217] doc: fill in missing zstd docs Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/65867 Reviewed-By: Trivikram Kamat Reviewed-By: Filip Skokan Reviewed-By: Robert Nagy --- doc/api/zlib.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/api/zlib.md b/doc/api/zlib.md index f4fe51036df1..229c72bb306e 100644 --- a/doc/api/zlib.md +++ b/doc/api/zlib.md @@ -2154,7 +2154,8 @@ added: v0.5.8 --> * `kind` **Default:** `zlib.constants.Z_FULL_FLUSH` for zlib-based streams, - `zlib.constants.BROTLI_OPERATION_FLUSH` for Brotli-based streams. + `zlib.constants.BROTLI_OPERATION_FLUSH` for Brotli-based streams, and + `zlib.constants.ZSTD_e_flush` for Zstd-based streams. * `callback` {Function} Flush pending data. Don't call this frivolously, premature flushes negatively @@ -2221,6 +2222,9 @@ Each Zstd-based class takes an `options` object. All options are optional. * `finishFlush` {integer} **Default:** `zlib.constants.ZSTD_e_end` * `chunkSize` {integer} **Default:** `16 * 1024` * `params` {Object} Key-value object containing indexed [Zstd parameters][]. +* `pledgedSrcSize` {number} Expected total size of the uncompressed input. It + must be a non-negative safe integer and must match the input size when + compression finishes. Only applicable to Zstd compressors. * `maxOutputLength` {integer} Limits output size when using [convenience methods][]. **Default:** [`buffer.kMaxLength`][] * `info` {boolean} If `true`, returns an object with `buffer` and `engine`. **Default:** `false` @@ -3061,6 +3065,8 @@ Compress a chunk of data with [`ZstdCompress`][]. ### `zlib.zstdDecompress(buffer[, options], callback)` +> Stability: 1 - Experimental + +### `stream.opened` + + + +* Type: {Promise} + +A promise that is immediately fulfilled, if the stream fits within +flow control limits or fulfilled when the pending stream is created. +It rejects, if a pending stream is closed with an error before being +created. + ### `stream.closed` @@ -68,7 +69,8 @@ See: * Merging pull requests The TSC can remove inactive collaborators or provide them with _emeritus_ -status. Emeriti may request that the TSC restore them to active status. +status. Emeriti may request that the TSC restore them to active status. See +[Restoring emeritus Collaborators](#restoring-emeritus-collaborators). A collaborator is automatically made emeritus (and removed from active collaborator status) if it has been more than 12 months since the collaborator @@ -335,6 +337,29 @@ After the nomination passes, a TSC member onboards the new collaborator. See [the onboarding guide](./onboarding.md) for details of the onboarding process. +### Restoring emeritus Collaborators + +An emeritus collaborator who has resumed contributing may request restoration to +active status by opening an issue in [the TSC issue tracker][]. The request +describes their recent contributions and their intent to take on collaborator +responsibilities again. There is no new nomination and no vote. The request +stays open for one week, matching the window for a collaborator nomination. If +no TSC member objects, the request passes. + +Before restoring access, a TSC member confirms that the account making the +request is still under the control of the same person. See +[The Authenticity of Contributors](#the-authenticity-of-contributors). + +After the request passes, a TSC member re-onboards the returning collaborator, +reversing the applicable +[offboarding tasks](./doc/contributing/offboarding.md). As in +[the onboarding guide][], the returning collaborator authors the pull request +moving themselves from the emeriti list back to the active list in the README. +That restarts the activity clock the [inactive collaborator workflow][] measures. + +An emeritus TSC member returning as a collaborator rejoins the TSC through a TSC +motion under [Section 3 of the TSC Charter][TSC Charter]. + ## Consensus seeking process The TSC follows a [Consensus Seeking][] decision-making model per the @@ -343,5 +368,8 @@ The TSC follows a [Consensus Seeking][] decision-making model per the [Consensus Seeking]: https://en.wikipedia.org/wiki/Consensus-seeking_decision-making [TSC Charter]: https://github.com/nodejs/TSC/blob/HEAD/TSC-Charter.md [discussion in the nodejs/collaborators]: https://github.com/nodejs/collaborators/discussions/categories/collaborator-nominations +[inactive collaborator workflow]: https://github.com/nodejs/node/blob/HEAD/.github/workflows/find-inactive-collaborators.yml [nodejs/help]: https://github.com/nodejs/help [nodejs/node]: https://github.com/nodejs/node +[the TSC issue tracker]: https://github.com/nodejs/TSC/issues +[the onboarding guide]: ./onboarding.md#exercise-make-a-pull-request-adding-yourself-to-the-readme diff --git a/doc/contributing/offboarding.md b/doc/contributing/offboarding.md index f9d8140b54b4..8f431b5e9a60 100644 --- a/doc/contributing/offboarding.md +++ b/doc/contributing/offboarding.md @@ -22,4 +22,8 @@ emeritus or leaves the project. the collaborator be removed from the Node.js coverity project if they had access. +An emeritus collaborator may later ask the TSC to restore them to active status. +See [Restoring emeritus Collaborators][]. + +[Restoring emeritus Collaborators]: https://github.com/nodejs/node/blob/HEAD/GOVERNANCE.md#restoring-emeritus-collaborators [`@nodejs/collaborators`]: https://github.com/orgs/nodejs/teams/collaborators/members From 7fe64727a26e80c36f5caba9c09e269bb8a4ac9f Mon Sep 17 00:00:00 2001 From: Jungwon Sohn Date: Sun, 13 Sep 2026 00:03:46 +0900 Subject: [PATCH 092/217] test: cover experimental stream iterator builtins Verify that stream/iter and zlib/iter are hidden by default and exposed under both supported specifier forms when --experimental-stream-iter is enabled. Assisted-by: Codex Signed-off-by: sjungwon03 PR-URL: https://github.com/nodejs/node/pull/65964 Reviewed-By: Daeyeon Jeong Reviewed-By: James M Snell Reviewed-By: Xuguang Mei --- .../test-module-builtin-experimental.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/parallel/test-module-builtin-experimental.js b/test/parallel/test-module-builtin-experimental.js index 282701e7a0ae..2a3d75e23c1b 100644 --- a/test/parallel/test-module-builtin-experimental.js +++ b/test/parallel/test-module-builtin-experimental.js @@ -24,6 +24,24 @@ for (const [id, flag] of experimentalBuiltins) { ], { status: 0 }); } +const schemelessExperimentalBuiltins = [ + ['stream/iter', '--experimental-stream-iter'], + ['zlib/iter', '--experimental-stream-iter'], +]; + +for (const [id, flag] of schemelessExperimentalBuiltins) { + const nodeBuiltin = `node:${id}`; + + spawnSyncAndAssert(process.execPath, [ + '-e', `const assert = require('node:assert'); const { builtinModules } = require('node:module'); assert(!builtinModules.includes('${id}')); assert(!builtinModules.includes('${nodeBuiltin}')); assert.throws(() => require('${id}'), { code: 'MODULE_NOT_FOUND' }); assert.throws(() => require('${nodeBuiltin}'), { code: 'ERR_UNKNOWN_BUILTIN_MODULE' });`, + ], { status: 0 }); + + spawnSyncAndAssert(process.execPath, [ + flag, + '-e', `const assert = require('node:assert'); const { builtinModules } = require('node:module'); assert(builtinModules.includes('${id}')); assert(!builtinModules.includes('${nodeBuiltin}')); require('${id}'); require('${nodeBuiltin}');`, + ], { status: 0 }); +} + // node:ffi is enabled by default in builds with FFI support and can be // disabled with --no-experimental-ffi. if (common.hasFFI) { From d03e7313cf33ade403de7a8361d5dd90ac2a541a Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Sat, 12 Sep 2026 21:20:47 +0200 Subject: [PATCH 093/217] meta: add joyeecheung as v8 currency strategic initiative champion Signed-off-by: Joyee Cheung PR-URL: https://github.com/nodejs/node/pull/65965 Reviewed-By: Filip Skokan Reviewed-By: Yagiz Nizipli Reviewed-By: Richard Lau Reviewed-By: Robert Nagy Reviewed-By: Chengzhong Wu Reviewed-By: Antoine du Hamel Reviewed-By: Luigi Pinca Reviewed-By: Xuguang Mei Reviewed-By: Marco Ippolito Reviewed-By: Matteo Collina --- doc/contributing/strategic-initiatives.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/contributing/strategic-initiatives.md b/doc/contributing/strategic-initiatives.md index 1b275178f9ee..ce68d17d8909 100644 --- a/doc/contributing/strategic-initiatives.md +++ b/doc/contributing/strategic-initiatives.md @@ -11,7 +11,7 @@ agenda to ensure they are active and have the support they need. | QUIC / HTTP3 | [James M Snell][jasnell] | | | Unified HTTP API | [James M Snell][jasnell] | | | Shadow Realm | [Chengzhong Wu][legendecas] | | -| V8 Currency | | | +| V8 Currency | [Joyee Cheung][joyeecheung] | | | Next-10 | [Jacob Smith][JakobJingleheimer] | | | Single executable apps | [Darshan Sen][RaisinTen] | | | Performance | [Rafael Gonzaga][RafaelGSS] | | From 2bf082453b8ffd224b84e631af47e6aab8b4e097 Mon Sep 17 00:00:00 2001 From: Sergey Sannikov Date: Sat, 12 Sep 2026 23:30:04 +0400 Subject: [PATCH 094/217] assert: fix TypeError on deepStrictEqual with null Map key or Set member deepStrictEqual() and util.isDeepStrictEqual() threw "Cannot read properties of null (reading 'constructor')" instead of comparing when a Map key or Set member was null/undefined (or another primitive) and lined up against object-only keys/members in the other collection with an equal count. The primitive/null handling was gated behind an optimization that is skipped when the counts match, letting such keys reach objectComparisonStart, which dereferences `.constructor`. Resolve primitive and null keys/members directly in every case. Signed-off-by: semx <7532921+semx@users.noreply.github.com> PR-URL: https://github.com/nodejs/node/pull/64449 Reviewed-By: Jordan Harband Reviewed-By: James M Snell --- lib/internal/util/comparisons.js | 16 +++++++++------- test/parallel/test-assert-deep.js | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/lib/internal/util/comparisons.js b/lib/internal/util/comparisons.js index 92594e8994fb..e6273858e890 100644 --- a/lib/internal/util/comparisons.js +++ b/lib/internal/util/comparisons.js @@ -683,16 +683,18 @@ function setObjectEquiv(array, a, b, mode, memo) { const comparator = mode !== kLoose ? objectComparisonStart : innerDeepEqual; const extraChecks = mode === kLoose || array.length !== a.size; for (const val1 of a) { - if (extraChecks) { - if (typeof val1 === 'object') { - if (b.has(val1)) { - continue; - } - } else if (b.has(val1)) { + // Primitive and null members can only match by identity, and must never + // reach objectComparisonStart (which throws on `val.constructor` for + // null/undefined). Resolve them directly for every such member. + if (typeof val1 !== 'object' || val1 === null) { + if (b.has(val1)) { continue; - } else if (mode !== kLoose) { + } + if (mode !== kLoose) { return false; } + } else if (extraChecks && b.has(val1)) { + continue; } let innerStart = start; diff --git a/test/parallel/test-assert-deep.js b/test/parallel/test-assert-deep.js index f7aded0d9aca..91faa7839854 100644 --- a/test/parallel/test-assert-deep.js +++ b/test/parallel/test-assert-deep.js @@ -278,6 +278,10 @@ test('es6 Maps and Sets', () => { assertDeepAndStrictEqual(new Set([[1, 2], [3, 4]]), new Set([[3, 4], [1, 2]])); assertNotDeepOrStrict(new Set([{ a: 0 }]), new Set([{ a: 1 }])); assertNotDeepOrStrict(new Set([Symbol()]), new Set([Symbol()])); + // A null/primitive member lined up against object-only members in the other + // set must report inequality, not throw on `member.constructor`. + assertNotDeepOrStrict(new Set([null, {}, {}]), new Set([{}, {}, {}])); + assertNotDeepOrStrict(new Set([undefined, {}, {}]), new Set([{}, {}, {}])); { const a = [ 1, 2 ]; @@ -298,6 +302,17 @@ test('es6 Maps and Sets', () => { new Map([[[1], 1], [{}, 2]]), new Map([[[1], 2], [{}, 1]]) ); + // A null/primitive key that lines up with object-only keys in the other map + // must report inequality, not throw on `key.constructor`. Refs: object keys + // of `b` equal in count to `a.size` used to skip the primitive-key handling. + assertNotDeepOrStrict( + new Map([[null, 1], [{}, 2]]), + new Map([[{}, 9], [{}, 9]]) + ); + assertNotDeepOrStrict( + new Map([[undefined, 1], [{}, 2]]), + new Map([[{}, 9], [{}, 9]]) + ); assertNotDeepOrStrict(new Set([1]), [1]); assertNotDeepOrStrict(new Set(), []); From 04e5c282a152a4159d2da22597d6a0b17c2e0cc8 Mon Sep 17 00:00:00 2001 From: Caleb Everett Date: Sat, 12 Sep 2026 15:45:11 -0700 Subject: [PATCH 095/217] stream: fix async iteration of undefined chunks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hand-rolled async iterator introduced in https://github.com/nodejs/node/pull/64447 unwraps thenable chunks before delivering them, but guards the `then` access with `chunk !== null` while the dereference itself requires `chunk != null`. An object mode stream carrying an `undefined` chunk therefore threw TypeError: Cannot read properties of undefined (reading 'then') Because the throw happens in a microtask rather than rejecting the iterator's promise, it surfaces as an uncaught exception that terminates the process instead of a catchable stream error. `undefined` is a legal chunk value: doc/api/stream.md documents object mode chunks as "any JavaScript value other than `null`" and reserves `null` as the end-of-stream sentinel. `readable.push(undefined)` returns true in object mode, so the stream accepts the value and then crashes on the way out. Read `then` with optional chaining at both sites so that `undefined` is delivered as a value while `null` keeps signalling end-of-stream. `then` is still read at most once, so a getter cannot observe a second access. Refs: https://github.com/nodejs/node/pull/64447 Assisted-by: a closed-source coding agent Signed-off-by: Caleb ツ Everett PR-URL: https://github.com/nodejs/node/pull/65969 Reviewed-By: Xuguang Mei Reviewed-By: James M Snell --- lib/internal/streams/readable.js | 10 +++-- .../test-stream-readable-async-iterators.js | 38 +++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/lib/internal/streams/readable.js b/lib/internal/streams/readable.js index 847b3837af4c..fa6ed7900831 100644 --- a/lib/internal/streams/readable.js +++ b/lib/internal/streams/readable.js @@ -1522,8 +1522,9 @@ function createAsyncIterator(stream, options) { const chunk = stream.destroyed ? null : stream.read(); if (chunk !== null) { // Read `then` only once so that a getter cannot observe (or throw - // on) a second access. - const then = chunk.then; + // on) a second access. `undefined` is a valid chunk value, so it must + // not be dereferenced here. + const then = chunk?.then; if (typeof then === 'function') { FunctionPrototypeCall(then, chunk, (value) => { inFlight = false; @@ -1595,8 +1596,9 @@ function createAsyncIterator(stream, options) { const chunk = stream.destroyed ? null : stream.read(); if (chunk !== null) { // Read `then` only once so that a getter cannot observe (or - // throw on) a second access. - const then = chunk.then; + // throw on) a second access. `undefined` is a valid chunk value, + // so it must not be dereferenced here. + const then = chunk?.then; if (typeof then === 'function') { inFlight = true; return FunctionPrototypeCall( diff --git a/test/parallel/test-stream-readable-async-iterators.js b/test/parallel/test-stream-readable-async-iterators.js index f9bcaea6057b..fb4b052bf288 100644 --- a/test/parallel/test-stream-readable-async-iterators.js +++ b/test/parallel/test-stream-readable-async-iterators.js @@ -989,5 +989,43 @@ async function tests() { })().then(common.mustCall()); } +{ + // An `undefined` chunk is a value, not end-of-stream. Here it is already + // buffered, so it is read on the synchronous fast path. + (async () => { + const r = new Readable({ objectMode: true, read() {} }); + r.push(undefined); + r.push(null); + + const it = r[Symbol.asyncIterator](); + assert.deepStrictEqual(await it.next(), { done: false, value: undefined }); + assert.strictEqual((await it.next()).done, true); + })().then(common.mustCall()); +} + +{ + // An `undefined` chunk pushed after next() is delivered once it arrives. + (async () => { + const r = new Readable({ objectMode: true, read() {} }); + const it = r[Symbol.asyncIterator](); + const next = it.next(); + setImmediate(() => { + r.push(undefined); + r.push(null); + }); + + assert.deepStrictEqual(await next, { done: false, value: undefined }); + assert.strictEqual((await it.next()).done, true); + })().then(common.mustCall()); +} + +{ + // Readable.from() delivers `undefined` values. + (async () => { + assert.deepStrictEqual(await Readable.from([undefined]).toArray(), + [undefined]); + })().then(common.mustCall()); +} + // To avoid missing some tests if a promise does not resolve tests().then(common.mustCall()); From 9b3085e660fa38a6f89d3c3872e37917a020cbba Mon Sep 17 00:00:00 2001 From: Jihwan Date: Sun, 13 Sep 2026 08:42:59 +0900 Subject: [PATCH 096/217] test_runner: fix quote escaping in JUnit Escape XML content before replacing double quotes and line feeds. Add a snapshot test for repeated quotes, literal quote references, and quotes mixed with ampersands, less-than signs, or a newline. Signed-off-by: hanityx PR-URL: https://github.com/nodejs/node/pull/65971 Refs: https://github.com/nodejs/node/pull/60274 Reviewed-By: Xuguang Mei Reviewed-By: Luigi Pinca --- lib/internal/test_runner/reporter/junit.js | 2 +- test/fixtures/test-runner/output/junit_quote.js | 9 +++++++++ .../test-runner/output/junit_quote.snapshot | 16 ++++++++++++++++ test/test-runner/test-output-junit-quote.mjs | 11 +++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/test-runner/output/junit_quote.js create mode 100644 test/fixtures/test-runner/output/junit_quote.snapshot create mode 100644 test/test-runner/test-output-junit-quote.mjs diff --git a/lib/internal/test_runner/reporter/junit.js b/lib/internal/test_runner/reporter/junit.js index 5052f5444c0d..e5a3fa961eac 100644 --- a/lib/internal/test_runner/reporter/junit.js +++ b/lib/internal/test_runner/reporter/junit.js @@ -22,7 +22,7 @@ const inspectOptions = { __proto__: null, colors: false, breakLength: Infinity } const HOSTNAME = hostname(); function escapeAttribute(s = '') { - return escapeContent(RegExpPrototypeSymbolReplace(/"/g, RegExpPrototypeSymbolReplace(/\n/g, s, ' '), '"')); + return RegExpPrototypeSymbolReplace(/"/g, RegExpPrototypeSymbolReplace(/\n/g, escapeContent(s), ' '), '"'); } function escapeContent(s = '') { diff --git a/test/fixtures/test-runner/output/junit_quote.js b/test/fixtures/test-runner/output/junit_quote.js new file mode 100644 index 000000000000..4f77d9ffbb0e --- /dev/null +++ b/test/fixtures/test-runner/output/junit_quote.js @@ -0,0 +1,9 @@ +// Flags: --test --test-reporter=junit +'use strict'; +const test = require('node:test'); + +test('quote"only', () => {}); +test('quote"only', () => {}); +test('amp&and"quote"', () => {}); +test('lt {}); +test('line\n"break', () => {}); diff --git a/test/fixtures/test-runner/output/junit_quote.snapshot b/test/fixtures/test-runner/output/junit_quote.snapshot new file mode 100644 index 000000000000..48d3ac2c53c2 --- /dev/null +++ b/test/fixtures/test-runner/output/junit_quote.snapshot @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/test/test-runner/test-output-junit-quote.mjs b/test/test-runner/test-output-junit-quote.mjs new file mode 100644 index 000000000000..6d1769eea83f --- /dev/null +++ b/test/test-runner/test-output-junit-quote.mjs @@ -0,0 +1,11 @@ +// Test that the output of test-runner/output/junit_quote.js matches +// test-runner/output/junit_quote.snapshot +import '../common/index.mjs'; +import * as fixtures from '../common/fixtures.mjs'; +import { spawnAndAssert, junitTransform, ensureCwdIsProjectRoot } from '../common/assertSnapshot.js'; + +ensureCwdIsProjectRoot(); +await spawnAndAssert( + fixtures.path('test-runner/output/junit_quote.js'), + junitTransform, +); From e4ca72a984342479527a38ecc76231f42ab4ec3f Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Sun, 13 Sep 2026 09:35:14 +0200 Subject: [PATCH 097/217] test: avoid allocations in external memory test Inspect the parsed V8 flags instead of allocating over a gigabyte and aborting child processes. This avoids memory pressure and core dumps in test-external-memory-reasonable-size. Also verify that Node disables the limit by default. Refs: https://github.com/nodejs/node/pull/65780 Signed-off-by: Filip Skokan Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/65975 Refs: https://github.com/nodejs/reliability/blob/main/reports/2026-09-11.md Reviewed-By: Antoine du Hamel Reviewed-By: Richard Lau Reviewed-By: James M Snell --- .../test-external-memory-reasonable-size.js | 39 ++++++------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/test/parallel/test-external-memory-reasonable-size.js b/test/parallel/test-external-memory-reasonable-size.js index f9ff1af8e5a1..2d79eec892c1 100644 --- a/test/parallel/test-external-memory-reasonable-size.js +++ b/test/parallel/test-external-memory-reasonable-size.js @@ -1,34 +1,19 @@ 'use strict'; -// V8 aborts the process when external memory grows by more than -// --external-memory-max-reasonable-size gigabytes in a single step. Node -// disables that check by default, but an explicit value on the command line -// must still be honored. +// Node.js disables V8's external memory reasonable size check by default, but +// explicit values on the command line must still be honored. // Refs: https://github.com/nodejs/node/issues/65534 -const common = require('../common'); -const assert = require('assert'); -const { execSync } = require('child_process'); -const { totalmem } = require('os'); +require('../common'); +const { spawnSyncAndAssert } = require('../common/child_process'); -// The smallest limit V8 accepts is 1 GB, so the child has to allocate more -// than that before the check can fire. -if (totalmem() < 4 * 1024 ** 3) - common.skip('not enough memory to exceed a 1 GB external memory limit'); - -for (const flag of [ - '--external-memory-max-reasonable-size=1', - '--external_memory_max_reasonable_size=1', +// Despite the "default" label, --v8-options prints the parsed flag values. +for (const [flags, expected] of [ + [[], 0], + [['--external-memory-max-reasonable-size=1'], 1], + [['--external_memory_max_reasonable_size=1'], 1], ]) { - // The child aborts with over a gigabyte resident, so keep it from writing a - // core file; on some hosts that dump alone outlasts the test timeout. - const [cmd, opts] = common.escapePOSIXShell`"${process.execPath}" ${flag} -e "new Float64Array(150_000_000)"`; - assert.throws( - () => execSync(common.isWindows ? cmd : `ulimit -c 0; ${cmd}`, { ...opts, stdio: 'pipe' }), - (err) => { - assert.notStrictEqual(err.status, 0, `${flag} was not honored, the child exited cleanly`); - assert.match(err.stderr.toString(), /kMaxReasonableBytes/); - return true; - }, - ); + spawnSyncAndAssert(process.execPath, [...flags, '--v8-options'], { + stdout: new RegExp(`default: --external-memory-max-reasonable-size=${expected}$`, 'm'), + }); } From 7131e3b437f4584544889d67677ed72aee27574e Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Fri, 11 Sep 2026 13:15:01 +0200 Subject: [PATCH 098/217] test: reduce ZIP64 stress test I/O A leading member just over 4 GiB followed by a small member exercises Zip64 sizes and offsets with less I/O. Reuse source buffers and verify raw Zip64 fields while retaining full readback and file-backed reserialization. Signed-off-by: Filip Skokan Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/65980 Reviewed-By: James M Snell Reviewed-By: LiviaMedeiros --- test/pummel/test-zlib-zip-slow.js | 94 +++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 30 deletions(-) diff --git a/test/pummel/test-zlib-zip-slow.js b/test/pummel/test-zlib-zip-slow.js index c3e334c06dec..ab023456890b 100644 --- a/test/pummel/test-zlib-zip-slow.js +++ b/test/pummel/test-zlib-zip-slow.js @@ -20,33 +20,32 @@ const { test } = require('node:test'); tmpdir.refresh(); const GiB = 1024 * 1024 * 1024; -const MEMBER_SIZE = 500 * 1024 * 1024; // Four ~500 MiB stored members... -const STORED_MEMBER_COUNT = 4; -const STREAMED_MEMBER_SIZE = 4.5 * GiB; // ...plus one >4 GiB streamed member: -// the total archive size (~6.5 GiB) pushes offsets over the 4 GiB Zip64 -// threshold, and the streamed member's own sizes exceed 32 bits too, so the -// per-entry Zip64 size fields (central header and data descriptor) are -// exercised as well as the offset promotion. Required free space includes -// generous slack over that total. -const REQUIRED_FREE_BYTES = 12 * GiB; const CHUNK_SIZE = 16 * 1024 * 1024; - -function fillChunk(seed) { - const chunk = Buffer.allocUnsafe(CHUNK_SIZE); - chunk.fill(seed & 0xff); - return chunk; -} +const STREAMED_MEMBER_SIZE = 4 * GiB + CHUNK_SIZE; +const TAIL_MEMBER_SIZE = 64 * 1024; +// The leading member needs Zip64 sizes; the small member after it needs a +// Zip64 offset. Only one member has to be large to exercise both paths. +const REQUIRED_FREE_BYTES = 8 * GiB; +const CENTRAL_FILE_HEADER_SIGNATURE = Buffer.from([0x50, 0x4b, 0x01, 0x02]); async function* repeatingChunks(totalSize, seed) { + const chunk = Buffer.alloc(Math.min(CHUNK_SIZE, totalSize), seed); let remaining = totalSize; while (remaining > 0) { - const size = Math.min(CHUNK_SIZE, remaining); - const chunk = fillChunk(seed); + const size = Math.min(chunk.length, remaining); remaining -= size; yield size === chunk.length ? chunk : chunk.subarray(0, size); } } +function assertZip64Extra(buffer, offset, values) { + assert.strictEqual(buffer.readUInt16LE(offset), 0x0001); + assert.strictEqual(buffer.readUInt16LE(offset + 2), values.length * 8); + for (let i = 0; i < values.length; i++) { + assert.strictEqual(buffer.readBigUInt64LE(offset + 4 + i * 8), BigInt(values[i])); + } +} + test('an archive larger than 4 GiB round-trips and triggers Zip64 via offset', async () => { let free; try { @@ -63,15 +62,14 @@ test('an archive larger than 4 GiB round-trips and triggers Zip64 via offset', a const dir = await fs.mkdtemp(path.join(tmpdir.path, 'zlib-zip-slow-')); const archivePath = path.join(dir, 'large.zip'); try { - const entries = []; - for (let i = 0; i < STORED_MEMBER_COUNT; i++) { - entries.push(zlib.ZipEntry.createStream(`stored-${i}.bin`, repeatingChunks(MEMBER_SIZE, i), { + const entries = [ + zlib.ZipEntry.createStream('streamed.bin', repeatingChunks(STREAMED_MEMBER_SIZE, 0xaa), { method: 'store', - })); - } - entries.push(zlib.ZipEntry.createStream('streamed.bin', repeatingChunks(STREAMED_MEMBER_SIZE, 0xaa), { - method: 'store', - })); + }), + zlib.ZipEntry.createStream('tail.bin', repeatingChunks(TAIL_MEMBER_SIZE, 2), { + method: 'store', + }), + ]; const handle = await fs.open(archivePath, 'w'); try { @@ -85,9 +83,38 @@ test('an archive larger than 4 GiB round-trips and triggers Zip64 via offset', a const stat = await fs.stat(archivePath); assert.ok(stat.size > 4 * GiB, `archive is only ${stat.size} bytes`); + const reader = await fs.open(archivePath, 'r'); + try { + // Read only the leading local header and the archive tail. The two + // central headers and the trailer fit comfortably in these 1024 bytes. + const local = Buffer.alloc(30); + const tail = Buffer.alloc(1024); + await reader.read(local, 0, local.length, 0); + await reader.read(tail, 0, tail.length, stat.size - tail.length); + assert.strictEqual(local.readUInt32LE(0), 0x04034b50); + const tailOffset = local.length + local.readUInt16LE(26) + + local.readUInt16LE(28) + STREAMED_MEMBER_SIZE + 24; // Zip64 data descriptor. + assert.ok(tailOffset > 0xffffffff); + + const bigCentral = tail.indexOf(CENTRAL_FILE_HEADER_SIGNATURE); + assert.notStrictEqual(bigCentral, -1); + assert.strictEqual(tail.readUInt32LE(bigCentral + 20), 0xffffffff); + assert.strictEqual(tail.readUInt32LE(bigCentral + 24), 0xffffffff); + assertZip64Extra(tail, bigCentral + 46 + tail.readUInt16LE(bigCentral + 28), + [STREAMED_MEMBER_SIZE, STREAMED_MEMBER_SIZE]); + + const tailCentral = tail.indexOf(CENTRAL_FILE_HEADER_SIGNATURE, bigCentral + 4); + assert.notStrictEqual(tailCentral, -1); + assert.strictEqual(tail.readUInt32LE(tailCentral + 42), 0xffffffff); + assertZip64Extra(tail, tailCentral + 46 + tail.readUInt16LE(tailCentral + 28), + [tailOffset]); + } finally { + await reader.close(); + } + const zip = await zlib.ZipFile.open(archivePath); try { - assert.strictEqual(zip.size, STORED_MEMBER_COUNT + 1); + assert.strictEqual(zip.size, 2); let seen = 0; for await (const chunk of await zip.stream('streamed.bin')) { @@ -96,12 +123,12 @@ test('an archive larger than 4 GiB round-trips and triggers Zip64 via offset', a } assert.strictEqual(seen, STREAMED_MEMBER_SIZE); - let storedSeen = 0; - for await (const chunk of await zip.stream('stored-2.bin')) { - storedSeen += chunk.length; + let tailSeen = 0; + for await (const chunk of await zip.stream('tail.bin')) { + tailSeen += chunk.length; assert.strictEqual(chunk[0], 2); } - assert.strictEqual(storedSeen, MEMBER_SIZE); + assert.strictEqual(tailSeen, TAIL_MEMBER_SIZE); // The streamed member's sizes genuinely exceed 32 bits (stored, so // compressed === uncompressed), which the reader must have resolved @@ -116,6 +143,13 @@ test('an archive larger than 4 GiB round-trips and triggers Zip64 via offset', a // descriptor) without needing a second copy on disk. let reserialized = 0; for await (const chunk of zlib.createZipArchive([big])) { + if (reserialized === 0) { + assert.strictEqual(chunk.readUInt16LE(6) & 0x08, 0); // No data descriptor. + assert.strictEqual(chunk.readUInt32LE(18), 0xffffffff); + assert.strictEqual(chunk.readUInt32LE(22), 0xffffffff); + assertZip64Extra(chunk, 30 + chunk.readUInt16LE(26), + [STREAMED_MEMBER_SIZE, STREAMED_MEMBER_SIZE]); + } reserialized += chunk.length; } assert.ok(reserialized > STREAMED_MEMBER_SIZE, From f23847a12259bc29173258f2a90582d7301c4061 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Fri, 11 Sep 2026 13:15:01 +0200 Subject: [PATCH 099/217] test: use named parameters in DH stress test Named modp14 parameters avoid generating and repeatedly validating a custom prime. Keep the existing exchange counts and FIPS rejection assertion. The separate deterministic padding test continues to cover imported prime parameters. Signed-off-by: Filip Skokan Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/65980 Reviewed-By: James M Snell Reviewed-By: LiviaMedeiros --- test/pummel/test-dh-regr.js | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/test/pummel/test-dh-regr.js b/test/pummel/test-dh-regr.js index c442fbc3a809..8a2e71745a38 100644 --- a/test/pummel/test-dh-regr.js +++ b/test/pummel/test-dh-regr.js @@ -34,7 +34,6 @@ const assert = require('assert'); const crypto = require('crypto'); const { hasOpenSSL, hasFIPS } = require('../common/crypto'); -let p; let iterations = 2000; if (hasFIPS(3)) { assert.throws(() => crypto.createDiffieHellman(1024), { @@ -42,22 +41,24 @@ if (hasFIPS(3)) { name: 'TypeError', }); - // Use a precomputed approved group instead of generating a 2048-bit prime - // for every test run. Its larger keys also make each pummel iteration more - // expensive, so use enough iterations to exercise the regression without - // making the FIPS job excessively slow. - p = crypto.getDiffieHellman('modp14').getPrime(); + // Keep a lower iteration count for FIPS jobs. iterations = 100; +} + +let createDH; +if (hasOpenSSL(3)) { + // OpenSSL 3 recognizes named groups without validating their primes. + createDH = () => crypto.getDiffieHellman('modp14'); } else { - // FIPS requires length >= 1024, but small parameters keep this pummel test - // from timing out in ordinary CI. - const length = crypto.getFips() === 1 ? 1024 : (hasOpenSSL(3) ? 512 : 256); - p = crypto.createDiffieHellman(length).getPrime(); + // Other backends validate each peer's parameters, so keep them small. + const length = crypto.getFips() === 1 ? 1024 : 256; + const prime = crypto.createDiffieHellman(length).getPrime(); + createDH = () => crypto.createDiffieHellman(prime); } for (let i = 0; i < iterations; i++) { - const a = crypto.createDiffieHellman(p); - const b = crypto.createDiffieHellman(p); + const a = createDH(); + const b = createDH(); a.generateKeys(); b.generateKeys(); From f523342bcd8208208628ed3c372b64de4e8077f5 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Fri, 11 Sep 2026 13:15:01 +0200 Subject: [PATCH 100/217] test: close WebAssembly test HTTP servers Unreferencing listeners leaves accepted connections alive until the keep-alive timeout. Close each single-use connection and its server when the response closes, including intentionally destroyed responses. Signed-off-by: Filip Skokan Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/65980 Reviewed-By: James M Snell Reviewed-By: LiviaMedeiros --- test/es-module/test-wasm-web-api.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/es-module/test-wasm-web-api.js b/test/es-module/test-wasm-web-api.js index ee1971133be5..6177c909e56c 100644 --- a/test/es-module/test-wasm-web-api.js +++ b/test/es-module/test-wasm-web-api.js @@ -16,7 +16,13 @@ const simpleWasmBytes = fixtures.readSync('simple.wasm'); // Sets up an HTTP server with the given response handler and calls fetch() to // obtain a Response from the newly created server. async function testRequest(handler) { - const server = createServer((_, res) => handler(res)).unref().listen(0); + const server = createServer(common.mustCall((_, res) => { + res.setHeader('Connection', 'close'); + res.once('close', common.mustCall(() => { + server.close(common.mustCall()); + })); + handler(res); + })).listen(0); await events.once(server, 'listening'); const { port } = server.address(); return fetch(`http://127.0.0.1:${port}/foo.wasm`); From 6314ccf93b2674b9c99243ae695d2d3655123add Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Fri, 11 Sep 2026 13:15:01 +0200 Subject: [PATCH 101/217] tools: reduce test runner timing overhead RunProcess sleeps after polling even when the child has already exited. Skip that sleep, saving up to 100 ms per test. Sort --time results in descending order to display the 20 slowest tests. Signed-off-by: Filip Skokan Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/65980 Reviewed-By: James M Snell Reviewed-By: LiviaMedeiros --- tools/test.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tools/test.py b/tools/test.py index 2c2a4d78d80a..a20c44e52516 100755 --- a/tools/test.py +++ b/tools/test.py @@ -741,10 +741,11 @@ def RunProcess(context, timeout, args, **rest): timed_out = True else: exit_code = process.poll() - time.sleep(sleep_time) - sleep_time = sleep_time * SLEEP_TIME_FACTOR - if sleep_time > MAX_SLEEP_TIME: - sleep_time = MAX_SLEEP_TIME + if exit_code is None: + time.sleep(sleep_time) + sleep_time = sleep_time * SLEEP_TIME_FACTOR + if sleep_time > MAX_SLEEP_TIME: + sleep_time = MAX_SLEEP_TIME return (process, exit_code, timed_out) @@ -1849,7 +1850,7 @@ def should_keep(case): print() sys.stderr.write("--- Total time: %s ---\n" % FormatTime(duration)) timed_tests = [ t for t in cases_to_run if not t.duration is None ] - timed_tests.sort(key=lambda x: x.duration) + timed_tests.sort(key=lambda x: x.duration, reverse=True) for i, entry in enumerate(timed_tests[:20], start=1): t = FormatTimedelta(entry.duration) sys.stderr.write("%4i (%s) %s\n" % (i, t, entry.GetLabel())) From 831ec42bcc7a8b31b62dba0a1623b9bce5c4ab94 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Fri, 11 Sep 2026 13:32:54 +0200 Subject: [PATCH 102/217] test: avoid idle HTTP/HTTPS connections Single-use requests otherwise wait for the keep-alive timeout. Use nonpersistent agents where agent selection is unrelated to coverage. For default-agent tests, close the server after consuming the response. Signed-off-by: Filip Skokan Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/65980 Reviewed-By: James M Snell Reviewed-By: LiviaMedeiros --- test/parallel/test-http-buffer-sanity.js | 1 + test/parallel/test-http-byteswritten.js | 2 +- test/parallel/test-http-client-check-http-token.js | 2 +- test/parallel/test-http-client-encoding.js | 1 + test/parallel/test-http-client-response-domain.js | 1 + test/parallel/test-http-decoded-auth.js | 2 +- test/parallel/test-http-default-port.js | 2 +- .../test-http-dont-set-default-headers-with-setHost.js | 1 + test/parallel/test-http-dont-set-default-headers.js | 1 + test/parallel/test-http-early-hints-invalid-argument.js | 4 ++-- test/parallel/test-http-head-request.js | 1 + test/parallel/test-http-hex-write.js | 2 +- test/parallel/test-http-outgoing-end-types.js | 2 +- test/parallel/test-http-outgoing-finish-writable.js | 1 + test/parallel/test-http-outgoing-finish.js | 1 + test/parallel/test-http-outgoing-properties.js | 2 ++ test/parallel/test-http-outgoing-write-types.js | 2 +- test/parallel/test-http-request-arguments.js | 2 +- test/parallel/test-http-request-large-payload.js | 1 + test/parallel/test-http-server-connection-list-when-close.js | 1 + test/parallel/test-http-server-delete-parser.js | 1 + test/parallel/test-http-server-multiheaders.js | 1 + test/parallel/test-http-server-multiheaders2.js | 1 + .../test-http-url.parse-auth-with-header-in-request.js | 1 + test/parallel/test-http-url.parse-auth.js | 1 + test/parallel/test-http-url.parse-basic.js | 5 ++++- test/parallel/test-http-url.parse-https.request.js | 5 ++++- test/parallel/test-http-url.parse-path.js | 1 + test/parallel/test-http-url.parse-post.js | 1 + test/parallel/test-http-url.parse-search.js | 1 + test/parallel/test-http-write-callbacks.js | 1 + test/parallel/test-http-write-empty-string.js | 2 +- test/parallel/test-http-zero-length-write.js | 2 +- test/parallel/test-https-drain.js | 1 + test/parallel/test-https-request-arguments.js | 1 + test/parallel/test-https-truncate.js | 2 +- test/parallel/test-https-unix-socket-self-signed.js | 1 + 37 files changed, 45 insertions(+), 15 deletions(-) diff --git a/test/parallel/test-http-buffer-sanity.js b/test/parallel/test-http-buffer-sanity.js index a235f3793a4f..e122976f0e3f 100644 --- a/test/parallel/test-http-buffer-sanity.js +++ b/test/parallel/test-http-buffer-sanity.js @@ -55,6 +55,7 @@ const server = http.Server(common.mustCallAtLeast(function(req, res) { server.listen(0, common.mustCall(() => { const req = http.request({ + agent: false, port: server.address().port, method: 'POST', path: '/', diff --git a/test/parallel/test-http-byteswritten.js b/test/parallel/test-http-byteswritten.js index 003b7dfbd049..475176e6c976 100644 --- a/test/parallel/test-http-byteswritten.js +++ b/test/parallel/test-http-byteswritten.js @@ -51,5 +51,5 @@ const httpServer = http.createServer(common.mustCall(function(req, res) { })); httpServer.listen(0, function() { - http.get({ port: this.address().port }); + http.get({ port: this.address().port, agent: false }); }); diff --git a/test/parallel/test-http-client-check-http-token.js b/test/parallel/test-http-client-check-http-token.js index ef2445ec66e5..7ab9aa83d761 100644 --- a/test/parallel/test-http-client-check-http-token.js +++ b/test/parallel/test-http-client-check-http-token.js @@ -29,6 +29,6 @@ server.listen(0, common.mustCall(() => { }); expectedSuccesses.forEach((method) => { - http.request({ method, port: server.address().port }).end(); + http.request({ method, port: server.address().port, agent: false }).end(); }); })); diff --git a/test/parallel/test-http-client-encoding.js b/test/parallel/test-http-client-encoding.js index a4701cdbd0ab..253496307286 100644 --- a/test/parallel/test-http-client-encoding.js +++ b/test/parallel/test-http-client-encoding.js @@ -29,6 +29,7 @@ const server = http.createServer((req, res) => { server.close(); }).listen(0, common.mustCall(() => { http.request({ + agent: false, port: server.address().port, encoding: 'utf8' }, common.mustCall((res) => { diff --git a/test/parallel/test-http-client-response-domain.js b/test/parallel/test-http-client-response-domain.js index 9975ca3f9498..da3d3a09ff0f 100644 --- a/test/parallel/test-http-client-response-domain.js +++ b/test/parallel/test-http-client-response-domain.js @@ -49,6 +49,7 @@ function test() { })); const req = http.get({ + agent: false, socketPath: common.PIPE, headers: { 'Content-Length': '1' }, method: 'POST', diff --git a/test/parallel/test-http-decoded-auth.js b/test/parallel/test-http-decoded-auth.js index 076c056253b6..4f7847133f58 100644 --- a/test/parallel/test-http-decoded-auth.js +++ b/test/parallel/test-http-decoded-auth.js @@ -43,6 +43,6 @@ for (const testCase of testCases) { server.listen(0, function() { // make the request const url = new URL(`http://${testCase.username}:${testCase.password}@localhost:${this.address().port}`); - http.request(url).end(); + http.request(url, { agent: false }).end(); }); } diff --git a/test/parallel/test-http-default-port.js b/test/parallel/test-http-default-port.js index 2005487502fe..874affcdf23c 100644 --- a/test/parallel/test-http-default-port.js +++ b/test/parallel/test-http-default-port.js @@ -44,7 +44,6 @@ for (const { mod, createServer } of [ assert.strictEqual(req.headers['x-port'], `${server.address().port}`); res.writeHead(200); res.end('ok'); - server.close(); })).listen(0, common.mustCall(() => { mod.globalAgent.defaultPort = server.address().port; mod.get({ @@ -54,6 +53,7 @@ for (const { mod, createServer } of [ 'x-port': server.address().port } }, common.mustCall((res) => { + res.on('end', common.mustCall(() => server.close())); res.resume(); })); })); diff --git a/test/parallel/test-http-dont-set-default-headers-with-setHost.js b/test/parallel/test-http-dont-set-default-headers-with-setHost.js index e2a4e39c24b8..418051127859 100644 --- a/test/parallel/test-http-dont-set-default-headers-with-setHost.js +++ b/test/parallel/test-http-dont-set-default-headers-with-setHost.js @@ -14,6 +14,7 @@ const server = http.createServer(common.mustCall(function(req, res) { })); server.listen(0, common.localhostIPv4, function() { http.request({ + agent: false, method: 'POST', host: common.localhostIPv4, port: this.address().port, diff --git a/test/parallel/test-http-dont-set-default-headers.js b/test/parallel/test-http-dont-set-default-headers.js index 3f73c11e5112..0b8e4c58f56b 100644 --- a/test/parallel/test-http-dont-set-default-headers.js +++ b/test/parallel/test-http-dont-set-default-headers.js @@ -17,6 +17,7 @@ const server = http.createServer(common.mustCall(function(req, res) { })); server.listen(0, common.localhostIPv4, function() { http.request({ + agent: false, method: 'POST', host: common.localhostIPv4, port: this.address().port, diff --git a/test/parallel/test-http-early-hints-invalid-argument.js b/test/parallel/test-http-early-hints-invalid-argument.js index edf613614bc7..b426ca3e840f 100644 --- a/test/parallel/test-http-early-hints-invalid-argument.js +++ b/test/parallel/test-http-early-hints-invalid-argument.js @@ -38,7 +38,7 @@ const testResBody = 'response content\n'; server.listen(0, common.mustCall(() => { const req = http.request({ - port: server.address().port, path: '/' + port: server.address().port, path: '/', agent: false }); req.end(); @@ -79,7 +79,7 @@ const testResBody = 'response content\n'; server.listen(0, common.mustCall(() => { const req = http.request({ - port: server.address().port, path: '/' + port: server.address().port, path: '/', agent: false }); req.end(); diff --git a/test/parallel/test-http-head-request.js b/test/parallel/test-http-head-request.js index 26d490d357dc..a9fcb2c166b3 100644 --- a/test/parallel/test-http-head-request.js +++ b/test/parallel/test-http-head-request.js @@ -35,6 +35,7 @@ function test(headers) { server.listen(0, common.mustCall(function() { const request = http.request({ + agent: false, port: this.address().port, method: 'HEAD', path: '/' diff --git a/test/parallel/test-http-hex-write.js b/test/parallel/test-http-hex-write.js index a3cbec6b36c0..4162811276d9 100644 --- a/test/parallel/test-http-hex-write.js +++ b/test/parallel/test-http-hex-write.js @@ -34,7 +34,7 @@ http.createServer(function(q, s) { s.end(); this.close(); }).listen(0, common.mustCall(function() { - http.request({ port: this.address().port }) + http.request({ port: this.address().port, agent: false }) .on('response', common.mustCall(function(res) { let data = ''; diff --git a/test/parallel/test-http-outgoing-end-types.js b/test/parallel/test-http-outgoing-end-types.js index 20b443bff2c1..48372a98e81b 100644 --- a/test/parallel/test-http-outgoing-end-types.js +++ b/test/parallel/test-http-outgoing-end-types.js @@ -14,5 +14,5 @@ const httpServer = http.createServer(common.mustCall(function(req, res) { })); httpServer.listen(0, common.mustCall(function() { - http.get({ port: this.address().port }); + http.get({ port: this.address().port, agent: false }); })); diff --git a/test/parallel/test-http-outgoing-finish-writable.js b/test/parallel/test-http-outgoing-finish-writable.js index e3c870164bac..e0d9b73702cf 100644 --- a/test/parallel/test-http-outgoing-finish-writable.js +++ b/test/parallel/test-http-outgoing-finish-writable.js @@ -25,6 +25,7 @@ server.listen(0); server.on('listening', common.mustCall(function() { const clientRequest = http.request({ + agent: false, port: server.address().port, method: 'GET', path: '/' diff --git a/test/parallel/test-http-outgoing-finish.js b/test/parallel/test-http-outgoing-finish.js index 0f71cccdf810..f2378d9e05b2 100644 --- a/test/parallel/test-http-outgoing-finish.js +++ b/test/parallel/test-http-outgoing-finish.js @@ -33,6 +33,7 @@ http.createServer(function(req, res) { this.close(); }).listen(0, function() { const req = http.request({ + agent: false, port: this.address().port, method: 'PUT' }); diff --git a/test/parallel/test-http-outgoing-properties.js b/test/parallel/test-http-outgoing-properties.js index 85c5b659a36d..a831765322b8 100644 --- a/test/parallel/test-http-outgoing-properties.js +++ b/test/parallel/test-http-outgoing-properties.js @@ -36,6 +36,7 @@ const OutgoingMessage = http.OutgoingMessage; server.on('listening', common.mustCall(function() { const clientRequest = http.request({ + agent: false, port: server.address().port, method: 'GET', path: '/' @@ -62,6 +63,7 @@ const OutgoingMessage = http.OutgoingMessage; server.on('listening', common.mustCall(() => { const req = http.request({ + agent: false, port: server.address().port, method: 'GET', path: '/' diff --git a/test/parallel/test-http-outgoing-write-types.js b/test/parallel/test-http-outgoing-write-types.js index 6257b87eea8e..0f2c686d5a7a 100644 --- a/test/parallel/test-http-outgoing-write-types.js +++ b/test/parallel/test-http-outgoing-write-types.js @@ -20,5 +20,5 @@ const httpServer = http.createServer(common.mustCall(function(req, res) { })); httpServer.listen(0, common.mustCall(function() { - http.get({ port: this.address().port }); + http.get({ port: this.address().port, agent: false }); })); diff --git a/test/parallel/test-http-request-arguments.js b/test/parallel/test-http-request-arguments.js index 5cdd514fd506..b08da9bc5256 100644 --- a/test/parallel/test-http-request-arguments.js +++ b/test/parallel/test-http-request-arguments.js @@ -18,7 +18,7 @@ const http = require('http'); common.mustCall(() => { http.get( 'http://example.com/testpath', - { hostname: 'localhost', port: server.address().port }, + { hostname: 'localhost', port: server.address().port, agent: false }, common.mustCall((res) => { res.resume(); }) diff --git a/test/parallel/test-http-request-large-payload.js b/test/parallel/test-http-request-large-payload.js index 3be100b74041..08fada1381f2 100644 --- a/test/parallel/test-http-request-large-payload.js +++ b/test/parallel/test-http-request-large-payload.js @@ -16,6 +16,7 @@ const server = http.createServer(function(req, res) { server.listen(0, function() { const req = http.request({ + agent: false, method: 'POST', port: this.address().port }); diff --git a/test/parallel/test-http-server-connection-list-when-close.js b/test/parallel/test-http-server-connection-list-when-close.js index a530b710c490..0c8308b63c53 100644 --- a/test/parallel/test-http-server-connection-list-when-close.js +++ b/test/parallel/test-http-server-connection-list-when-close.js @@ -5,6 +5,7 @@ const http = require('http'); function request(server) { http.get({ + agent: false, port: server.address().port, path: '/', }, (res) => { diff --git a/test/parallel/test-http-server-delete-parser.js b/test/parallel/test-http-server-delete-parser.js index 4215ee2f9df7..6b5a3e13f503 100644 --- a/test/parallel/test-http-server-delete-parser.js +++ b/test/parallel/test-http-server-delete-parser.js @@ -14,6 +14,7 @@ const server = http.createServer(common.mustCall((req, res) => { server.listen(0, '127.0.0.1', common.mustCall(() => { const req = http.request({ + agent: false, port: server.address().port, host: '127.0.0.1', method: 'GET', diff --git a/test/parallel/test-http-server-multiheaders.js b/test/parallel/test-http-server-multiheaders.js index fea84a8d4a7e..e15dbd0fcaed 100644 --- a/test/parallel/test-http-server-multiheaders.js +++ b/test/parallel/test-http-server-multiheaders.js @@ -48,6 +48,7 @@ const server = http.createServer(common.mustCall((req, res) => { server.listen(0, function() { http.get({ + agent: false, host: 'localhost', port: this.address().port, path: '/', diff --git a/test/parallel/test-http-server-multiheaders2.js b/test/parallel/test-http-server-multiheaders2.js index 0408afa1b13e..85f2fb09f931 100644 --- a/test/parallel/test-http-server-multiheaders2.js +++ b/test/parallel/test-http-server-multiheaders2.js @@ -100,6 +100,7 @@ const headers = [] server.listen(0, function() { http.get({ + agent: false, host: 'localhost', port: this.address().port, path: '/', diff --git a/test/parallel/test-http-url.parse-auth-with-header-in-request.js b/test/parallel/test-http-url.parse-auth-with-header-in-request.js index ea5793ee18ae..e4834c32a644 100644 --- a/test/parallel/test-http-url.parse-auth-with-header-in-request.js +++ b/test/parallel/test-http-url.parse-auth-with-header-in-request.js @@ -41,6 +41,7 @@ const server = http.createServer(function(request, response) { server.listen(0, function() { const testURL = url.parse(`http://asdf:qwer@localhost:${this.address().port}`); + testURL.agent = false; // The test here is if you set a specific authorization header in the // request we should not override that with basic auth testURL.headers = { diff --git a/test/parallel/test-http-url.parse-auth.js b/test/parallel/test-http-url.parse-auth.js index 2bb531158645..287c27b9eb91 100644 --- a/test/parallel/test-http-url.parse-auth.js +++ b/test/parallel/test-http-url.parse-auth.js @@ -42,6 +42,7 @@ server.listen(0, function() { const port = this.address().port; // username = "user", password = "pass:" const testURL = url.parse(`http://user:pass%3A@localhost:${port}`); + testURL.agent = false; // make the request http.request(testURL).end(); diff --git a/test/parallel/test-http-url.parse-basic.js b/test/parallel/test-http-url.parse-basic.js index d0c230977178..223d1d7af25a 100644 --- a/test/parallel/test-http-url.parse-basic.js +++ b/test/parallel/test-http-url.parse-basic.js @@ -43,7 +43,6 @@ const server = http.createServer(function(request, response) { check(request); response.writeHead(200, {}); response.end('ok'); - server.close(); }); server.listen(0, common.mustCall(function() { @@ -54,5 +53,9 @@ server.listen(0, common.mustCall(function() { // Since there is a little magic with the agent // make sure that an http request uses the http.Agent assert.ok(clientRequest.agent instanceof http.Agent); + clientRequest.on('response', common.mustCall((response) => { + response.on('end', common.mustCall(() => server.close())); + response.resume(); + })); clientRequest.end(); })); diff --git a/test/parallel/test-http-url.parse-https.request.js b/test/parallel/test-http-url.parse-https.request.js index ff819adc2b84..e20c3a0ec7bc 100644 --- a/test/parallel/test-http-url.parse-https.request.js +++ b/test/parallel/test-http-url.parse-https.request.js @@ -45,7 +45,6 @@ const server = https.createServer(httpsOptions, function(request, response) { check(request); response.writeHead(200, {}); response.end('ok'); - server.close(); }); server.listen(0, common.mustCall(function() { @@ -57,5 +56,9 @@ server.listen(0, common.mustCall(function() { // Since there is a little magic with the agent // make sure that the request uses the https.Agent assert.ok(clientRequest.agent instanceof https.Agent); + clientRequest.on('response', common.mustCall((response) => { + response.on('end', common.mustCall(() => server.close())); + response.resume(); + })); clientRequest.end(); })); diff --git a/test/parallel/test-http-url.parse-path.js b/test/parallel/test-http-url.parse-path.js index 25e4838c4afa..04fe12a4ff1f 100644 --- a/test/parallel/test-http-url.parse-path.js +++ b/test/parallel/test-http-url.parse-path.js @@ -40,6 +40,7 @@ const server = http.createServer(function(request, response) { server.listen(0, function() { const testURL = url.parse(`http://localhost:${this.address().port}/asdf`); + testURL.agent = false; // make the request http.request(testURL).end(); diff --git a/test/parallel/test-http-url.parse-post.js b/test/parallel/test-http-url.parse-post.js index db5ee78fe6eb..447a1b6a3fdc 100644 --- a/test/parallel/test-http-url.parse-post.js +++ b/test/parallel/test-http-url.parse-post.js @@ -47,6 +47,7 @@ const server = http.createServer(function(request, response) { server.listen(0, function() { testURL = url.parse(`http://localhost:${this.address().port}/asdf?qwer=zxcv`); + testURL.agent = false; testURL.method = 'POST'; // make the request diff --git a/test/parallel/test-http-url.parse-search.js b/test/parallel/test-http-url.parse-search.js index 0759c779d3ff..80f435a6789a 100644 --- a/test/parallel/test-http-url.parse-search.js +++ b/test/parallel/test-http-url.parse-search.js @@ -41,6 +41,7 @@ const server = http.createServer(function(request, response) { server.listen(0, function() { const port = this.address().port; const testURL = url.parse(`http://localhost:${port}/asdf?qwer=zxcv`); + testURL.agent = false; // make the request http.request(testURL).end(); diff --git a/test/parallel/test-http-write-callbacks.js b/test/parallel/test-http-write-callbacks.js index 1f90e5135be6..3b29f7c2f5d6 100644 --- a/test/parallel/test-http-write-callbacks.js +++ b/test/parallel/test-http-write-callbacks.js @@ -71,6 +71,7 @@ server.on('checkContinue', common.mustCall((req, res) => { server.listen(0, common.mustCall(function() { const req = http.request({ + agent: false, port: this.address().port, method: 'PUT', headers: { 'expect': '100-continue' } diff --git a/test/parallel/test-http-write-empty-string.js b/test/parallel/test-http-write-empty-string.js index 88eff08f7666..05e97a4865c3 100644 --- a/test/parallel/test-http-write-empty-string.js +++ b/test/parallel/test-http-write-empty-string.js @@ -39,7 +39,7 @@ const server = http.createServer(function(request, response) { }); server.listen(0, common.mustCall(() => { - http.get({ port: server.address().port }, common.mustCall((res) => { + http.get({ port: server.address().port, agent: false }, common.mustCall((res) => { let response = ''; assert.strictEqual(res.statusCode, 200); diff --git a/test/parallel/test-http-zero-length-write.js b/test/parallel/test-http-zero-length-write.js index dfaa7b92fb7d..92905fd9755a 100644 --- a/test/parallel/test-http-zero-length-write.js +++ b/test/parallel/test-http-zero-length-write.js @@ -75,7 +75,7 @@ const server = http.createServer(common.mustCall((req, res) => { })); server.listen(0, common.mustCall(function() { - const req = http.request({ port: this.address().port, method: 'POST' }); + const req = http.request({ port: this.address().port, method: 'POST', agent: false }); let actual = ''; req.on('response', common.mustCall((res) => { res.setEncoding('utf8'); diff --git a/test/parallel/test-https-drain.js b/test/parallel/test-https-drain.js index 5d7bf9736458..b9a5c3d3bda8 100644 --- a/test/parallel/test-https-drain.js +++ b/test/parallel/test-https-drain.js @@ -45,6 +45,7 @@ const server = https.createServer(options, function(req, res) { server.listen(0, common.mustCall(function() { let resumed = false; const req = https.request({ + agent: false, method: 'POST', port: this.address().port, rejectUnauthorized: false diff --git a/test/parallel/test-https-request-arguments.js b/test/parallel/test-https-request-arguments.js index 9dc80094be0d..e68f757be81b 100644 --- a/test/parallel/test-https-request-arguments.js +++ b/test/parallel/test-https-request-arguments.js @@ -32,6 +32,7 @@ const options = { 'https://example.com/testpath', { + agent: false, hostname: 'localhost', port: server.address().port, rejectUnauthorized: false diff --git a/test/parallel/test-https-truncate.js b/test/parallel/test-https-truncate.js index beed36cd7c08..eaaedea1afc7 100644 --- a/test/parallel/test-https-truncate.js +++ b/test/parallel/test-https-truncate.js @@ -47,7 +47,7 @@ function httpsTest() { }); server.listen(0, function() { - const opts = { port: this.address().port, rejectUnauthorized: false }; + const opts = { port: this.address().port, rejectUnauthorized: false, agent: false }; https.get(opts).on('response', function(res) { test(res); }); diff --git a/test/parallel/test-https-unix-socket-self-signed.js b/test/parallel/test-https-unix-socket-self-signed.js index 9db92ac2aed4..5a3b76e11794 100644 --- a/test/parallel/test-https-unix-socket-self-signed.js +++ b/test/parallel/test-https-unix-socket-self-signed.js @@ -21,6 +21,7 @@ const server = https.createServer(options, common.mustCall((req, res) => { server.listen(common.PIPE, common.mustCall(() => { https.get({ + agent: false, socketPath: common.PIPE, rejectUnauthorized: false }); From 37f24eddc9550c22428e9ea65778a86eff975edd Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Fri, 11 Sep 2026 13:32:54 +0200 Subject: [PATCH 103/217] test: reuse fixed primes in DH tests Use the modp14 prime instead of generating fresh parameters for tests of constructors, key setters, and memory retention. Keep generic DiffieHellman instances and the existing setter and leak assertions. Signed-off-by: Filip Skokan Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/65980 Reviewed-By: James M Snell Reviewed-By: LiviaMedeiros --- test/parallel/test-crypto-dh-constructor.js | 9 +++------ test/parallel/test-crypto-dh-generate-keys.js | 6 ++---- test/parallel/test-crypto-dh-leak.js | 6 ++---- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/test/parallel/test-crypto-dh-constructor.js b/test/parallel/test-crypto-dh-constructor.js index 28747ac3a726..edf7ab08e44c 100644 --- a/test/parallel/test-crypto-dh-constructor.js +++ b/test/parallel/test-crypto-dh-constructor.js @@ -5,17 +5,14 @@ if (!common.hasCrypto) const assert = require('assert'); const crypto = require('crypto'); -const { hasOpenSSL, hasFIPS } = require('../common/crypto'); +const { hasFIPS } = require('../common/crypto'); -const size = hasFIPS(3) ? - 2048 : (crypto.getFips() === 1 || hasOpenSSL(3) ? 1024 : 256); -const dh1 = crypto.createDiffieHellman(size); -const p1 = dh1.getPrime('buffer'); +const prime = crypto.getDiffieHellman('modp14').getPrime('buffer'); { const DiffieHellman = crypto.DiffieHellman; - const dh = DiffieHellman(p1, 'buffer'); + const dh = DiffieHellman(prime, 'buffer'); assert(dh instanceof DiffieHellman, 'DiffieHellman is expected to return a ' + 'new instance when called without `new`'); } diff --git a/test/parallel/test-crypto-dh-generate-keys.js b/test/parallel/test-crypto-dh-generate-keys.js index d074ba957516..65efb369c08e 100644 --- a/test/parallel/test-crypto-dh-generate-keys.js +++ b/test/parallel/test-crypto-dh-generate-keys.js @@ -6,11 +6,9 @@ if (!common.hasCrypto) const assert = require('assert'); const crypto = require('crypto'); -const { hasOpenSSL, hasFIPS } = require('../common/crypto'); { - const size = hasFIPS(3) ? - 2048 : (crypto.getFips() === 1 || hasOpenSSL(3) ? 1024 : 256); + const prime = crypto.getDiffieHellman('modp14').getPrime(); function unlessInvalidState(f) { try { @@ -23,7 +21,7 @@ const { hasOpenSSL, hasFIPS } = require('../common/crypto'); } function testGenerateKeysChangesKeys(setup, expected) { - const dh = crypto.createDiffieHellman(size); + const dh = crypto.createDiffieHellman(prime); setup(dh); const firstPublicKey = unlessInvalidState(() => dh.getPublicKey()); const firstPrivateKey = unlessInvalidState(() => dh.getPrivateKey()); diff --git a/test/parallel/test-crypto-dh-leak.js b/test/parallel/test-crypto-dh-leak.js index 8d5141eef4b1..f58a5a152b78 100644 --- a/test/parallel/test-crypto-dh-leak.js +++ b/test/parallel/test-crypto-dh-leak.js @@ -9,13 +9,11 @@ if (common.isASan) const assert = require('assert'); const crypto = require('crypto'); -const { hasOpenSSL, hasFIPS } = require('../common/crypto'); const before = process.memoryUsage.rss(); { - const size = hasFIPS(3) ? - 2048 : (crypto.getFips() === 1 || hasOpenSSL(3) ? 1024 : 256); - const dh = crypto.createDiffieHellman(size); + const prime = crypto.getDiffieHellman('modp14').getPrime(); + const dh = crypto.createDiffieHellman(prime); const publicKey = dh.generateKeys(); const privateKey = dh.getPrivateKey(); for (let i = 0; i < 5e4; i += 1) { From 5c7e6b4a6e4773f3fefdf899e5818fdccac579d6 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Fri, 11 Sep 2026 13:32:54 +0200 Subject: [PATCH 104/217] test: synchronize ordered runner events Release the slow fixture over a local socket after the fast fixture emits its bypassed completion event. This removes the fixed 30-second delay while preserving event-order assertions and a bounded failure timeout. Signed-off-by: Filip Skokan Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/65980 Reviewed-By: James M Snell Reviewed-By: LiviaMedeiros --- .../execution-ordered-bypass/slow.mjs | 12 +++++---- .../test-runner-execution-ordered-bypass.mjs | 25 ++++++++++++++++--- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/test/fixtures/test-runner/execution-ordered-bypass/slow.mjs b/test/fixtures/test-runner/execution-ordered-bypass/slow.mjs index 4ee60ffe8537..21f6f2b68676 100644 --- a/test/fixtures/test-runner/execution-ordered-bypass/slow.mjs +++ b/test/fixtures/test-runner/execution-ordered-bypass/slow.mjs @@ -1,9 +1,11 @@ import { test } from 'node:test'; -import { setTimeout as sleep } from 'node:timers/promises'; +import { once } from 'node:events'; +import { connect } from 'node:net'; test('slow', async () => { - // Long enough that fast-fail's process can spawn, run, and round-trip its - // bypassed test:complete to the host on slow CI, but short enough that the - // test does not waste much time when the bypass is working. - await sleep(30_000); + // The host closes this connection after receiving fast-fail's bypassed + // test:complete event, so this test cannot finish before that event arrives. + const socket = connect(Number(process.argv[2]), '127.0.0.1'); + socket.resume(); + await once(socket, 'end'); }); diff --git a/test/parallel/test-runner-execution-ordered-bypass.mjs b/test/parallel/test-runner-execution-ordered-bypass.mjs index ac1c97ee007d..75e9eb51d7d9 100644 --- a/test/parallel/test-runner-execution-ordered-bypass.mjs +++ b/test/parallel/test-runner-execution-ordered-bypass.mjs @@ -1,8 +1,10 @@ // Flags: --no-warnings -import '../common/index.mjs'; +import { mustCall, platformTimeout } from '../common/index.mjs'; import * as fixtures from '../common/fixtures.mjs'; import assert from 'node:assert'; +import { once } from 'node:events'; +import { createServer } from 'node:net'; import { test, run } from 'node:test'; const files = [ @@ -10,15 +12,29 @@ const files = [ fixtures.path('test-runner', 'execution-ordered-bypass', 'fast-fail.mjs'), ]; -test('execution-ordered events bypass FileTest declaration-order buffer', async () => { +test('execution-ordered events bypass FileTest declaration-order buffer', { + timeout: platformTimeout(30_000), +}, async (t) => { + const { promise: fastCompleted, resolve: releaseSlow } = Promise.withResolvers(); + const server = createServer(mustCall((socket) => { + t.after(() => socket.destroy()); + fastCompleted.then(mustCall(() => { + socket.end(); + })); + })); + t.after(() => server.close()); + await once(server.listen(0, '127.0.0.1'), 'listening'); + // Concurrency must be a number so the runner does not collapse it to 1 on // single-core CI runners (where `concurrency: true` resolves to // `availableParallelism() - 1`). Without two slots the runner spawns the - // files sequentially and fast-fail never starts while slow is sleeping. + // files sequentially and fast-fail never starts while slow is waiting. const stream = run({ files, isolation: 'process', concurrency: 2, + argv: [String(server.address().port)], + signal: t.signal, }); const events = []; @@ -27,6 +43,9 @@ test('execution-ordered events bypass FileTest declaration-order buffer', async if (data.name === 'slow' || data.name === 'fast-fail') { events.push(`complete:${data.name}`); } + if (data.name === 'fast-fail') { + releaseSlow(); + } }); stream.on('test:fail', (data) => { From 1a15aff897c86c18b5b5873a5c1f8a4feedf10a7 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Fri, 11 Sep 2026 13:32:55 +0200 Subject: [PATCH 105/217] test: skip retries in DNS timeout coverage A single query attempt exercises the configured timeout without the default retry backoff. Retry behavior has separate coverage. Signed-off-by: Filip Skokan Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/65980 Reviewed-By: James M Snell Reviewed-By: LiviaMedeiros --- test/parallel/test-dns-channel-timeout.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/parallel/test-dns-channel-timeout.js b/test/parallel/test-dns-channel-timeout.js index 1e4dac548973..0c9c7c31caee 100644 --- a/test/parallel/test-dns-channel-timeout.js +++ b/test/parallel/test-dns-channel-timeout.js @@ -22,10 +22,11 @@ for (const ctor of [dns.Resolver, dns.promises.Resolver]) { for (const timeout of [-1, 0, 1]) new ctor({ timeout }); // OK } +// One attempt is enough to exercise the timeout without retry backoff. for (const timeout of [0, 1, 2]) { const server = dgram.createSocket('udp4'); server.bind(0, '127.0.0.1', common.mustCall(() => { - const resolver = new dns.Resolver({ timeout }); + const resolver = new dns.Resolver({ timeout, tries: 1 }); resolver.setServers([`127.0.0.1:${server.address().port}`]); resolver.resolve4('nodejs.org', common.mustCall((err) => { assert.throws(() => { throw err; }, { @@ -40,7 +41,7 @@ for (const timeout of [0, 1, 2]) { for (const timeout of [0, 1, 2]) { const server = dgram.createSocket('udp4'); server.bind(0, '127.0.0.1', common.mustCall(() => { - const resolver = new dns.promises.Resolver({ timeout }); + const resolver = new dns.promises.Resolver({ timeout, tries: 1 }); resolver.setServers([`127.0.0.1:${server.address().port}`]); resolver.resolve4('nodejs.org').catch(common.mustCall((err) => { assert.throws(() => { throw err; }, { From 71a54aaf5fbc7a6acb93dea206d32432b3b3a534 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Fri, 11 Sep 2026 13:32:55 +0200 Subject: [PATCH 106/217] test: collect timeout signals explicitly Force collection on a later turn while the timeout sources are only retained by AbortSignal.any(). Shorten the first timeout and clear the watchdog after the assertion. This preserves the source-retention regression check without waiting ten seconds on successful runs. Signed-off-by: Filip Skokan Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/65980 Reviewed-By: James M Snell Reviewed-By: LiviaMedeiros --- .../test-abort-controller-any-timeout.js | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/test/parallel/test-abort-controller-any-timeout.js b/test/parallel/test-abort-controller-any-timeout.js index 2d94afaa63d9..675be3af703c 100644 --- a/test/parallel/test-abort-controller-any-timeout.js +++ b/test/parallel/test-abort-controller-any-timeout.js @@ -1,28 +1,42 @@ +// Flags: --expose-gc 'use strict'; -require('../common'); +const common = require('../common'); const assert = require('assert'); const { once } = require('node:events'); const { describe, it } = require('node:test'); describe('AbortSignal.any() with timeout signals', () => { it('should abort when the first timeout signal fires', async () => { - const signal = AbortSignal.any([AbortSignal.timeout(9000), AbortSignal.timeout(110000)]); + const signal = AbortSignal.any([ + AbortSignal.timeout(common.platformTimeout(1000)), + AbortSignal.timeout(110000), + ]); + let timeout; const abortPromise = Promise.race([ once(signal, 'abort').then(() => { throw signal.reason; }), - new Promise((resolve) => setTimeout(resolve, 10000)), + new Promise((resolve) => { + timeout = setTimeout(resolve, common.platformTimeout(10000)); + }), ]); - // The promise should be aborted by the 9000ms timeout - await assert.rejects( - () => abortPromise, - { - name: 'TimeoutError', - message: 'The operation was aborted due to timeout' - } - ); + // Collect after this turn so the WeakRefs no longer keep the timeout + // signals alive by themselves. + setImmediate(common.mustCall(() => globalThis.gc())); + + try { + await assert.rejects( + () => abortPromise, + { + name: 'TimeoutError', + message: 'The operation was aborted due to timeout' + } + ); + } finally { + clearTimeout(timeout); + } }); }); From 21702072533a763e30af57e3488c6cff160c9350 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Fri, 11 Sep 2026 13:32:55 +0200 Subject: [PATCH 107/217] test: unref cancelled broadcast source timer The source delay should not keep the process alive after cancellation. Keep the blocked-source cancellation assertions and unref its timer. Signed-off-by: Filip Skokan Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/65980 Reviewed-By: James M Snell Reviewed-By: LiviaMedeiros --- test/parallel/test-stream-iter-broadcast-from.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/parallel/test-stream-iter-broadcast-from.js b/test/parallel/test-stream-iter-broadcast-from.js index 39d92c2aef49..928d4d472f06 100644 --- a/test/parallel/test-stream-iter-broadcast-from.js +++ b/test/parallel/test-stream-iter-broadcast-from.js @@ -122,8 +122,8 @@ async function testBroadcastFromCancelWhileBlocked() { async function* slowSource() { const enc = new TextEncoder(); yield [enc.encode('chunk1')]; - // Simulate a long delay - the cancel should unblock this - await new Promise((resolve) => setTimeout(resolve, 10000)); + // Simulate a long delay without keeping the cancelled source alive. + await new Promise((resolve) => setTimeout(resolve, 10000).unref()); yield [enc.encode('chunk2')]; sourceFinished = true; } From f2aa27d82d21cbc3fe06a4a13a872ac3f6022027 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Fri, 11 Sep 2026 13:40:19 +0200 Subject: [PATCH 108/217] test: overlap SLH-DSA signature checks Start each asynchronous signature before the synchronous checks for the same algorithm. Keep every sign, verify, and invalid-digest assertion, with only one asynchronous signature outstanding. This reduces elapsed time when CPU capacity is available without reducing coverage. Signed-off-by: Filip Skokan Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/65980 Reviewed-By: James M Snell Reviewed-By: LiviaMedeiros --- test/pummel/test-crypto-pqc-sign-verify-slh-dsa.mjs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/pummel/test-crypto-pqc-sign-verify-slh-dsa.mjs b/test/pummel/test-crypto-pqc-sign-verify-slh-dsa.mjs index 772d9bab6f61..52b018e7028d 100644 --- a/test/pummel/test-crypto-pqc-sign-verify-slh-dsa.mjs +++ b/test/pummel/test-crypto-pqc-sign-verify-slh-dsa.mjs @@ -13,6 +13,9 @@ import { promisify } from 'node:util'; import { randomBytes, sign, verify } from 'node:crypto'; import fixtures from '../common/fixtures.js'; +const pSign = promisify(sign); +const pVerify = promisify(verify); + function getKeyFileName(type, suffix) { return `${type.replaceAll('-', '_')}_${suffix}.pem`; } @@ -37,6 +40,8 @@ for (const [asymmetricKeyType, sigLen] of [ }; const data = randomBytes(32); + // Start the async signature before the sync work to overlap the two. + const signaturePromise = pSign(undefined, data, keys.private); // sync { @@ -48,9 +53,7 @@ for (const [asymmetricKeyType, sigLen] of [ // async { - const pSign = promisify(sign); - const pVerify = promisify(verify); - const signature = await pSign(undefined, data, keys.private); + const signature = await signaturePromise; assert.strictEqual(signature.byteLength, sigLen); assert.strictEqual(await pVerify(undefined, randomBytes(32), keys.public, signature), false); assert.strictEqual(await pVerify(undefined, data, keys.public, signature), true); From a05023f3313156498cf1e8026f3522e79caf608b Mon Sep 17 00:00:00 2001 From: Shelley Vohr Date: Sun, 13 Sep 2026 14:31:04 +0200 Subject: [PATCH 109/217] src: fix crash on empty, foreign or truncated --snapshot-blob files `node --snapshot-blob main.js` aborted with an assertion when the file was empty (`ReadFileSync()` insists on reading one item), was not a Node.js snapshot (`CHECK_EQ(magic, kMagic)`) or had a zero-length startup blob, and read past the end of the buffer when a snapshot was truncated, because `BlobDeserializer` trusted every length field in the blob. `EmbedderSnapshotData::FromFile()` is documented to return an empty pointer for an invalid snapshot and crashed the same way. Bounds-check each read in `BlobDeserializer` and record the failure, have `SnapshotData::FromBlob()` print why and return false, and let `ReadFileSync()` return an empty vector for an empty file. Also add the missing space in the "built with Node.js version" messages. Refs: https://github.com/nodejs/node/pull/38905 Refs: https://github.com/nodejs/node/pull/47933 Signed-off-by: Shelley Vohr PR-URL: https://github.com/nodejs/node/pull/65955 Reviewed-By: Yagiz Nizipli Reviewed-By: James M Snell --- src/blob_serializer_deserializer-inl.h | 14 ++++++ src/blob_serializer_deserializer.h | 3 ++ src/node_file_utils.cc | 1 + src/node_snapshotable.cc | 30 +++++++++---- test/parallel/test-snapshot-invalid-blob.js | 49 +++++++++++++++++++++ 5 files changed, 88 insertions(+), 9 deletions(-) create mode 100644 test/parallel/test-snapshot-invalid-blob.js diff --git a/src/blob_serializer_deserializer-inl.h b/src/blob_serializer_deserializer-inl.h index 2a2aed1171f7..876c79578302 100644 --- a/src/blob_serializer_deserializer-inl.h +++ b/src/blob_serializer_deserializer-inl.h @@ -111,6 +111,11 @@ std::vector BlobDeserializer::ReadVector() { if (count == 0) { return std::vector(); } + // Every element takes at least one byte, so this bounds the allocation. + if (count > sink.size() - read_total) { + ok = false; + return std::vector(); + } if (is_debug) { Debug("Reading %d vector elements...\n", count); } @@ -143,6 +148,10 @@ std::string_view BlobDeserializer::ReadStringView(StringLogMode mode) { Debug("ReadStringView() read an empty view\n"); return std::string_view(); } + if (length > sink.size() - read_total) { + ok = false; + return std::string_view(); + } std::string_view result(sink.data() + read_total, length); Debug("%p, read %zu bytes", result.data(), result.size()); @@ -167,6 +176,11 @@ void BlobDeserializer::ReadArithmetic(T* out, size_t count) { } size_t size = sizeof(T) * count; + if (!ok || count > (sink.size() - read_total) / sizeof(T)) { + ok = false; + memset(out, 0, size); + return; + } memcpy(out, sink.data() + read_total, size); if (is_debug) { diff --git a/src/blob_serializer_deserializer.h b/src/blob_serializer_deserializer.h index cd63d0477675..e4d1efe99f53 100644 --- a/src/blob_serializer_deserializer.h +++ b/src/blob_serializer_deserializer.h @@ -45,6 +45,9 @@ class BlobDeserializer : public BlobSerializerDeserializer { size_t read_total = 0; std::string_view sink; + // Cleared when a read would go past the end of `sink`; that read and all + // later ones yield zeroes and empty views, so callers can check at the end. + bool ok = true; Impl* impl() { return static_cast(this); } const Impl* impl() const { return static_cast(this); } diff --git a/src/node_file_utils.cc b/src/node_file_utils.cc index 5f0ef1492135..ced08b1d0ff2 100644 --- a/src/node_file_utils.cc +++ b/src/node_file_utils.cc @@ -240,6 +240,7 @@ std::vector ReadFileSync(FILE* fp) { CHECK_EQ(err, 0); std::vector contents(size); + if (size == 0) return contents; size_t num_read = fread(contents.data(), size, 1, fp); CHECK_EQ(num_read, 1); return contents; diff --git a/src/node_snapshotable.cc b/src/node_snapshotable.cc index 945a5ee3f7a7..b23c73b1c44f 100644 --- a/src/node_snapshotable.cc +++ b/src/node_snapshotable.cc @@ -186,12 +186,16 @@ v8::StartupData SnapshotDeserializer::Read() { int raw_size = ReadArithmetic(); Debug("size=%d\n", raw_size); - CHECK_GT(raw_size, 0); // There should be no startup data of size 0. + if (raw_size <= 0 || + static_cast(raw_size) > sink.size() - read_total) { + ok = false; + return v8::StartupData{nullptr, 0}; + } // The data pointer of v8::StartupData would be deleted so it must be new'ed. - std::unique_ptr buf = std::unique_ptr(new char[raw_size]); - ReadArithmetic(buf.get(), raw_size); + char* buf = new char[raw_size]; + ReadArithmetic(buf, raw_size); - return v8::StartupData{buf.release(), raw_size}; + return v8::StartupData{buf, raw_size}; } template <> @@ -645,10 +649,14 @@ bool SnapshotData::FromBlob(SnapshotData* out, std::string_view in) { // Metadata uint32_t magic = r.ReadArithmetic(); r.Debug("Read magic %" PRIx32 "\n", magic); - CHECK_EQ(magic, kMagic); + if (!r.ok || magic != kMagic) { + fprintf(stderr, "The startup snapshot is not a Node.js snapshot blob.\n"); + return false; + } out->metadata = r.Read(); r.Debug("Read metadata\n"); - if (!out->Check()) { + if (!r.ok || !out->Check()) { + if (!r.ok) fprintf(stderr, "The startup snapshot is truncated.\n"); return false; } @@ -660,13 +668,17 @@ bool SnapshotData::FromBlob(SnapshotData* out, std::string_view in) { out->code_cache = r.ReadVector(); r.Debug("SnapshotData::FromBlob() read %d bytes\n", r.read_total); + if (!r.ok) { + fprintf(stderr, "The startup snapshot is truncated.\n"); + return false; + } return true; } bool SnapshotData::Check() const { if (metadata.node_version != per_process::metadata.versions.node) { fprintf(stderr, - "Failed to load the startup snapshot because it was built with" + "Failed to load the startup snapshot because it was built with " "Node.js version %s and the current Node.js version is %s.\n", metadata.node_version.c_str(), NODE_VERSION); @@ -675,7 +687,7 @@ bool SnapshotData::Check() const { if (metadata.node_arch != per_process::metadata.arch) { fprintf(stderr, - "Failed to load the startup snapshot because it was built with" + "Failed to load the startup snapshot because it was built with " "architecture %s and the architecture is %s.\n", metadata.node_arch.c_str(), NODE_ARCH); @@ -684,7 +696,7 @@ bool SnapshotData::Check() const { if (metadata.node_platform != per_process::metadata.platform) { fprintf(stderr, - "Failed to load the startup snapshot because it was built with" + "Failed to load the startup snapshot because it was built with " "platform %s and the current platform is %s.\n", metadata.node_platform.c_str(), NODE_PLATFORM); diff --git a/test/parallel/test-snapshot-invalid-blob.js b/test/parallel/test-snapshot-invalid-blob.js new file mode 100644 index 000000000000..9157a2d3a482 --- /dev/null +++ b/test/parallel/test-snapshot-invalid-blob.js @@ -0,0 +1,49 @@ +'use strict'; + +// This tests that Node.js reports an error, rather than crashing, when the +// file passed to --snapshot-blob is empty, is not a snapshot, or is truncated. + +require('../common'); +const { + spawnSyncAndExit, + spawnSyncAndExitWithoutError, +} = require('../common/child_process'); +const tmpdir = require('../common/tmpdir'); +const fixtures = require('../common/fixtures'); +const fs = require('fs'); + +tmpdir.refresh(); +const entry = fixtures.path('empty.js'); + +function expectFailure(blobPath, stderr) { + spawnSyncAndExit(process.execPath, ['--snapshot-blob', blobPath, entry], { + cwd: tmpdir.path, + }, { + status: 14, + signal: null, + stderr, + }); +} + +{ + const blobPath = tmpdir.resolve('empty.blob'); + fs.writeFileSync(blobPath, ''); + expectFailure(blobPath, /not a Node\.js snapshot blob/); +} + +{ + const blobPath = tmpdir.resolve('garbage.blob'); + fs.writeFileSync(blobPath, Buffer.alloc(4096, 0x61)); + expectFailure(blobPath, /not a Node\.js snapshot blob/); +} + +{ + const blobPath = tmpdir.resolve('snapshot.blob'); + spawnSyncAndExitWithoutError(process.execPath, [ + '--snapshot-blob', blobPath, '--build-snapshot', entry, + ], { cwd: tmpdir.path }); + const blob = fs.readFileSync(blobPath); + const truncatedPath = tmpdir.resolve('truncated.blob'); + fs.writeFileSync(truncatedPath, blob.subarray(0, blob.length >> 1)); + expectFailure(truncatedPath, /truncated/); +} From 3666d6a238acfa74e23f1564189ac7a72a457d8e Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Sun, 13 Sep 2026 16:21:48 +0200 Subject: [PATCH 110/217] test: schedule WPT variants individually Discover WPT tasks through their existing JavaScript drivers so the Python runner can schedule, report, and rerun generated paths. Assisted-by: Codex Signed-off-by: Filip Skokan PR-URL: https://github.com/nodejs/node/pull/65984 Fixes: https://github.com/nodejs/node/issues/51854 Reviewed-By: Aviv Keller Reviewed-By: James M Snell Reviewed-By: Matteo Collina --- test/common/wpt.js | 87 ++++++++++-- test/parallel/test-common-wpt-runner.js | 147 ++++++++++++++++++++ test/tools/test_wpt_runner.py | 170 ++++++++++++++++++++++++ test/wpt/README.md | 9 ++ test/wpt/test-compression.js | 3 +- test/wpt/test-streams.js | 3 +- test/wpt/test-wasm-jsapi.mjs | 3 +- test/wpt/test-webcrypto.js | 3 +- test/wpt/test-webstorage.js | 2 +- test/wpt/testcfg.py | 112 +++++++++++++++- tools/test.py | 28 ++-- 11 files changed, 531 insertions(+), 36 deletions(-) create mode 100644 test/parallel/test-common-wpt-runner.js create mode 100644 test/tools/test_wpt_runner.py diff --git a/test/common/wpt.js b/test/common/wpt.js index 4ad788ae029d..e2668d75da7d 100644 --- a/test/common/wpt.js +++ b/test/common/wpt.js @@ -96,8 +96,8 @@ class ReportResult { // Checkout https://github.com/web-platform-tests/wpt.fyi/tree/main/api#results-creation // for more details. class WPTReport { - constructor(testPath) { - this.filename = `report-${testPath.replaceAll('/', '-')}.json`; + constructor(testPath, suffix = '') { + this.filename = `report-${testPath.replaceAll('/', '-')}${suffix}.json`; this.filepath = path.join(__dirname, `../../out/wpt/${this.filename}`); /** @type {Map} */ this.results = new Map(); @@ -461,12 +461,14 @@ class WPTTestSpec { /** * Whether a command line argument selects this spec. Accepts the source file * name, which selects every global and variant generated from it, or a test - * path as printed alongside the results, which selects only this one. + * path as printed alongside the results. Omitting its query selects all + * variants of that global. * @param {string} arg * @returns {boolean} */ isSelectedBy(arg) { - if (arg === this.getTestPath()) { + const testPath = this.getTestPath(); + if (arg === testPath || arg === testPath.split('?')[0]) { return true; } const [filename, variant = ''] = arg.split('?'); @@ -608,7 +610,7 @@ class StatusLoader { return result; } - load() { + load(source) { const dir = path.join(__dirname, '..', 'wpt'); let result; @@ -625,7 +627,7 @@ class StatusLoader { this.rules.addRules(result); const subDir = fixtures.path('wpt', this.path); - const list = this.grep(subDir); + const list = source === undefined ? this.grep(subDir) : [path.join(subDir, source)]; for (const file of list) { const relativePath = path.relative(subDir, file); const match = this.rules.match(relativePath); @@ -813,14 +815,29 @@ const backends = { }; class WPTRunner { - constructor(path, { - concurrency = os.availableParallelism() - 1 || 1, - backend = 'thread', - } = {}) { + constructor(path, options = {}) { + let { + concurrency = os.availableParallelism() - 1 || 1, + backend = 'thread', + } = options; if (!Number.isInteger(concurrency) || concurrency < 1) { throw new TypeError('WPT concurrency must be a positive integer'); } + if (process.env.NODE_TEST_WPT !== undefined) { + this.managed = JSON.parse(process.env.NODE_TEST_WPT); + if (!this.managed || !['list', 'run'].includes(this.managed.mode) || + (this.managed.mode === 'run' && + (['source', 'key'].some((key) => + typeof this.managed[key] !== 'string' || !this.managed[key]) || + (this.managed.variant !== undefined && typeof this.managed.variant !== 'string')))) { + throw new Error('Invalid WPT runner configuration'); + } + } + this.isListing = this.managed?.mode === 'list'; + this.serial = options.concurrency === 1; + if (this.managed?.mode === 'run') concurrency = 1; + // RISC-V has very limited virtual address space in the currently common // sv39 mode, in which we can only create a very limited number of wasm // memories(27 from a fresh node repl). Limit the concurrency to avoid @@ -860,7 +877,7 @@ class WPTRunner { this.initScript = null; this.status = new StatusLoader(path); - this.status.load(); + this.status.load(this.managed?.mode === 'run' ? this.managed.source : undefined); this.statusFile = this.status.statusFile; this.specs = new Set(this.status.specs); @@ -872,8 +889,9 @@ class WPTRunner { this.subtestCounts = { passed: 0, failed: 0, expectedFailures: 0, skipped: 0, unexpectedPasses: 0 }; - if (process.env.WPT_REPORT != null) { - this.report = new WPTReport(path); + if (process.env.WPT_REPORT != null && !this.isListing) { + const suffix = this.managed ? `-${process.env.TEST_SERIAL_ID || process.pid}` : ''; + this.report = new WPTReport(path, suffix); } } @@ -958,7 +976,40 @@ class WPTRunner { // TODO(joyeecheung): work with the upstream to port more tests in .html // to .js. async runJsTests() { + if (this.isListing) { + const groups = new Map(); + for (const spec of this.specs) { + const key = spec.getStatusKey(); + if (!groups.has(key)) groups.set(key, []); + groups.get(key).push(spec); + } + const tests = [...groups.values()].flatMap((specs) => { + // Strict expected failures are checked across all query variants. + // Keep that group together; all other variants are independent tasks. + const grouped = specs.some((spec) => + spec.failedTests.some((name) => isUnexpectedPass(spec, name))); + return (grouped ? [specs[0]] : specs).map((spec) => { + const selector = grouped ? spec.getTestPath().split('?')[0] : spec.getTestPath(); + return { + source: spec.filename.split(path.sep).join('/'), + key: spec.getStatusKey(), + ...(grouped ? {} : { variant: spec.variant }), + id: selector.slice(this.path.length + 1), + selector, + }; + }); + }); + console.log(`NODE_TEST_WPT_MANIFEST:${JSON.stringify({ + version: 1, + serial: this.serial, + tests: tests.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)), + })}`); + return; + } const queue = this.buildQueue(); + if (this.managed && queue.length === 0) { + console.log('1..0 # SKIP No runnable WPT variants'); + } const run = limit(this.concurrency); const jobs = []; @@ -1313,11 +1364,19 @@ class WPTRunner { buildQueue() { const queue = []; this.skippedSpecCount = 0; - const arg = process.argv[2]; + const key = this.managed?.mode === 'run' ? this.managed.key : undefined; + const variant = this.managed?.variant; + const matches = (spec) => spec.getStatusKey() === key && + (variant === undefined || spec.variant === variant); + const arg = key === undefined ? process.argv[2] : undefined; + if (key !== undefined && ![...this.specs].some(matches)) { + throw new Error(`${key}${variant ?? ''} not found!`); + } if (this.inspectBrk && !arg) { throw new Error('WPT_INSPECT requires a WPT test path'); } for (const spec of this.specs) { + if (key !== undefined && !matches(spec)) continue; if (arg) { if (spec.isSelectedBy(arg)) { queue.push(spec); diff --git a/test/parallel/test-common-wpt-runner.js b/test/parallel/test-common-wpt-runner.js new file mode 100644 index 000000000000..ea41caa6cefc --- /dev/null +++ b/test/parallel/test-common-wpt-runner.js @@ -0,0 +1,147 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const { spawnSync } = require('child_process'); +const tmpdir = require('../common/tmpdir'); + +if (process.env.NODE_TEST_WPT_REPORT_DIR) { + const { WPTRunner } = require('../common/wpt'); + const runner = new WPTRunner('html/webappapis/atob'); + runner.report.filepath = path.join(process.env.NODE_TEST_WPT_REPORT_DIR, runner.report.filename); + runner.runJsTests(); +} else if (process.env.NODE_TEST_WPT_QUERY_PROBE) { + const { WPTRunner, WPTTestSpec } = require('../common/wpt'); + const runner = new WPTRunner('compression'); + if (runner.managed?.mode === 'run') assert.strictEqual(runner.concurrency, 1); + runner.specs = new Set(['?pass', '?fail'].map((query) => { + const spec = new WPTTestSpec('compression', 'compression-bad-chunks.any.js', [], query, 'window'); + spec.failedTests = ['expected across queries']; + if (process.env.NODE_TEST_WPT_QUERY_PROBE === 'flaky' || + (process.env.NODE_TEST_WPT_QUERY_PROBE === 'mixed' && query === '?pass')) { + spec.flakyTests = [...spec.failedTests]; + } + return spec; + })); + runner.setScriptModifier((script) => { + if (!script.filename.endsWith('compression-bad-chunks.any.js')) return; + script.code = `test(() => assert_true(${process.env.NODE_TEST_WPT_QUERY_PROBE === 'missing'} || + location.search === '?pass'), 'expected across queries');`; + }); + runner.runJsTests(); +} else { + main(); +} + +function main() { + tmpdir.refresh(); + const env = { ...process.env }; + for (const key of ['NODE_TEST_WPT', 'WPT_REPORT', 'WPT_INSPECT']) delete env[key]; + const driver = (name) => path.join(__dirname, '../wpt', `test-${name}.js`); + function invoke(file, config, overrides = {}, status = 0) { + const result = spawnSync(process.execPath, [file], { + env: { ...env, ...overrides, NODE_TEST_WPT: JSON.stringify(config) }, + encoding: 'utf8', timeout: common.platformTimeout(10_000), + maxBuffer: 10 * 1024 * 1024, + }); + assert.ifError(result.error); + assert.strictEqual(result.status, status, result.stdout + result.stderr); + return result.stdout + result.stderr; + } + + function discover(name, overrides, file = driver(name)) { + const stdout = invoke(file, { mode: 'list' }, overrides); + const lines = stdout.split('\n').filter((line) => line.startsWith('NODE_TEST_WPT_MANIFEST:')); + assert.strictEqual(lines.length, 1); + assert.doesNotMatch(stdout, /\[PASS\]/); + const manifest = JSON.parse(lines[0].slice('NODE_TEST_WPT_MANIFEST:'.length)); + assert.strictEqual(manifest.version, 1); + assert.strictEqual(new Set(manifest.tests.map((test) => test.id)).size, manifest.tests.length); + return manifest; + } + + const atob = discover('atob'); + assert.strictEqual(atob.serial, false); + assert.deepStrictEqual(atob.tests.map((test) => test.id), ['base64.any.html', 'base64.any.worker.html']); + const reportRoot = path.join(tmpdir.path, 'reports'); + const canReport = ['darwin', 'linux', 'win32'].includes(process.platform); + if (canReport) fs.mkdirSync(reportRoot, { recursive: true }); + for (const [index, group] of atob.tests.entries()) { + const serial = `group-probe-${process.pid}-${index}`; + const reportPath = path.join(reportRoot, `report-html-webappapis-atob-${serial}.json`); + try { + const stdout = invoke(canReport ? __filename : driver('atob'), + { mode: 'run', source: group.source, key: group.key, variant: group.variant }, canReport ? { + WPT_REPORT: '1', TEST_SERIAL_ID: serial, NODE_TEST_WPT_REPORT_DIR: reportRoot, + } : {}); + const results = stdout.split('\n').filter((line) => line.startsWith('[PASS]')); + assert.ok(results.length > 0); + assert.ok(results.every((line) => line.includes(`${group.id}:`))); + if (canReport) { + const report = JSON.parse(fs.readFileSync(reportPath, 'utf8')); + assert.deepStrictEqual(report.results.map((result) => result.test), + [`/html/webappapis/atob/${group.id}`]); + } + } finally { + fs.rmSync(reportPath, { force: true }); + } + } + + const encoding = discover('encoding'); + const queryGroups = encoding.tests.filter((test) => test.source === 'api-invalid-label.any.js'); + assert.strictEqual(queryGroups.length, 8); + assert.deepStrictEqual([...new Set(queryGroups.map((test) => test.key))], + ['api-invalid-label.any.html', 'api-invalid-label.any.worker.html']); + const timers = discover('timers'); + assert.strictEqual(timers.serial, true); + const skipped = timers.tests.find((test) => test.source === 'negative-settimeout.any.js'); + assert.ok(skipped); + const skippedOutput = invoke(driver('timers'), { mode: 'run', source: skipped.source, key: skipped.key }); + assert.match(skippedOutput, /\[SKIPPED\].*unreliable in Node\.js/); + assert.match(skippedOutput, /1\.\.0 # SKIP/); + assert.doesNotMatch(skippedOutput, /\[PASS\]/); + + if (common.hasSQLite) { + const root = path.join(tmpdir.path, 'discovery'); + const directory = path.join(root, '.tmp.probe'); + fs.mkdirSync(directory, { recursive: true }); + const sentinel = path.join(directory, 'sentinel'); + fs.writeFileSync(sentinel, 'preserved'); + assert.strictEqual(discover('webstorage', { NODE_TEST_DIR: root, TEST_SERIAL_ID: 'probe' }).serial, true); + assert.strictEqual(fs.readFileSync(sentinel, 'utf8'), 'preserved'); + } + + const config = { mode: 'run', source: 'compression-bad-chunks.any.js', key: 'compression-bad-chunks.any.html' }; + for (const probe of ['combined', 'mixed']) { + const strict = discover('compression', { NODE_TEST_WPT_QUERY_PROBE: probe }, __filename); + assert.deepStrictEqual(strict.tests, [{ + source: config.source, key: config.key, id: config.key, selector: `compression/${config.key}`, + }]); + } + const flaky = discover('compression', { NODE_TEST_WPT_QUERY_PROBE: 'flaky' }, __filename); + assert.deepStrictEqual(flaky.tests.map((test) => test.variant).sort(), ['?fail', '?pass']); + assert.deepStrictEqual(flaky.tests.map((test) => test.id).sort(), + [`${config.key}?fail`, `${config.key}?pass`]); + const single = invoke(__filename, { ...config, variant: '?pass' }, { NODE_TEST_WPT_QUERY_PROBE: 'flaky' }); + assert.match(single, /\.any\.html\?pass:/); + assert.doesNotMatch(single, /\.any\.html\?fail:/); + const combined = invoke(__filename, config, { NODE_TEST_WPT_QUERY_PROBE: 'combined' }); + assert.match(combined, /\.any\.html\?pass:/); + assert.match(combined, /\.any\.html\?fail:/); + const missing = invoke(__filename, config, { NODE_TEST_WPT_QUERY_PROBE: 'missing' }, 1); + assert.match(missing, /Found 2 unexpected passes/); + + for (const query of ['', '?fail']) { + const direct = spawnSync(process.execPath, [__filename, `compression/${config.key}${query}`], { + env: { ...env, NODE_TEST_WPT_QUERY_PROBE: 'combined' }, + encoding: 'utf8', timeout: common.platformTimeout(10_000), + }); + assert.ifError(direct.error); + assert.strictEqual(direct.status, 0, direct.stdout + direct.stderr); + assert.match(direct.stdout, /\.any\.html\?fail:/); + if (query) assert.doesNotMatch(direct.stdout, /\.any\.html\?pass:/); + else assert.match(direct.stdout, /\.any\.html\?pass:/); + } +} diff --git a/test/tools/test_wpt_runner.py b/test/tools/test_wpt_runner.py new file mode 100644 index 000000000000..5897e396cb4f --- /dev/null +++ b/test/tools/test_wpt_runner.py @@ -0,0 +1,170 @@ +import contextlib +import json +import os +import shlex +import sys +import tempfile +import unittest +import warnings +from unittest import mock + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) +sys.path.insert(0, os.path.join(ROOT, 'tools')) +import test as runner + +wpt = runner.get_module('testcfg', os.path.join(ROOT, 'test', 'wpt')) + + +class WPTConfigurationTest(unittest.TestCase): + def setUp(self): + self.stack = contextlib.ExitStack() + self.addCleanup(self.stack.close) + self.stack.enter_context(warnings.catch_warnings()) + # SimpleTestCase's existing source reader does not explicitly close files. + warnings.filterwarnings('ignore', category=ResourceWarning, module='testpy', + message=r'unclosed file .*test-example\.js') + self.root = self.stack.enter_context(tempfile.TemporaryDirectory()) + self.wrapper = os.path.join(self.root, 'test-example.js') + with open(self.wrapper, 'w', encoding='utf8') as source: + source.write('// Flags: --expose-gc\n// Env: GROUP_TEST=kept\n') + self.context = runner.Context( + ROOT, False, sys.executable, [], False, 5, lambda args: args, + False, False, 1, False) + self.config = wpt.GetConfiguration(self.context, self.root) + self.manifest = {'version': 1, 'serial': False, 'tests': [ + {'source': 'nested/a.any.js', 'key': 'nested/a.any.html', 'id': 'nested/a.any.html', + 'selector': 'example/nested/a.any.html'}, + {'source': 'nested/a.any.js', 'key': 'nested/a.any.worker.html', + 'id': 'nested/a.any.worker.html', 'selector': 'example/nested/a.any.worker.html'}, + {'source': 'z.any.js', 'key': 'z.any.html', 'id': 'z.any.html', 'selector': 'example/z.any.html'}, + ]} + self.discovery = self.stack.enter_context(mock.patch.object( + runner, 'Execute', side_effect=self.discover)) + + def discover(self, command, context, timeout=None, env=None, **kwargs): + self.assertEqual(json.loads(env['NODE_TEST_WPT']), {'mode': 'list'}) + self.assertEqual(command[-1], self.wrapper) + return runner.CommandOutput( + 0, False, wpt.MANIFEST_PREFIX + json.dumps(self.manifest) + '\n', '') + + def cases(self, selector='wpt/test-example'): + return self.config.ListTests(['wpt'], runner.SplitPath(selector), 'none', 'release') + + def test_discovery_creates_one_case_per_status_group(self): + cases = self.cases() + self.assertEqual([case.group for case in cases], self.manifest['tests']) + self.assertTrue(all(case.parallel for case in cases)) + self.assertEqual([case.GetName() for case in cases], + ['wpt/test-example/' + group['id'] for group in self.manifest['tests']]) + self.assertEqual(len({tuple(case.path) for case in cases}), 3) + self.assertEqual(self.discovery.call_count, 1) + + def test_group_request_preserves_wrapper_startup_flags(self): + self.config.additional_flags = ['--trace-warnings'] + case = self.cases('wpt/test-example/nested/a.any.worker.html')[0] + configuration = case.GetRunConfiguration() + self.assertEqual(configuration['command'], + [sys.executable, '--expose-gc', '--trace-warnings', self.wrapper]) + self.assertEqual(configuration['envs']['GROUP_TEST'], 'kept') + self.assertEqual(json.loads(configuration['envs']['NODE_TEST_WPT']), { + 'mode': 'run', 'source': 'nested/a.any.js', 'key': 'nested/a.any.worker.html', + }) + self.assertEqual(case.GetReportingName(configuration['command']), + 'wpt/test-example/nested/a.any.worker.html') + + def test_group_selection_and_serial_suites(self): + self.manifest['serial'] = True + cases = self.cases('wpt/test-example/nested/*') + self.assertEqual(len(cases), 2) + self.assertTrue(all(not case.parallel for case in cases)) + selected = self.cases('wpt/test-example/nested/a.any.worker.html') + self.assertEqual([case.group['id'] for case in selected], ['nested/a.any.worker.html']) + + def test_variant_queries_are_literal_and_base_selects_all_queries(self): + group = self.manifest['tests'][0] + variants = ['', '?q=[a]+(b)|c', '?q=aaab', '?q=*', '?q=source.js'] + self.manifest['tests'] = [ + {**group, 'id': group['id'] + variant, 'selector': group['selector'] + variant, + 'variant': variant} for variant in variants] + base = 'wpt/test-example/' + group['id'] + self.assertEqual([case.group['variant'] for case in self.cases(base)], variants) + for variant in variants[1:]: + with self.subTest(variant=variant): + selector = base + variant + self.assertEqual(runner.NormalizePath(selector), selector) + selected = self.cases(selector) + self.assertEqual(len(selected), 1) + case = selected[0] + self.assertEqual(case.GetName(), selector) + request = json.loads(case.GetRunConfiguration()['envs']['NODE_TEST_WPT']) + self.assertEqual(request, {'mode': 'run', 'source': group['source'], + 'key': group['key'], 'variant': variant}) + empty = self.cases(base)[0] + self.assertEqual(json.loads(empty.GetRunConfiguration()['envs']['NODE_TEST_WPT'])['variant'], '') + + def test_discovery_is_cached_and_uses_existing_directory(self): + absent = os.path.join(self.root, 'not-created') + with mock.patch.dict(os.environ, {'NODE_TEST_DIR': absent}): + self.cases() + self.cases('wpt/test-example/nested/*') + self.assertEqual(self.discovery.call_count, 1) + self.assertFalse(os.path.exists(absent)) + command, _, _, env = self.discovery.call_args.args + self.assertIn('--expose-gc', command) + self.assertEqual(env['NODE_TEST_DIR'], self.root) + + def test_whole_wrapper_feature_skip_keeps_original_test(self): + self.discovery.side_effect = None + self.discovery.return_value = runner.CommandOutput(0, False, '1..0 # Skipped: no feature\n', '') + cases = self.cases() + self.assertEqual(len(cases), 1) + self.assertEqual(cases[0].path, ['wpt', 'test-example']) + self.assertEqual(cases[0].GetRunConfiguration()['command'][-1], self.wrapper) + self.assertNotIn('NODE_TEST_WPT', cases[0].GetRunConfiguration()['envs']) + + def test_malformed_discovery_is_an_error(self): + self.manifest['tests'] = 'not a list' + with self.assertRaisesRegex(RuntimeError, 'WPT discovery failed'): + self.cases() + + def test_failure_commands_preserve_actual_command_and_selection(self): + self.config.additional_flags = ["--title=space ' $|?", '--trace-warnings'] + query = "?q=space ' | $(echo)" + for group in self.manifest['tests'][:2]: + group.update(id=group['id'] + query, selector=group['selector'] + query, variant=query) + self.manifest['tests'].append({'source': 'empty.any.js', 'key': 'empty.any.html', + 'id': 'empty.any.html', 'selector': 'example/empty.any.html', 'variant': ''}) + cases = self.cases() + self.assertEqual(len(cases), 4) + self.context.processor = lambda args: ['valgrind', '--tool=memcheck', *args, 'suffix with |'] + with mock.patch.object(sys, 'platform', 'linux'): + for case, group in zip(cases, self.manifest['tests']): + command = case.GetRunConfiguration()['command'] + expected = [sys.executable, '--expose-gc', "--title=space ' $|?", '--trace-warnings', + self.wrapper] + self.assertEqual(command, expected) + command = self.context.processor(command) + rerun = ['valgrind', '--tool=memcheck', *expected, group['selector'], 'suffix with |'] + self.assertEqual(shlex.split(case.GetFailureCommand(command)), rerun) + failure = runner.TestOutput(case, command, + runner.CommandOutput(1, False, '', 'probe failure'), False) + printer = runner.ProgressIndicator([], runner.RUN, 0) + self.assertIn('Command: ' + shlex.join(rerun), printer.GetFailureOutput(failure)) + + def test_failure_command_quotes_powershell_metacharacters(self): + case = self.cases()[0] + case.file = 'test driver.js' + case.group['selector'] = "example/a.any.html?q=O'Brien|x" + command = ['node', case.file] + with mock.patch.object(sys, 'platform', 'win32'): + self.assertEqual(case.GetFailureCommand(command), + "& 'node' 'test driver.js' 'example/a.any.html?q=O''Brien|x'") + + def test_ordinary_failure_command_is_unchanged(self): + case = runner.TestCase(self.context, ['parallel', 'test-example'], 'none', 'release') + command = [sys.executable, '--expose-gc', 'test/parallel/path with spaces.js'] + self.assertEqual(case.GetFailureCommand(command), runner.EscapeCommand(command)) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/wpt/README.md b/test/wpt/README.md index ac96e3b247ba..1b6911740a06 100644 --- a/test/wpt/README.md +++ b/test/wpt/README.md @@ -20,6 +20,15 @@ Run a WPT module through the Python test runner: tools/test.py wpt/test-url ``` +Select a generated path from the Python runner's output: + +```bash +tools/test.py 'wpt/test-webcrypto/derive_bits_keys/hkdf.https.any.worker.html?1-1000' +``` + +Omit the query string to select all its variants. Variants sharing strict +expected-failure rules run together. + Pass a source file to its module runner to run all globals and variants generated from that file: diff --git a/test/wpt/test-compression.js b/test/wpt/test-compression.js index 404cb23687ca..23800734922a 100644 --- a/test/wpt/test-compression.js +++ b/test/wpt/test-compression.js @@ -2,8 +2,7 @@ const { WPTRunner } = require('../common/wpt'); -// Runs each spec in its own process; this suite has crashed the runner in CI. -const runner = new WPTRunner('compression', { backend: 'process' }); +const runner = new WPTRunner('compression'); runner.pretendGlobalThisAs('Window'); diff --git a/test/wpt/test-streams.js b/test/wpt/test-streams.js index e9d23348db07..71c25fbd56b2 100644 --- a/test/wpt/test-streams.js +++ b/test/wpt/test-streams.js @@ -2,8 +2,7 @@ const { WPTRunner } = require('../common/wpt'); -// Runs each spec in its own process; this suite has crashed the runner in CI. -const runner = new WPTRunner('streams', { backend: 'process' }); +const runner = new WPTRunner('streams'); // Set a script that will be executed in the worker before running the tests. runner.pretendGlobalThisAs('Window'); diff --git a/test/wpt/test-wasm-jsapi.mjs b/test/wpt/test-wasm-jsapi.mjs index 67050e67cf7b..4e53da5bdd88 100644 --- a/test/wpt/test-wasm-jsapi.mjs +++ b/test/wpt/test-wasm-jsapi.mjs @@ -15,8 +15,7 @@ try { } if (supportsSimd) { - // Runs each spec in its own process; this suite has crashed the runner in CI. - const runner = new WPTRunner('wasm/jsapi', { backend: 'process' }); + const runner = new WPTRunner('wasm/jsapi'); runner.setFlags(['--experimental-wasm-modules']); runner.runJsTests(); diff --git a/test/wpt/test-webcrypto.js b/test/wpt/test-webcrypto.js index a435d429af9f..409b94451ba1 100644 --- a/test/wpt/test-webcrypto.js +++ b/test/wpt/test-webcrypto.js @@ -6,8 +6,7 @@ if (!common.hasCrypto) const { WPTRunner } = require('../common/wpt'); -// Runs each spec in its own process; this suite has crashed the runner in CI. -const runner = new WPTRunner('WebCryptoAPI', { backend: 'process' }); +const runner = new WPTRunner('WebCryptoAPI'); // Experimental warnings drown out the actual test output. runner.setFlags(['--disable-warning=ExperimentalWarning']); diff --git a/test/wpt/test-webstorage.js b/test/wpt/test-webstorage.js index a677659f4b04..b0cc3d6bf13d 100644 --- a/test/wpt/test-webstorage.js +++ b/test/wpt/test-webstorage.js @@ -6,7 +6,7 @@ const { WPTRunner } = require('../common/wpt'); const { join } = require('node:path'); const runner = new WPTRunner('webstorage', { concurrency: 1 }); -tmpdir.refresh(); +if (!runner.isListing) tmpdir.refresh(); runner.setFlags([ '--localstorage-file', join(tmpdir.path, 'wpt-tests.localstorage'), diff --git a/test/wpt/testcfg.py b/test/wpt/testcfg.py index 3c356cf474d8..2f3097da96df 100644 --- a/test/wpt/testcfg.py +++ b/test/wpt/testcfg.py @@ -1,6 +1,114 @@ -import sys, os +import json +import os +import re +import shlex +import sys + sys.path.append(os.path.join(os.path.dirname(__file__), '..')) import testpy +import test + + +MANIFEST_PREFIX = 'NODE_TEST_WPT_MANIFEST:' + + +class WPTTestCase(testpy.SimpleTestCase): + def __init__(self, path, file, arch, mode, context, config, group, serial): + super(WPTTestCase, self).__init__( + path, file, arch, mode, context, config, config.additional_flags) + self.group = group + self.parallel = not serial + + def GetName(self): + return '/'.join(self.path) + + def GetReportingName(self, command): + return self.GetName() + + def GetFailureCommand(self, command): + command = list(command) + command.insert(command.index(self.file) + 1, self.group['selector']) + if sys.platform == 'win32': + # PowerShell quoting also protects query metacharacters such as | and &. + return '& ' + ' '.join("'" + arg.replace("'", "''") + "'" for arg in command) + return shlex.join(command) + + def GetRunConfiguration(self): + configuration = super(WPTTestCase, self).GetRunConfiguration() + request = { + 'mode': 'run', 'source': self.group['source'], 'key': self.group['key'], + } + if 'variant' in self.group: + request['variant'] = self.group['variant'] + configuration['envs']['NODE_TEST_WPT'] = json.dumps(request) + return configuration + + +class WPTTestConfiguration(testpy.SimpleTestConfiguration): + def __init__(self, context, root): + super(WPTTestConfiguration, self).__init__(context, root, 'wpt') + self.manifests = {} + + def _Discover(self, wrapper): + key = (wrapper.file, wrapper.arch, wrapper.mode) + if key in self.manifests: + return self.manifests[key] + configuration = wrapper.GetRunConfiguration() + configuration['envs']['NODE_TEST_WPT'] = json.dumps({'mode': 'list'}) + # common/tmpdir resolves this path before Main creates the execution dir. + configuration['envs']['NODE_TEST_DIR'] = os.path.abspath(self.root) + output = test.Execute( + self.context.processor(configuration['command']), self.context, + self.context.GetTimeout(wrapper.mode), configuration['envs']) + if output.exit_code != 0 or output.timed_out: + raise RuntimeError('WPT discovery failed for %s:\n%s%s' % ( + wrapper.file, output.stdout, output.stderr)) + manifests = [line[len(MANIFEST_PREFIX):] for line in output.stdout.splitlines() + if line.startswith(MANIFEST_PREFIX)] + if not manifests and test.skip_regex.search(output.stdout): + self.manifests[key] = None + return None + try: + if len(manifests) != 1: + raise ValueError('expected exactly one WPT manifest') + manifest = json.loads(manifests[0]) + if (manifest.get('version') != 1 or + not isinstance(manifest.get('serial'), bool) or + not isinstance(manifest.get('tests'), list) or + any(not isinstance(group.get(field), str) + for group in manifest['tests'] for field in ['source', 'key', 'id', 'selector']) or + any('variant' in group and not isinstance(group['variant'], str) + for group in manifest['tests'])): + raise ValueError('invalid WPT manifest') + except (AttributeError, TypeError, ValueError) as error: + raise RuntimeError('WPT discovery failed for %s: %s' % ( + wrapper.file, error)) from error + self.manifests[key] = manifest + return manifest + + def ListTests(self, current_path, path, arch, mode): + wrappers = super(WPTTestConfiguration, self).ListTests( + current_path, path[:2], arch, mode) + selector = '/'.join(part.pattern for part in path[2:]) + pattern = re.escape(selector).replace(r'\*', '.*') + r'(?:/.*)?' + result = [] + for wrapper in wrappers: + manifest = self._Discover(wrapper) + if manifest is None: + result.append(wrapper) + continue + for group in manifest['tests']: + case_path = wrapper.path + group['id'].split('/') + # Query selectors are literal; a query-free path selects all its variants. + if '?' in selector: + selected = group['id'] == selector + else: + selected = not selector or re.fullmatch(pattern, group['id'].split('?')[0]) + if selected: + result.append(WPTTestCase(case_path, wrapper.file, arch, mode, + self.context, self, group, manifest['serial'])) + return result + def GetConfiguration(context, root): - return testpy.SimpleTestConfiguration(context, root, 'wpt') + return WPTTestConfiguration(context, root) diff --git a/tools/test.py b/tools/test.py index a20c44e52516..90df2cfb2fac 100755 --- a/tools/test.py +++ b/tools/test.py @@ -126,7 +126,7 @@ def GetFailureOutput(self, failure): if failure.output.stdout: output += ["--- stdout ---"] output += [failure.output.stdout.strip()] - output += ["Command: %s" % EscapeCommand(failure.command)] + output += ["Command: %s" % failure.test.GetFailureCommand(failure.command)] if failure.HasCrashed(): output += ["--- %s ---" % PrintCrashed(failure.output.exit_code)] if failure.HasTimedOut(): @@ -360,9 +360,7 @@ def HasRun(self, output): # Print test name as (for example) "parallel/test-assert". Tests that are # scraped from the addons documentation are all named test.js, making it # hard to decipher what test is running when only the filename is printed. - prefix = abspath(join(dirname(__file__), '../test')) + os.sep - command = output.command[-1] - command = NormalizePath(command, prefix) + command = output.test.GetReportingName(output.command) if output.UnexpectedOutput(): status_line = 'not ok %i %s' % (self._done, command) @@ -425,9 +423,7 @@ def HasRun(self, output): # Print test name as (for example) "parallel/test-assert". Tests that are # scraped from the addons documentation are all named test.js, making it # hard to decipher what test is running when only the filename is printed. - prefix = abspath(join(dirname(__file__), '../test')) + os.sep - command = output.command[-1] - command = NormalizePath(command, prefix) + command = output.test.GetReportingName(output.command) stdout = output.output.stdout.strip() printed_file = False @@ -473,7 +469,7 @@ def HasRun(self, output): stderr = output.output.stderr.strip() if len(stderr): print(self.templates['stderr'] % stderr) - print("Command: %s" % EscapeCommand(output.command)) + print("Command: %s" % output.test.GetFailureCommand(output.command)) if output.HasCrashed(): print("--- %s ---" % PrintCrashed(output.output.exit_code)) if output.HasTimedOut(): @@ -573,6 +569,13 @@ def __init__(self, context, path, arch, mode): self.serial_id = 0 self.thread_id = 0 + def GetReportingName(self, command): + prefix = abspath(join(dirname(__file__), '../test')) + os.sep + return NormalizePath(command[-1], prefix) + + def GetFailureCommand(self, command): + return EscapeCommand(command) + def IsNegative(self): return self.context.expect_fail @@ -1545,6 +1548,8 @@ def NormalizePath(path, prefix='test/'): path = path.replace('\\', '/') if path.startswith(prefix): path = path[len(prefix):] + if '?' in path or '#' in path: + return path if path.endswith('.js'): path = path[:-3] elif path.endswith('.mjs'): @@ -1800,7 +1805,8 @@ def Main(): sys.exit(1) def should_keep(case): - if any((s in case.file) for s in options.skip_tests): + if any(s in case.file or s in '/'.join(case.path) + for s in options.skip_tests): return False elif SKIP in case.outcomes: return False @@ -1826,7 +1832,7 @@ def should_keep(case): # Must ensure the list of tests is sorted before selecting, to avoid # silent errors if this file is changed to list the tests in a way that # can be different in different machines - cases_to_run.sort(key=lambda c: (c.arch, c.mode, c.file)) + cases_to_run.sort(key=lambda c: (c.arch, c.mode, c.file, c.path)) cases_to_run = [ cases_to_run[i] for i in range(options.run[0], len(cases_to_run), @@ -1860,7 +1866,7 @@ def should_keep(case): elif result['failed']: print("\nFailed tests:") for failure in result['failed']: - print(EscapeCommand(failure.command)) + print(failure.test.GetFailureCommand(failure.command)) else: print("\nTest aborted.") return exitcode From c518831d001229e034513aacedaa1f45042603d4 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Wed, 9 Sep 2026 00:13:04 +0000 Subject: [PATCH 111/217] benchmark: add --csv option to compare.js with --analyze Add a `--csv {filename}` option to benchmark/compare.js to capture the CSV when the `--analyze` option is used Signed-off-by: James M Snell Assisted-by: Opencode PR-URL: https://github.com/nodejs/node/pull/65922 Reviewed-By: Antoine du Hamel Reviewed-By: Robert Nagy Reviewed-By: Filip Skokan Reviewed-By: Matteo Collina --- benchmark/compare.js | 102 ++++++++++-------- .../writing-and-running-benchmarks.md | 17 +++ test/parallel/test-benchmark-compare.js | 46 ++++++++ 3 files changed, 123 insertions(+), 42 deletions(-) create mode 100644 test/parallel/test-benchmark-compare.js diff --git a/benchmark/compare.js b/benchmark/compare.js index 77874e8af6c1..38fba5ba267b 100644 --- a/benchmark/compare.js +++ b/benchmark/compare.js @@ -1,6 +1,7 @@ 'use strict'; const { spawn, fork } = require('node:child_process'); +const { closeSync, openSync, writeSync } = require('node:fs'); const { inspect } = require('util'); const path = require('path'); const CLI = require('./_cli.js'); @@ -27,7 +28,9 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] ... --no-progress don't show benchmark progress indicator --analyze perform statistical analysis after benchmarks complete (Welch's t-test, effect size) instead - of printing csv output + of printing csv output to stdout + --csv filename write csv output to filename (can be combined + with --analyze). Use - to write to stdout. --scale 1000 rate-to-integer multiplier for histogram precision when using --analyze (default: 1000) --max-regression N exit with code 1 if any statistically @@ -60,6 +63,16 @@ if (benchmarks.length === 0) { return; } +const cvsToStdout = cli.optional.csv === '-'; +const csvFd = cli.optional.csv === undefined || cvsToStdout ? + null : + openSync(cli.optional.csv, 'w'); +const outputCsv = !analyze || csvFd !== null || cvsToStdout; + +function writeCsv(line) { + writeSync(csvFd || process.stdout.fd, `${line}\n`); +} + // When --analyze is set, collect results for statistical analysis. const results = analyze ? new Map() : null; @@ -78,17 +91,19 @@ for (const filename of benchmarks) { } // queue.length = binary.length * runs * benchmarks.length -// Print csv header (unless analyzing inline). -if (!analyze) { - console.log('"binary","filename","configuration","rate","time"'); +// Print csv header unless only analyzing inline. +if (outputCsv) { + writeCsv('"binary","filename","configuration","rate","time"'); } const kStartOfQueue = 0; -const showProgress = !cli.optional['no-progress']; +const showProgress = !cli.optional['no-progress'] && !cvsToStdout; let progress; if (showProgress) { - progress = new BenchmarkProgress(queue, benchmarks, { analyze }); + progress = new BenchmarkProgress(queue, benchmarks, { + analyze: analyze || csvFd !== null, + }); progress.startQueue(kStartOfQueue); } @@ -126,11 +141,13 @@ if (showProgress) { results.set(name, { old: [], new: [] }); } results.get(name)[job.binary].push(data.rate); - } else { + } + + if (outputCsv) { // Escape quotes (") for correct csv formatting - conf = conf.replace(/"/g, '""'); - console.log(`"${job.binary}","${job.filename}","${conf}",` + - `${data.rate},${data.time}`); + const csvConf = conf.replace(/"/g, '""'); + writeCsv(`"${job.binary}","${job.filename}","${csvConf}",` + + `${data.rate},${data.time}`); } if (showProgress) { // One item in the subqueue has been completed. @@ -153,8 +170,9 @@ if (showProgress) { // If there are more benchmarks execute the next if (i + 1 < queue.length) { recursive(i + 1); - } else if (analyze) { - printAnalysis(results, scale, maxRegression); + } else { + if (csvFd !== null) closeSync(csvFd); + if (analyze) printAnalysis(results, scale, maxRegression); } }); })(kStartOfQueue); @@ -261,41 +279,41 @@ function printAnalysis(results, scale, maxRegression) { const pad = (s, n) => s + ' '.repeat(Math.max(0, n - s.length)); const rpad = (s, n) => ' '.repeat(Math.max(0, n - s.length)) + s; - console.log(`${pad('', maxNameLen)} confidence` + - ` improvement accuracy (*) (**) (***)`); + writeSync(process.stdout.fd, `${pad('', maxNameLen)} confidence` + + ` improvement accuracy (*) (**) (***)\n`); for (const row of rows) { const imp = `${row.improvement >= 0 ? '+' : ''}${row.improvement.toFixed(2)} %`; - console.log( - `${pad(row.name, maxNameLen)} ${pad(row.stars, 10)}` + + writeSync(process.stdout.fd, + `${pad(row.name, maxNameLen)} ${pad(row.stars, 10)}` + ` ${rpad(imp, 11)}` + ` ±${row.ci95.toFixed(2)}%` + ` ±${row.ci99.toFixed(2)}%` + ` ±${row.ci999.toFixed(2)}%` + - `${row.inconclusive ? ' (inconclusive)' : ''}`, + `${row.inconclusive ? ' (inconclusive)' : ''}\n`, ); } if (skipped > 0) { - console.log(''); - console.log( - `Note: ${skipped} configuration${skipped === 1 ? ' was' : 's were'}` + + writeSync(process.stdout.fd, '\n'); + writeSync(process.stdout.fd, + `Note: ${skipped} configuration${skipped === 1 ? ' was' : 's were'}` + ` skipped because Welch's t-test requires at least 2 samples per` + - ` binary. Use --runs 2 or higher.`, + ` binary. Use --runs 2 or higher.\n`, ); } // --- Bar chart visualization --- printChart(rows, maxNameLen); - console.log(''); - console.log( - `Rates were scaled by ${scale}x into HdrHistogram (3 significant figures).\n` + - `Use --scale to adjust precision if needed.\n`, + writeSync(process.stdout.fd, '\n'); + writeSync(process.stdout.fd, + `Rates were scaled by ${scale}x into HdrHistogram (3 significant figures).\n` + + `Use --scale to adjust precision if needed.\n\n`, ); const anyFamilyWise = rows.filter((r) => r.pAdjusted < 0.05).length; - console.log( - `Be aware that when doing many comparisons the risk of a false-positive\n` + + writeSync(process.stdout.fd, + `Be aware that when doing many comparisons the risk of a false-positive\n` + `result increases. In this case, there are ${rows.length} comparisons, ` + `you can thus\nexpect the following amount of false-positive results:\n` + ` ${(rows.length * 0.05).toFixed(2)} false positives, when considering ` + @@ -307,19 +325,19 @@ function printAnalysis(results, scale, maxRegression) { `\nThe stars above are per-benchmark and uncorrected. Adjusting for the ` + `size of\nthis comparison set (Holm-Bonferroni), ${anyFamilyWise} ` + `comparison${anyFamilyWise === 1 ? '' : 's'} remain${anyFamilyWise === 1 ? 's' : ''} ` + - `significant at 5%.\n--max-regression uses the corrected values.`, + `significant at 5%.\n--max-regression uses the corrected values.\n`, ); // Gate: exit with error if any regression is shown to exceed the limit. if (maxRegression > 0) { if (underpowered > 0) { - console.log(''); - console.log( - `Note: ${underpowered} of ${rows.length} comparison` + + writeSync(process.stdout.fd, '\n'); + writeSync(process.stdout.fd, + `Note: ${underpowered} of ${rows.length} comparison` + `${rows.length === 1 ? '' : 's'} could not resolve an effect as ` + `small as ${maxRegression}%, and are marked (inconclusive). They are ` + `not\nevidence of no regression -- the samples are too noisy to tell. ` + - `Raise --runs,\nor pin cores with --set CPUSET, to narrow them.`, + `Raise --runs,\nor pin cores with --set CPUSET, to narrow them.\n`, ); } @@ -340,18 +358,18 @@ function printAnalysis(results, scale, maxRegression) { ); if (failures.length > 0) { - console.log(''); - console.log( - `FAIL: ${failures.length} benchmark${failures.length === 1 ? '' : 's'}` + + writeSync(process.stdout.fd, '\n'); + writeSync(process.stdout.fd, + `FAIL: ${failures.length} benchmark${failures.length === 1 ? '' : 's'}` + ` regressed by more than ${maxRegression}%` + ` (interval excludes the threshold,\n` + - `family-wise corrected across ${rows.length} comparisons):`, + `family-wise corrected across ${rows.length} comparisons):\n`, ); for (const f of failures) { - console.log( - ` ${f.name} ${f.improvement.toFixed(2)}% ` + + writeSync(process.stdout.fd, + ` ${f.name} ${f.improvement.toFixed(2)}% ` + `(95% CI up to ${(f.improvement + f.ci95).toFixed(2)}%, ` + - `adjusted p=${f.pAdjusted.toExponential(2)})`, + `adjusted p=${f.pAdjusted.toExponential(2)})\n`, ); } process.exitCode = 1; @@ -388,8 +406,8 @@ function printChart(rows, maxNameLen) { axisCenter + ' '.repeat(Math.max(0, halfWidth - Math.ceil(axisCenter.length / 2) - axisRight.length)) + axisRight; - console.log(''); - console.log(leftLabel); + writeSync(process.stdout.fd, '\n'); + writeSync(process.stdout.fd, `${leftLabel}\n`); for (const row of rows) { const imp = row.improvement; @@ -421,6 +439,6 @@ function printChart(rows, maxNameLen) { const label = `${row.improvement >= 0 ? '+' : ''}${row.improvement.toFixed(2)}%`; const sig = row.stars.trim(); - console.log(`${pad(row.name, maxNameLen)} ${chars.join('')} ${label} ${sig}`); + writeSync(process.stdout.fd, `${pad(row.name, maxNameLen)} ${chars.join('')} ${label} ${sig}\n`); } } diff --git a/doc/contributing/writing-and-running-benchmarks.md b/doc/contributing/writing-and-running-benchmarks.md index a4c8e20244cd..bf397c123a24 100644 --- a/doc/contributing/writing-and-running-benchmarks.md +++ b/doc/contributing/writing-and-running-benchmarks.md @@ -416,6 +416,8 @@ module, you can use the `--filter` option:_ --set variable=value set benchmark variable (can be repeated) --no-progress don't show benchmark progress indicator --analyze perform statistical analysis inline (no R needed) + --csv filename write csv output to filename (can be combined + with --analyze) --scale 1000 rate multiplier for --analyze precision --max-regression N exit with code 1 if any significant regression exceeds N% (implies --analyze) @@ -429,6 +431,14 @@ The simplest way to get statistical results is to pass `--analyze`: node benchmark/compare.js --old ./node-main --new ./node-pr-5134 --analyze string_decoder ``` +Use `--csv` to retain the raw benchmark results. If you pass both `--csv` and +`--analyze`, both the raw results and the analysis are printed: + +```bash +node benchmark/compare.js --old ./node-main --new ./node-pr-5134 \ + --analyze --csv compare-pr-5134.csv string_decoder +``` + This runs the benchmarks and prints the analysis directly: ```console @@ -438,6 +448,13 @@ string_decoder/string-decoder.js n=2500000 chunkLen=16 inLen=128 encoding='utf8' ... ``` +Use `-csv -` to output the raw results to stdout with the analysis. + +```bash +node benchmark/compare.js --old ./node-main --new ./node-pr-5134 \ + --analyze --csv - string_decoder +``` + The `--analyze` mode uses the histogram API's `welchTest()` method to perform the same Welch's t-test that the R script uses. Benchmark rates are scaled to integers for the histogram (controlled by `--scale`, default 1000). With the diff --git a/test/parallel/test-benchmark-compare.js b/test/parallel/test-benchmark-compare.js new file mode 100644 index 000000000000..29795d6b743a --- /dev/null +++ b/test/parallel/test-benchmark-compare.js @@ -0,0 +1,46 @@ +'use strict'; + +require('../common'); + +const assert = require('node:assert'); +const { spawnSyncAndExitWithoutError } = require('../common/child_process'); +const { readFileSync } = require('node:fs'); +const path = require('node:path'); +const tmpdir = require('../common/tmpdir'); + +const compare = path.resolve(__dirname, '../../benchmark/compare.js'); + +tmpdir.refresh(); + +const csv = tmpdir.resolve('compare.csv'); +spawnSyncAndExitWithoutError(process.execPath, [ + compare, + '--old', process.execPath, + '--new', process.execPath, + '--runs', '1', + '--filter', 'buffer-compare-offset.js', + '--set', 'method=offset', + '--set', 'size=16', + '--set', 'n=1', + '--no-progress', + '--analyze', + '--csv', csv, + 'buffers', +], { + encoding: 'utf8', + timeout: 30_000, +}, { + stderr: '', + stdout(stdout) { + assert.match(stdout, /confidence\s+improvement\s+accuracy/); + assert.doesNotMatch(stdout, /"binary","filename"/); + }, +}); + +const lines = readFileSync(csv, 'utf8').trim().split('\n'); +const filename = path.join('buffers', 'buffer-compare-offset.js'); +assert.strictEqual(lines[0], + '"binary","filename","configuration","rate","time"'); +assert.strictEqual(lines.length, 3); +assert(lines[1].startsWith(`"old","${filename}",`)); +assert(lines[2].startsWith(`"new","${filename}",`)); From fa19ecad9d07a8eda9cf61517da15301677d21b8 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Wed, 9 Sep 2026 06:02:35 +0000 Subject: [PATCH 112/217] test: improve sequential test performance Make a couple of sequential tests clean up eagerly to reduce runtime. Signed-off-by: James M Snell Assisted-by: Opencode PR-URL: https://github.com/nodejs/node/pull/65928 Reviewed-By: Luigi Pinca Reviewed-By: Filip Skokan --- test/sequential/test-net-connect-econnrefused.js | 11 ++++------- test/sequential/test-pipe.js | 5 ++++- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/sequential/test-net-connect-econnrefused.js b/test/sequential/test-net-connect-econnrefused.js index 67f5820221c5..c51e99e59b85 100644 --- a/test/sequential/test-net-connect-econnrefused.js +++ b/test/sequential/test-net-connect-econnrefused.js @@ -32,7 +32,7 @@ let rounds = 1; let reqs = 0; let port; -const server = net.createServer().listen(0, common.mustCall(() => { +const server = net.createServer().listen(0, common.localhostIPv4, common.mustCall(() => { port = server.address().port; server.close(common.mustCall(pummel)); })); @@ -40,17 +40,14 @@ const server = net.createServer().listen(0, common.mustCall(() => { function pummel() { let pending; for (pending = 0; pending < ATTEMPTS_PER_ROUND; pending++) { - net.createConnection({ port, autoSelectFamily: false }).on('error', common.mustCallAtLeast((error) => { - // Family autoselection might be skipped if only a single address is returned by DNS. - const actualError = Array.isArray(error.errors) ? error.errors[0] : error; - + net.createConnection({ host: common.localhostIPv4, port }).on('error', common.mustCall((error) => { console.log('pending', pending, 'rounds', rounds); - assert.strictEqual(actualError.code, 'ECONNREFUSED'); + assert.strictEqual(error.code, 'ECONNREFUSED'); if (--pending > 0) return; if (rounds === ROUNDS) return check(); rounds++; pummel(); - }, 0)); + })); reqs++; } } diff --git a/test/sequential/test-pipe.js b/test/sequential/test-pipe.js index 7515e4c705b0..39b11e17d51a 100644 --- a/test/sequential/test-pipe.js +++ b/test/sequential/test-pipe.js @@ -93,7 +93,10 @@ function startClient() { port: common.PORT, method: 'GET', path: '/', - headers: { 'content-length': buffer.length }, + headers: { + 'connection': 'close', + 'content-length': buffer.length, + }, }, common.mustCall((res) => { res.setEncoding('utf8'); res.on('data', common.mustCall((string) => { From 5a5abd8fd08c335a8898cf28e87ff6c1ecdfeda2 Mon Sep 17 00:00:00 2001 From: Colin McDonnell <3084745+colinhacks@users.noreply.github.com> Date: Sun, 13 Sep 2026 23:44:37 -0700 Subject: [PATCH 113/217] src: avoid copying SEA snapshot data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Colin McDonnell <3084745+colinhacks@users.noreply.github.com> PR-URL: https://github.com/nodejs/node/pull/65876 Reviewed-By: Filip Skokan Reviewed-By: Gürgün Dayıoğlu --- src/env.h | 7 +++++- src/node.cc | 6 ++++- src/node_snapshotable.cc | 52 +++++++++++++++++++++++++++------------- 3 files changed, 46 insertions(+), 19 deletions(-) diff --git a/src/env.h b/src/env.h index 29754ddb6371..9149b91fe4bb 100644 --- a/src/env.h +++ b/src/env.h @@ -640,6 +640,7 @@ struct SnapshotData { // The result of v8::SnapshotCreator::CreateBlob() during the snapshot // building process. v8::StartupData v8_snapshot_blob_data{nullptr, 0}; + DataOwnership v8_snapshot_blob_data_ownership = DataOwnership::kOwned; IsolateDataSerializeInfo isolate_data_info; // TODO(joyeecheung): there should be a vector of env_info once we snapshot @@ -659,7 +660,11 @@ struct SnapshotData { bool Check() const; static bool FromFile(SnapshotData* out, FILE* in); static bool FromBlob(SnapshotData* out, const std::vector& in); - static bool FromBlob(SnapshotData* out, std::string_view in); + // If the V8 data is not owned, `in` must outlive `out`. + static bool FromBlob( + SnapshotData* out, + std::string_view in, + DataOwnership v8_snapshot_blob_data_ownership = DataOwnership::kOwned); static const SnapshotData* FromEmbedderWrapper( const EmbedderSnapshotData* data); diff --git a/src/node.cc b/src/node.cc index 959e6552cb7e..581d0f1986a4 100644 --- a/src/node.cc +++ b/src/node.cc @@ -1574,7 +1574,11 @@ bool LoadSnapshotData(const SnapshotData** snapshot_data_ptr) { std::unique_ptr read_data = std::make_unique(); std::string_view snapshot = sea.main_code_or_snapshot; - if (SnapshotData::FromBlob(read_data.get(), snapshot)) { + // The SEA resource remains mapped for the process lifetime, so V8 can + // consume the startup data directly from the executable image. + if (SnapshotData::FromBlob(read_data.get(), + snapshot, + SnapshotData::DataOwnership::kNotOwned)) { *snapshot_data_ptr = read_data.release(); return true; } else { diff --git a/src/node_snapshotable.cc b/src/node_snapshotable.cc index b23c73b1c44f..2624b73178b1 100644 --- a/src/node_snapshotable.cc +++ b/src/node_snapshotable.cc @@ -158,6 +158,30 @@ class SnapshotDeserializer : public BlobDeserializer { template requires(!std::is_arithmetic_v && !std::same_as) T Read(); + + v8::StartupData ReadV8StartupData(SnapshotData::DataOwnership ownership) { + Debug("Read()\n"); + + int raw_size = ReadArithmetic(); + Debug("size=%d\n", raw_size); + + if (raw_size <= 0 || + static_cast(raw_size) > sink.size() - read_total) { + ok = false; + return v8::StartupData{nullptr, 0}; + } + if (ownership == SnapshotData::DataOwnership::kOwned) { + // The data pointer of v8::StartupData would be deleted so it must be + // new'ed. + char* buf = new char[raw_size]; + ReadArithmetic(buf, raw_size); + return v8::StartupData{buf, raw_size}; + } + + const char* data = sink.data() + read_total; + read_total += raw_size; + return v8::StartupData{data, raw_size}; + } }; class SnapshotSerializer : public BlobSerializer { @@ -181,21 +205,7 @@ class SnapshotSerializer : public BlobSerializer { // [ |raw_size| bytes ] contents template <> v8::StartupData SnapshotDeserializer::Read() { - Debug("Read()\n"); - - int raw_size = ReadArithmetic(); - Debug("size=%d\n", raw_size); - - if (raw_size <= 0 || - static_cast(raw_size) > sink.size() - read_total) { - ok = false; - return v8::StartupData{nullptr, 0}; - } - // The data pointer of v8::StartupData would be deleted so it must be new'ed. - char* buf = new char[raw_size]; - ReadArithmetic(buf, raw_size); - - return v8::StartupData{buf, raw_size}; + return ReadV8StartupData(SnapshotData::DataOwnership::kOwned); } template <> @@ -640,7 +650,9 @@ bool SnapshotData::FromBlob(SnapshotData* out, const std::vector& in) { return FromBlob(out, std::string_view(in.data(), in.size())); } -bool SnapshotData::FromBlob(SnapshotData* out, std::string_view in) { +bool SnapshotData::FromBlob(SnapshotData* out, + std::string_view in, + DataOwnership v8_snapshot_blob_data_ownership) { SnapshotDeserializer r(in); r.Debug("SnapshotData::FromBlob()\n"); @@ -660,7 +672,9 @@ bool SnapshotData::FromBlob(SnapshotData* out, std::string_view in) { return false; } - out->v8_snapshot_blob_data = r.Read(); + out->v8_snapshot_blob_data = + r.ReadV8StartupData(v8_snapshot_blob_data_ownership); + out->v8_snapshot_blob_data_ownership = v8_snapshot_blob_data_ownership; r.Debug("Read isolate_data_info\n"); out->isolate_data_info = r.Read(); out->env_info = r.Read(); @@ -709,6 +723,7 @@ bool SnapshotData::Check() const { SnapshotData::~SnapshotData() { if (data_ownership == DataOwnership::kOwned && + v8_snapshot_blob_data_ownership == DataOwnership::kOwned && v8_snapshot_blob_data.data != nullptr && !IsFirstSnapshotBlob(v8_snapshot_blob_data.data)) { delete[] v8_snapshot_blob_data.data; @@ -831,6 +846,9 @@ namespace node { // -- v8_snapshot_blob_data begins -- { v8_snapshot_blob_data, v8_snapshot_blob_size }, // -- v8_snapshot_blob_data ends -- + // -- v8_snapshot_blob_data_ownership begins -- + SnapshotData::DataOwnership::kNotOwned, + // -- v8_snapshot_blob_data_ownership ends -- // -- isolate_data_info begins -- )" << data->isolate_data_info << R"( From 88b12c03cd8f2c2a8d32a05846a8eb24833326de Mon Sep 17 00:00:00 2001 From: Yuya Inoue <65857152+inoway46@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:11:10 +0900 Subject: [PATCH 114/217] test_runner: avoid reusing v8 serializers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A serializer must not be used after releaseBuffer() is called. Use a dedicated instance to calculate the header length and create a new serializer for each test event. Add a regression test that serializes the same object twice and verifies that both frames can be deserialized independently. Signed-off-by: inoway46 Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/65951 Reviewed-By: Stefan Stojanovic Reviewed-By: Moshe Atlow Reviewed-By: Chemi Atlow Reviewed-By: James M Snell Reviewed-By: Ulises Gascón --- lib/internal/test_runner/reporter/v8-serializer.js | 7 ++++--- test/parallel/test-runner-v8-deserializer.mjs | 9 +++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/internal/test_runner/reporter/v8-serializer.js b/lib/internal/test_runner/reporter/v8-serializer.js index c75bfcdac478..0f0cf20902b5 100644 --- a/lib/internal/test_runner/reporter/v8-serializer.js +++ b/lib/internal/test_runner/reporter/v8-serializer.js @@ -9,11 +9,12 @@ const { serializeError } = require('internal/error_serdes'); module.exports = async function* v8Reporter(source) { - const serializer = new DefaultSerializer(); - serializer.writeHeader(); - const headerLength = TypedArrayPrototypeGetLength(serializer.releaseBuffer()); + const headerSerializer = new DefaultSerializer(); + headerSerializer.writeHeader(); + const headerLength = TypedArrayPrototypeGetLength(headerSerializer.releaseBuffer()); for await (const item of source) { + const serializer = new DefaultSerializer(); const originalError = item.data.details?.error; if (originalError) { // Error is overridden with a serialized version, so that it can be diff --git a/test/parallel/test-runner-v8-deserializer.mjs b/test/parallel/test-runner-v8-deserializer.mjs index 7f2c0155c973..3a4db367ca6d 100644 --- a/test/parallel/test-runner-v8-deserializer.mjs +++ b/test/parallel/test-runner-v8-deserializer.mjs @@ -85,6 +85,15 @@ describe('v8 deserializer', common.mustCall(() => { assert.deepStrictEqual(reported, [reportedDiagnosticEvent]); }); + it('should serialize a repeated object as independent messages', async () => { + const repeatedChunks = await toArray(serializer([diagnosticEvent, diagnosticEvent])); + const reported = await collectReported(repeatedChunks); + assert.deepStrictEqual(reported, [ + reportedDiagnosticEvent, + reportedDiagnosticEvent, + ]); + }); + it('should deserialize a serialized chunk after non-serialized chunk', async () => { const reported = await collectReported([Buffer.concat([Buffer.from('unknown'), ...chunks])]); assert.deepStrictEqual(reported, [ From 205443721d214ceb683dbd57e41d4e3df04ffd70 Mon Sep 17 00:00:00 2001 From: Sankalp Thakur <31366524+sankalpsthakur@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:57:19 +0530 Subject: [PATCH 115/217] http2: fix onread assert when destroying session from stream handler When session.destroy() runs from a 'stream' handler, MakeCallback drains nextTick while nghttp2 is still inside mem_recv. Close is deferred for that window (see #64166), so later HEADERS in the same buffer created C++ streams without a JS wrapper or onread, and DATA delivery aborted with Assertion failed: onread->IsFunction(). - Reject new streams while the session is closing - Destroy the C++ handle if on_headers runs after JS destroy - Drop DATA when onread is not installed (defensive) Fixes: https://github.com/nodejs/node/issues/64850 Signed-off-by: Sankalp Thakur PR-URL: https://github.com/nodejs/node/pull/65116 Reviewed-By: Matteo Collina Reviewed-By: Tim Perry --- lib/internal/http2/core.js | 9 ++- src/node_http2.cc | 10 ++++ ...st-http2-session-destroy-stream-handler.js | 60 +++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 test/parallel/test-http2-session-destroy-stream-handler.js diff --git a/lib/internal/http2/core.js b/lib/internal/http2/core.js index fcf94db8a1e1..286bae090a10 100644 --- a/lib/internal/http2/core.js +++ b/lib/internal/http2/core.js @@ -358,8 +358,15 @@ function emit(self, ...args) { // the block of headers on. function onSessionHeaders(handle, id, cat, flags, headers, sensitiveHeaders) { const session = this[kOwner]; - if (session.destroyed) + // Session may have been destroyed mid-receive (e.g. session.destroy() from a + // 'stream' handler drained via nextTick inside MakeCallback while nghttp2 is + // still walking the receive buffer). Tear down the C++ stream so subsequent + // DATA frames do not call CallJSOnreadMethod with a missing onread. + if (session.destroyed) { + handle.rstStream(NGHTTP2_REFUSED_STREAM); + handle.destroy(); return; + } const type = session[kType]; session[kUpdateTimer](); diff --git a/src/node_http2.cc b/src/node_http2.cc index d6edcd28d117..0589b5f6844a 100644 --- a/src/node_http2.cc +++ b/src/node_http2.cc @@ -1049,6 +1049,16 @@ int Http2Session::OnBeginHeadersCallback(nghttp2_session* handle, // The common case is that we're creating a new stream. The less likely // case is that we're receiving a set of trailers if (!stream) [[likely]] { + // Close() may be deferred while mem_recv is in progress (see + // Http2Session::Close). A 'stream' handler that calls session.destroy() + // runs via nextTick from MakeCallback during that window, so later + // HEADERS in the same receive buffer must not create a C++ stream + // whose JS wrapper (and onread) is never installed. + if (session->is_closing()) { + nghttp2_submit_rst_stream( + session->session(), NGHTTP2_FLAG_NONE, id, NGHTTP2_REFUSED_STREAM); + return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + } if (!session->CanAddStream() || Http2Stream::New(session, id, frame->headers.cat) == nullptr) [[unlikely]] { diff --git a/test/parallel/test-http2-session-destroy-stream-handler.js b/test/parallel/test-http2-session-destroy-stream-handler.js new file mode 100644 index 000000000000..35c5b16471ef --- /dev/null +++ b/test/parallel/test-http2-session-destroy-stream-handler.js @@ -0,0 +1,60 @@ +'use strict'; + +const common = require('../common'); +if (!common.hasCrypto) + common.skip('missing crypto'); + +const http2 = require('http2'); + +// Regression test for https://github.com/nodejs/node/issues/64850 +// +// Destroying the session from a 'stream' handler runs (via nextTick drained +// from MakeCallback) while nghttp2 is still inside mem_recv. Close is deferred +// for that window; later HEADERS/DATA in the same buffer must not abort with +// Assertion failed: onread->IsFunction(). + +const STREAMS = 8; +const BODY = Buffer.alloc(2048, 'a'); +const ROUNDS = 40; + +const server = http2.createServer({ + settings: { maxConcurrentStreams: 4 }, +}); + +server.on('session', (session) => session.on('error', () => {})); + +server.on('stream', (stream) => { + stream.on('error', () => {}); + stream.session.destroy(); +}); + +server.listen(0, '127.0.0.1', common.mustCall(() => { + const port = server.address().port; + const origin = `http://127.0.0.1:${port}`; + let remaining = ROUNDS; + + const round = () => { + if (remaining-- <= 0) { + server.close(); + return; + } + + const session = http2.connect(origin); + session.on('error', () => {}); + session.on('close', () => setImmediate(round)); + + session.on('connect', () => { + for (let i = 0; i < STREAMS; i++) { + const stream = session.request({ + ':path': `/${i}`, + ':method': 'POST', + }); + stream.on('error', () => {}); + stream.resume(); + stream.end(BODY); + } + }); + }; + + round(); +})); From 6515db8ab6a7be7392d06467881a9ca3d0bfcb34 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Mon, 14 Sep 2026 13:44:13 +0200 Subject: [PATCH 116/217] test: fix RSA/DSA wrong-passphrase flake Re-encrypting keys on each FIPS test run can produce ciphertext that decrypts with valid padding under the wrong password, causing a decoder error instead of the expected bad decrypt. Signed-off-by: Filip Skokan Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/65983 Refs: https://github.com/nodejs/node/actions/runs/34595689099/job/103251404626?pr=65980 Reviewed-By: James M Snell Reviewed-By: Matteo Collina --- test/fixtures/keys/Makefile | 6 +- test/fixtures/keys/dsa_private_encrypted.pem | 41 ++++++------- test/fixtures/keys/rsa_private_encrypted.pem | 60 ++++++++++---------- test/parallel/test-crypto-rsa-dsa.js | 31 ++-------- test/parallel/test-tls-passphrase.js | 45 ++++----------- 5 files changed, 67 insertions(+), 116 deletions(-) diff --git a/test/fixtures/keys/Makefile b/test/fixtures/keys/Makefile index 0960e69f36fd..bad4b6aed3ca 100644 --- a/test/fixtures/keys/Makefile +++ b/test/fixtures/keys/Makefile @@ -844,7 +844,8 @@ dsa_private.pem: dsa_params.pem openssl gendsa -out dsa_private.pem dsa_params.pem dsa_private_encrypted.pem: dsa_private.pem - openssl dsa -aes256 -in dsa_private.pem -passout 'pass:password' -out dsa_private_encrypted.pem + openssl pkcs8 -topk8 -v2 aes-256-cbc -v2prf hmacWithSHA256 -iter 2048 -saltlen 16 \ + -in $< -passout 'pass:password' -out $@ dsa_private_pkcs8.pem: dsa_private.pem openssl pkcs8 -topk8 -inform PEM -outform PEM -in dsa_private.pem -out dsa_private_pkcs8.pem -nocrypt @@ -868,7 +869,8 @@ rsa_private.pem: openssl genrsa -out rsa_private.pem 2048 rsa_private_encrypted.pem: rsa_private.pem - openssl rsa -aes256 -in rsa_private.pem -passout 'pass:password' -out rsa_private_encrypted.pem + openssl pkcs8 -topk8 -v2 aes-256-cbc -v2prf hmacWithSHA256 -iter 2048 -saltlen 16 \ + -in $< -passout 'pass:password' -out $@ rsa_private_pkcs8.pem: rsa_private.pem openssl pkcs8 -topk8 -inform PEM -outform PEM -in rsa_private.pem -out rsa_private_pkcs8.pem -nocrypt diff --git a/test/fixtures/keys/dsa_private_encrypted.pem b/test/fixtures/keys/dsa_private_encrypted.pem index 49b3375baf71..88da77e0a3b7 100644 --- a/test/fixtures/keys/dsa_private_encrypted.pem +++ b/test/fixtures/keys/dsa_private_encrypted.pem @@ -1,23 +1,18 @@ ------BEGIN DSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: AES-256-CBC,FABA263DD471F214EF3E02699B837C20 - -Tj2+4x9MEIaQGFQ4o7hk12MriVYyvLO5aCbqq7LG5uhVk546/+bJc6hewdSwb6oT -MYPbuV+QTdtqshqFESA0McyGlj4w1tOg5TomP84NTKvwTO1EirVLMukfF3dqaguw -C117AZJkGbqgbi6lZ2bG0Hta6HRbhI5+ODFtOp3rKQ2KwVmtL7zw6vt3PCISeMHN -fLqikDc2+YoI9V1FJis9/FATyqV8yrJYYQQpP1RQN+gDY4SSs/eUr+Me7RNy6Lz7 -oH0tDaPGbiafwrZe1okksjxT2JQz1Q3hciBPikgdQIoE2NWTUlOeRYX0T0N2n37S -6Odbcr522e+2XjcLj34Ozthp+Q5mIDcLuakazxkXhq0RyhJ7vo+xA2YiP7Q3vH7g -oAnsJPFNVY6wJhprZi2VofKIUJUiajAXGDVX2yEIG/DOA9rnx0ZP+zopXMi4ptu0 -RzWyAL+P4jn0b8vgPf9CYJmn4VNfOcVmomZ1Bw6hzqTE2FnThJCXU3l2eaC/wcSR -uMRp8c6IM8AR5DUzUBKIckkvXj1m5iSZoKuR8dB7s9BhrRtBAI7K3G254G06sByv -0pnft8r+BkMqgdfG4rJQoQJw7tVYln+pL/gYPDuYsqyJ9kFuHDqtBvlozqXY5AL1 -XQaXoD6xMACEoJSIv5y+TzFzXwFQrDW+G1724YOSbiioUfGD0tRfjj2ei63PThQr -Z50SryfKQQf4UgcJeokMhmRWT2vPXEFWEP1b2FMEQxBy6fyKcqwZBAbhqF6usGEB -nwr/S1HXQAGEsWoc/Z4yynB7uhOwWu/Vpj+V6B98NmC7EUX15Why8zCsT9gzaIgC -M6sZafHhcmjfwc+lL9xFlU/wnAOz0LeKZWry3D0sXZn1r2FRlOJdtLLx01Sve/MU -ZRsgEDTkzv8E9dDltMeq8HQDCgLT1USTMWcY1kMELBj7y7ZdCWjH1QhTq2KlId+o -1X28zJOsOL/XRseUSlpjmSSLRw1QQEypNCY2+tcvViAvn3AifipBbdzUNhvygLhc -a2+5rYsd8BBEFnMJx7lDiyqXGnZkBbhbCSIudppNcjC+akFlFp6fBzkp4mKBuKpc -hwBBdfqdEyzqu6SVHM8nGV/aDoRuu9shV6MX0y/KnIgLedudn8aN2eLgjR5k1+99 ------END DSA PRIVATE KEY----- +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIC1TBfBgkqhkiG9w0BBQ0wUjAxBgkqhkiG9w0BBQwwJAQQUHwLuG5KIFDnU6OC +5ADEzQICCAAwDAYIKoZIhvcNAgkFADAdBglghkgBZQMEASoEEOrzdylfA3E36eSA +aLsD6C0EggJwrFQCTATMWun5ffHFf42epvxuRH3Dv96gOejJxmItv3fqF7cfn+oB +82gsGwdcjme6eGu7rg4lKi5e85WEImcg+xQdHnEmx62yP73CBfpzCs5KObbBd8ft +EjlqQ+EWQ68+acdHOZ/xDqBiJnkisEVEnsTo9ZE7ZLVBKRP6Hr31eQ/7EKWBb6Ax +oF+6+WgvyH8AuOO17CLjMOGO9pFl2TvJ1wsQtAIb2ZEAmPp0Z/lT9qXaoJfAz4pO +o3+2woDPDm9PA6WXF8zD+zPBwnSBSPjvkwtsMp1CdhrAxaeEo+gbDqJjCI9/2m8j +gAqkP+l7civZ7P7ydOPmlkiSdl4TXfhvfhk4RCSQbg6+YozggJO0ZiroCK7G5ro6 +4T+t2ck206kylOZV1xPrbW+Kt88vIVSxNs8dwHplEtb1VdoAV4EKfh3PEcsuSc4W +NEGvaAEmzckAMG3GJgjZYS8EWfpLsuNRcvRJsbMeQCC4Iex9m4+KJVUmMmIBlBxV +aoFEfYNGpkOdMV7l1Ne77c4HGI7NS6Yy+JcxYoc5kByTxaWOSvPSoQ3YQ+bY3GoT +tAQRvoieBnzkkyjKaS0Qgnfkdfe4Ry7hSjToo3mOnA8wj6sjRtZdIQihTa1jZ3vP +ELJ8lPTVBiPUC/AZ0C12s/JExY3y598Rlt7RQRA6ZN6PP9gnuuqR58sq/XoLnwBQ +tHK8kz5BU4EZ2neMi7noHDmArgh083WGFX6WUAfDu9R8+hydnee86Wm4xThXk5NS +TIGqwBal/5EgrSCbkcwxRw9xB0afMD4+muatKv6NEcRKT/VjhxdWP4gHONrh3tJF +/pn2+4MtHeK8 +-----END ENCRYPTED PRIVATE KEY----- diff --git a/test/fixtures/keys/rsa_private_encrypted.pem b/test/fixtures/keys/rsa_private_encrypted.pem index f1914289ec4f..6a605a055551 100644 --- a/test/fixtures/keys/rsa_private_encrypted.pem +++ b/test/fixtures/keys/rsa_private_encrypted.pem @@ -1,30 +1,30 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: AES-256-CBC,DB3D20E60E8FDC3356BD79712FF8EF7E - -K+vu0U3IFTJBBi6zW5Zng80O1jXq/ZmlOFs/j/SQpPwfW1Do9i/Dwa7ntBlTwrCm -sd3IIPgu2ikfLwxvbxsZN540oCaCqaZ/bmmyzH3MyVDA9MllUu+X8+Q3ATzcYa9R -U5XfF5DAXsSRnstCbmKagWVQpO0oX8k3ratfny6Ixq86Y82tK8+o5YiBFq1kqa+9 -4yat7IWQbqV5ifUtUPCHZwEqBt+WKazX05BqERjkckHdpfaDrBvSSPXTwoLm6uRR -ktkUVpO4tHMZ4VlcTfFtpz8gdYYod0nM6vz26hvbESHSwztSgMhmKdsE5eqmYfgu -F4WkEN4bqAiPjKK3jnUKPt/vg2oKYFQlVYFl9QnBjiRqcQTi3e9lwn1hI7uoMb6g -HuaCc57JJHPN/ZLP3ts4ZxFbwUjTGioh5Zh6WozG3L3+Ujwq/sDrAskRyzdcuP7I -Rs3oLbHY03OHyg8IbxR5Iu89l6FLqnR45yvbxXtZ7ImGOPM5Z9pB1CzDhGDx2F6g -J/Kf/7ZF2DmYUVbVKDfESEDhRfuMAVzhasDPTRqipSA5QvJVQY+J/6QDPrNNmHVB -4e4ouHIDWERUf0t1Be7THvP3X8OJozj2HApzqa5ZCaJDo8eaL8TCD5uH75ID5URJ -VscGHaUXT8/sxfHi1x8BibW5W5J/akFsnrnJU/1BZgGznIxjf5tKfHGppSIVdlKP -3ghYNmEIFPNJ6cxuUA0D2IOV4uO3FTCU6seIzvJhYkmXnticcZYGtmGxXKrodtzS -J1YuaNkkO/YRZah285lQ6QCIhCFo4Oa4ILjgoTQISuw7nQj5ESyncauzLUBXKX0c -XDUej64KNTvVF9UXdG48fYvNmSZWCnTye4UmPu17FmwpVra38U+EdoLyWyMIAI5t -rP6Hhgc9BxOo41Im9QpTcAPfKAknP8Rbm3ACJG5T9FKq/c29d1E//eFR6SL51e/a -yWdCgJN/FJOAX60+erPwoVoRFEttAeDPkklgFGdc8F4LIYAig9gEZ92ykFFz3fWz -jIcUVLrL+IokFbPVUBoMihqVyMQsWH+5Qq9wjxf6EDIf0BVtm9U4BJoOkPStFIfF -Kof7OVv7izyL8R/GIil9VQs9ftwkIUPeXx2Hw0bE3HJ3C8K4+mbLg3tKhGnBDU5Z -Xm5mLHoCRBa3ZRFWZtigX7POszdLAzftYo8o65Be4OtPS+tQAORk9gHsXATv7dDB -OGw61x5KA55LHVHhWaRvu3J8E7nhxw0q/HskyZhDC+Y+Xs6vmQSb4nO4ET4NYX1P -m3PMdgGoqRDJ2jZw4eoQdRKCM0EHSepSAYpO1tcAXhPZS4ITogoRgPpVgOebEQUL -nKNeNu/BxMSH/IH15jjDLF3TiEoguF9xdTaCxIBzE1SFpVO0u9m9vXpWdPThVgsb -VcEI487p7v9iImP3BYPT8ZYvytC26EH0hyOrwhahTvTb4vXghkLIyvPUg1lZHc6e -aPHb2AzYAHLnp/ehDQGKWrCOJ1JE2vBv8ZkLa+XZo7YASXBRZitPOMlvykEyzxmR -QAmNhKGvFmeM2mmHAp0aC03rgF3lxNsXQ1CyfEdq3UV9ReSnttq8gtrJfCwxV+wY ------END RSA PRIVATE KEY----- +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIFNTBfBgkqhkiG9w0BBQ0wUjAxBgkqhkiG9w0BBQwwJAQQ2cq1Ok3DMP2A/RoK +n3/9ZgICCAAwDAYIKoZIhvcNAgkFADAdBglghkgBZQMEASoEEGRKDlxB0C8WgbEc +enhkuocEggTQQK9exbwIpgV5QRoHDMACpXuWtsZgdpCanhtA53pP8tELYc4MzbIp +90dLQWxqi/mSlqr1ifoMnCFEAKVUV8mDoh0i5uvQhxOC6TB0HBGgb7dEgITIC7NT +PTeHNepQyW97SkTMKoTnOogxgWeAHKYsSALHJw+45q4qARE7vo1O8CHE6v6Liv4F +ZaW/8p2O4qUTRWGshUG6tC4/YN6r/iYtoSXUzOTiTeLckhV/9QsP5pGF3LQXIU9I +5xh+I1BdcSZ/7sGiHED0BB4yF2lOUKKUwCNFUTddhApewYm/yCW8wTYB7vvOI8mz +47V8rfPj40Qzislrc5Sm96XpQp8DrLV4KKkA8F1MPEchHgOcdq+1UKCRW2kfUOWS +T9wSXPHlmuKxHsrXk3pBGdnshbp2d5d4uwVhi9bPA7J2DN25ayIe9H8LbYfPo58C +CiCnjLG02nKYSLjFZR8K/z2j+xmGyXVcHuGZzR02O3G4G89VlxxfazxYbZZvSNbM +w6/TjbSDmvOYLRxRRdbZRWixgyn4QRmg5QmWBbqsS1YmVOG3HafHdAtS2B++DRF3 +S5Waaa/mrqzMesl+TLYJ759gHOqODoppm1iwmVzcBuoDLYNUI0AZZ/pEGs3oGk4M +faNiVMcsFkMWPSzgTv/viWJgJeQdWIos75nyTUgDJiFmxMT6HcWZxEFBM5IMqoZJ +JXfFGIwPo5pjw8kFHh/4Mj3qCuQ1jxrnRFdC3EhSy+JCQf0S4XTYws4GBZ4psRTX +KOb+VqizDL+0A9d88QIQX3BWfCZ6QhFHd4JdLhZt1myjsLu50NaraHBb/9pEcBUb +/hIKTxQCgXZMw1XaJeXRtbObYeB1JjPZYfuaxwvYIQ1CrPVHcCEZYH/GlQ2ppqCf +RIZY6Tp522nQ7l7uOhhJozkXZsbI5/MzaYQguz+dRAWBckQTAgStRJ2UeCP215CN +cSY/nSeIssjov+Rs2twi/AFglET7f0Osg2YXjEl6zRde80D7/74a/YASw/qGP/+q +nxwVmMpcLCfvfvb7JN0YofU5TLbOYt3lneydIe8aZhR15HlamnnZS8cs+2FeiFu2 +I4tH+7FAiB/HuYSTJSKWvTbZX100aCDqsyYW0UF4mrNgcIdjjrmWAJL2thcX5JmW ++PTTbDKGtsnGm5QUqtYHQTWelHNeWXwxveAWReG3d7v5vx+ioY1xWKkEzrRjGtNJ +nmNvFBSjlz28fxhlJ6XV3DO2N3VvPwiXLYKE99idl2mtX0WWSCcnsYEKWRSccxe7 +quwxYxWa1l4t07OUCbMCsTz4weBMc/7DK/xUSKETp/C8IR6o/F8iK49KjpIQFnjQ +abJy1RSNxRRawqDdLqZiLjAZydBq7lJ4R3LTqEdw4zQGpFdGa/YAshkCPsIGCj3q +8ONcd4TQC4qd+tBhvJiyuuwvt+an5n9Axlo93NJweTTVDgR3FLsJL2FGV0FWa9o0 +aiUYdlL1fkAPOvVT97OLjmM/9RBirSUGR+Dwr6O3tRaB1bDmpKUqTz190Lhd3Ee/ +zWOJa6ubhUjls1KY4Rm/f28VheNRp6PQn+QdIdht/1oEqlssv/yPeVe1U6tW2oDW +ksbE0O1JvbUhKKhTD3cFY6x5A7iRflJpSEEoPqbHu8X237gEtgqJcKI= +-----END ENCRYPTED PRIVATE KEY----- diff --git a/test/parallel/test-crypto-rsa-dsa.js b/test/parallel/test-crypto-rsa-dsa.js index bda98652cf59..dd38ecd4d68d 100644 --- a/test/parallel/test-crypto-rsa-dsa.js +++ b/test/parallel/test-crypto-rsa-dsa.js @@ -26,42 +26,19 @@ const keyPem = fixtures.readKey('rsa_private.pem'); const rsaKeySize = 2048; const rsaPubPem = fixtures.readKey('rsa_public.pem', 'ascii'); const rsaKeyPem = fixtures.readKey('rsa_private.pem', 'ascii'); -const rsaKeyPemEncryptedLegacy = fixtures.readKey( +// Fixed ciphertexts keep wrong passwords from occasionally producing valid +// padding and a decoder error instead of the expected bad decrypt. +const rsaKeyPemEncrypted = fixtures.readKey( 'rsa_private_encrypted.pem', 'ascii'); -const rsaKeyPemEncrypted = fips3 ? - crypto.createPrivateKey(rsaKeyPem).export({ - type: 'pkcs8', - format: 'pem', - cipher: 'aes-256-cbc', - passphrase: 'password', - }) : rsaKeyPemEncryptedLegacy; const dsaPubPem = fixtures.readKey('dsa_public.pem', 'ascii'); const dsaKeyPem = fixtures.readKey('dsa_private.pem', 'ascii'); -const dsaKeyPemEncryptedLegacy = fixtures.readKey( +const dsaKeyPemEncrypted = fixtures.readKey( 'dsa_private_encrypted.pem', 'ascii'); -const dsaKeyPemEncrypted = fips3 ? - crypto.createPrivateKey(dsaKeyPem).export({ - type: 'pkcs8', - format: 'pem', - cipher: 'aes-256-cbc', - passphrase: 'password', - }) : dsaKeyPemEncryptedLegacy; const rsaPkcs8KeyPem = fixtures.readKey('rsa_private_pkcs8.pem'); const dsaPkcs8KeyPem = fixtures.readKey('dsa_private_pkcs8.pem'); const ec = new TextEncoder(); -if (fips3) { - for (const key of [rsaKeyPemEncryptedLegacy, dsaKeyPemEncryptedLegacy]) { - assert.throws(() => crypto.createPrivateKey({ - key, - passphrase: 'password', - }), { - code: 'ERR_OSSL_EVP_UNSUPPORTED', - }); - } -} - const openssl1DecryptError = { message: 'error:06065064:digital envelope routines:EVP_DecryptFinal_ex:' + 'bad decrypt', diff --git a/test/parallel/test-tls-passphrase.js b/test/parallel/test-tls-passphrase.js index 1fe2c1ec11cf..6e1b50c0bb10 100644 --- a/test/parallel/test-tls-passphrase.js +++ b/test/parallel/test-tls-passphrase.js @@ -38,33 +38,6 @@ assert(Buffer.isBuffer(cert)); assert.strictEqual(typeof passKey.toString(), 'string'); assert.strictEqual(typeof cert.toString(), 'string'); -if (hasFIPS(3)) { - const encryptedKeyOptions = { - key: passKey, - passphrase: 'password', - cert, - }; - assert.throws(() => tls.Server(encryptedKeyOptions), { - code: 'ERR_OSSL_EVP_UNSUPPORTED', - }); - assert.throws(() => tls.connect(encryptedKeyOptions), { - code: 'ERR_OSSL_EVP_UNSUPPORTED', - }); - - const server = tls.Server({ key: rawKey, passphrase: 'ignored', cert }); - server.listen(0, common.mustCall(function() { - const client = tls.connect({ - port: this.address().port, - key: rawKey, - passphrase: 'ignored', - cert, - rejectUnauthorized: false, - }, common.mustCall(() => client.end())); - client.on('close', common.mustCall(() => server.close())); - })); - return; -} - function onSecureConnect() { return common.mustCall(function() { this.end(); }); } @@ -252,6 +225,10 @@ server.listen(0, common.mustCall(function() { })).unref(); const errMessageDecrypt = /bad[ _]decrypt/i; +// TLS supplies an empty password when the passphrase is omitted. OpenSSL 4 +// FIPS rejects it during PBKDF2 password-length checks, before decryption. +const missingPassphraseError = hasFIPS(4) ? + { code: 'ERR_OSSL_PASSWORD_STRENGTH_TOO_WEAK' } : errMessageDecrypt; // Missing passphrase assert.throws(function() { @@ -261,7 +238,7 @@ assert.throws(function() { cert: cert, rejectUnauthorized: false }); -}, errMessageDecrypt); +}, missingPassphraseError); assert.throws(function() { tls.connect({ @@ -270,7 +247,7 @@ assert.throws(function() { cert: cert, rejectUnauthorized: false }); -}, errMessageDecrypt); +}, missingPassphraseError); assert.throws(function() { tls.connect({ @@ -279,14 +256,14 @@ assert.throws(function() { cert: cert, rejectUnauthorized: false }); -}, errMessageDecrypt); +}, missingPassphraseError); // Invalid passphrase assert.throws(function() { tls.connect({ port: server.address().port, key: passKey, - passphrase: 'invalid', + passphrase: 'wrong-password', cert: cert, rejectUnauthorized: false }); @@ -296,7 +273,7 @@ assert.throws(function() { tls.connect({ port: server.address().port, key: [passKey], - passphrase: 'invalid', + passphrase: 'wrong-password', cert: cert, rejectUnauthorized: false }); @@ -306,7 +283,7 @@ assert.throws(function() { tls.connect({ port: server.address().port, key: [{ pem: passKey }], - passphrase: 'invalid', + passphrase: 'wrong-password', cert: cert, rejectUnauthorized: false }); @@ -315,7 +292,7 @@ assert.throws(function() { assert.throws(function() { tls.connect({ port: server.address().port, - key: [{ pem: passKey, passphrase: 'invalid' }], + key: [{ pem: passKey, passphrase: 'wrong-password' }], passphrase: 'password', // Valid but unused cert: cert, rejectUnauthorized: false From 549694349de8bf8ee82cdf40dd43eea7b2d6e4d4 Mon Sep 17 00:00:00 2001 From: Rafael Gonzaga Date: Mon, 14 Sep 2026 14:32:13 -0300 Subject: [PATCH 117/217] doc: clarify permission model scope for output paths Flags such --trace-event-file or any other flag that specifies a directory are subject to permission model rules, but a "bypass" isn't considered a vulnerability while it doesn't pose a risk to the user application Signed-off-by: RafaelGSS PR-URL: https://github.com/nodejs/node/pull/66004 Reviewed-By: Filip Skokan Reviewed-By: Luigi Pinca --- SECURITY.md | 10 +++++++++- doc/api/permissions.md | 8 ++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index 17d35fd4432f..f5a4ef3370ad 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -522,7 +522,15 @@ The following are **not** vulnerabilities in Node.js: * **Operator-controlled flags**: Behavior unlocked by flags the operator explicitly passes (e.g., `--localstorage-file`) is the operator's responsibility. The permission model does not restrict how Node.js behaves - when the operator intentionally configures it. + when the operator intentionally configures it. This includes any file or + resource that Node.js itself creates, writes, or reads at a location the + operator selected through a flag, including every path derived from a + template or pattern in that flag. For example, trace files rotated by + `--trace-event-file-pattern` (`${rotation}`) being written without a + matching `--allow-fs-write` entry is not a permission model bypass. Such + paths are part of the operator's configuration, not application file-system + access. Inconsistent checks on these paths are treated as regular bugs and + should be reported through the public issue tracker. * **`node:sqlite` and the permission model**: `DatabaseSync` operates with the same file-system privileges as the process. Using SQL pragmas or built-in diff --git a/doc/api/permissions.md b/doc/api/permissions.md index b677013d7978..dc5e3e6111ca 100644 --- a/doc/api/permissions.md +++ b/doc/api/permissions.md @@ -354,6 +354,14 @@ There are constraints you need to know before using this system: to read files before environment initialization. As a result, such flags are not subject to the rules of the Permission Model. The same applies for V8 flags that can be set via runtime through `v8.setFlagsFromString`. +* Files that Node.js itself creates, writes, or reads at a location selected + by an operator flag may not be consistently checked against the Permission + Model, in particular when the flag accepts a template or pattern that + expands to several paths. For example, trace files rotated by + `--trace-event-file-pattern` (`${rotation}`) can be written even when the + expanded path is not covered by `--allow-fs-write`. Because the location is + chosen by the operator, gaps like this are treated as regular bugs rather + than vulnerabilities. Please report them through the regular issue tracker. * OpenSSL engines cannot be requested at runtime when the Permission Model is enabled, affecting the built-in crypto, https, and tls modules. * Run-Time Loadable Extensions cannot be loaded when the Permission Model is From 32401c222943284ef2318e37af3c8d9f7e21cdae Mon Sep 17 00:00:00 2001 From: Donghoon Kang Date: Tue, 15 Sep 2026 04:45:00 +0900 Subject: [PATCH 118/217] perf_hooks: reuse buffer for uv metrics Reuse an aliased Int32Array to transfer uv metrics from C++ to JavaScript instead of allocating a new V8 array on every access. Assisted-by: Codex Signed-off-by: HoonDongKang PR-URL: https://github.com/nodejs/node/pull/65985 Reviewed-By: James M Snell Reviewed-By: Daeyeon Jeong Reviewed-By: Chengzhong Wu --- lib/internal/perf/nodetiming.js | 9 ++--- src/node_perf.cc | 35 +++++++++++++------ src/node_perf_common.h | 3 ++ src/node_snapshotable.cc | 3 ++ .../fixtures/test-nodetiming-uvmetricsinfo.js | 10 +++++- typings/internalBinding/performance.d.ts | 3 +- 6 files changed, 46 insertions(+), 17 deletions(-) diff --git a/lib/internal/perf/nodetiming.js b/lib/internal/perf/nodetiming.js index a9e0c3f252ce..5de5e3e6644f 100644 --- a/lib/internal/perf/nodetiming.js +++ b/lib/internal/perf/nodetiming.js @@ -29,6 +29,7 @@ const { }, loopIdleTime, uvMetricsInfo, + uvMetricsBuffer, } = internalBinding('performance'); class PerformanceNodeTiming { @@ -129,11 +130,11 @@ class PerformanceNodeTiming { enumerable: true, configurable: true, get: () => { - const metrics = uvMetricsInfo(); + uvMetricsInfo(); return { - loopCount: metrics[0], - events: metrics[1], - eventsWaiting: metrics[2], + loopCount: uvMetricsBuffer[0], + events: uvMetricsBuffer[1], + eventsWaiting: uvMetricsBuffer[2], }; }, }, diff --git a/src/node_perf.cc b/src/node_perf.cc index 177c2a789854..75a62b89a534 100644 --- a/src/node_perf.cc +++ b/src/node_perf.cc @@ -14,7 +14,6 @@ namespace node { namespace performance { -using v8::Array; using v8::Context; using v8::DontDelete; using v8::Function; @@ -57,7 +56,12 @@ PerformanceState::PerformanceState(Isolate* isolate, offsetof(performance_state_internal, observers), NODE_PERFORMANCE_ENTRY_TYPE_INVALID, root, - MAYBE_FIELD_PTR(info, observers)) { + MAYBE_FIELD_PTR(info, observers)), + uv_metrics(isolate, + offsetof(performance_state_internal, uv_metrics), + 3, + root, + MAYBE_FIELD_PTR(info, uv_metrics)) { if (info == nullptr) { // For performance states initialized from scratch, reset // all the milestones and initialize the time origin. @@ -81,9 +85,15 @@ PerformanceState::SerializeInfo PerformanceState::Serialize( // We'll re-initialize them after deserialization. ResetMilestones(); + // Do not retain runtime metrics in the snapshot. + for (size_t i = 0; i < uv_metrics.Length(); ++i) { + uv_metrics[i] = 0; + } + SerializeInfo info{root.Serialize(context, creator), milestones.Serialize(context, creator), - observers.Serialize(context, creator)}; + observers.Serialize(context, creator), + uv_metrics.Serialize(context, creator)}; return info; } @@ -105,6 +115,7 @@ void PerformanceState::Deserialize(v8::Local context, root.Deserialize(context); milestones.Deserialize(context); observers.Deserialize(context); + uv_metrics.Deserialize(context); // Re-initialize the time origin and timestamp i.e. the process start time. Initialize(time_origin, time_origin_timestamp); @@ -116,6 +127,7 @@ std::ostream& operator<<(std::ostream& o, << " " << i.root << ", // root\n" << " " << i.milestones << ", // milestones\n" << " " << i.observers << ", // observers\n" + << " " << i.uv_metrics << ", // uv_metrics\n" << "}"; return o; } @@ -265,17 +277,13 @@ void LoopIdleTime(const FunctionCallbackInfo& args) { void UvMetricsInfo(const FunctionCallbackInfo& args) { Environment* env = Environment::GetCurrent(args); - Isolate* isolate = env->isolate(); uv_metrics_t metrics; // uv_metrics_info always return 0 CHECK_EQ(uv_metrics_info(env->event_loop(), &metrics), 0); - Local data[] = { - Integer::New(isolate, metrics.loop_count), - Integer::New(isolate, metrics.events), - Integer::New(isolate, metrics.events_waiting), - }; - Local arr = Array::New(env->isolate(), data, arraysize(data)); - args.GetReturnValue().Set(arr); + AliasedInt32Array& buffer = env->performance_state()->uv_metrics; + buffer[0] = static_cast(metrics.loop_count); + buffer[1] = static_cast(metrics.events); + buffer[2] = static_cast(metrics.events_waiting); } void CreateELDHistogram(const FunctionCallbackInfo& args) { @@ -366,6 +374,11 @@ void CreatePerContextProperties(Local target, target->Set(context, FIXED_ONE_BYTE_STRING(isolate, "milestones"), state->milestones.GetJSArray()).Check(); + target + ->Set(context, + FIXED_ONE_BYTE_STRING(isolate, "uvMetricsBuffer"), + state->uv_metrics.GetJSArray()) + .Check(); Local constants = Object::New(isolate); diff --git a/src/node_perf_common.h b/src/node_perf_common.h index 01e7f35241ac..aa84ba55b08e 100644 --- a/src/node_perf_common.h +++ b/src/node_perf_common.h @@ -62,6 +62,7 @@ class PerformanceState { AliasedBufferIndex root; AliasedBufferIndex milestones; AliasedBufferIndex observers; + AliasedBufferIndex uv_metrics; }; explicit PerformanceState(v8::Isolate* isolate, @@ -78,6 +79,7 @@ class PerformanceState { AliasedUint8Array root; AliasedFloat64Array milestones; AliasedUint32Array observers; + AliasedInt32Array uv_metrics; uint64_t performance_last_gc_start_mark = 0; uint16_t current_gc_type = 0; @@ -92,6 +94,7 @@ class PerformanceState { // doubles first so that they are always sizeof(double)-aligned double milestones[NODE_PERFORMANCE_MILESTONE_INVALID]; uint32_t observers[NODE_PERFORMANCE_ENTRY_TYPE_INVALID]; + int32_t uv_metrics[3]; }; }; diff --git a/src/node_snapshotable.cc b/src/node_snapshotable.cc index 2624b73178b1..050a14f0678f 100644 --- a/src/node_snapshotable.cc +++ b/src/node_snapshotable.cc @@ -406,6 +406,7 @@ size_t SnapshotSerializer::Write(const ImmediateInfo::SerializeInfo& data) { // [ 4/8 bytes ] snapshot index of root // [ 4/8 bytes ] snapshot index of milestones // [ 4/8 bytes ] snapshot index of observers +// [ 4/8 bytes ] snapshot index of uv_metrics template <> performance::PerformanceState::SerializeInfo SnapshotDeserializer::Read() { Debug("Read()\n"); @@ -414,6 +415,7 @@ performance::PerformanceState::SerializeInfo SnapshotDeserializer::Read() { result.root = ReadArithmetic(); result.milestones = ReadArithmetic(); result.observers = ReadArithmetic(); + result.uv_metrics = ReadArithmetic(); if (is_debug) { std::string str = ToStr(result); Debug("Read() %s\n", str); @@ -432,6 +434,7 @@ size_t SnapshotSerializer::Write( size_t written_total = WriteArithmetic(data.root); written_total += WriteArithmetic(data.milestones); written_total += WriteArithmetic(data.observers); + written_total += WriteArithmetic(data.uv_metrics); Debug("Write() wrote %d bytes\n", written_total); diff --git a/test/fixtures/test-nodetiming-uvmetricsinfo.js b/test/fixtures/test-nodetiming-uvmetricsinfo.js index 59b1cc8ebf11..038ca8b79904 100644 --- a/test/fixtures/test-nodetiming-uvmetricsinfo.js +++ b/test/fixtures/test-nodetiming-uvmetricsinfo.js @@ -40,7 +40,15 @@ function safeMetricsInfo(cb) { fs.open(__filename, 'r', (err) => { assert.ifError(err); }); + + const saved = { ...info }; + safeMetricsInfo((nextInfo) => { + assert.notStrictEqual(nextInfo, info); + assert.ok(nextInfo.loopCount > saved.loopCount); + // Updating the shared buffer must not change earlier results. + assert.deepStrictEqual(info, saved); + }); } safeMetricsInfo(openFile); -} \ No newline at end of file +} diff --git a/typings/internalBinding/performance.d.ts b/typings/internalBinding/performance.d.ts index fa9a3810fc7a..dc4d1e20c6b3 100644 --- a/typings/internalBinding/performance.d.ts +++ b/typings/internalBinding/performance.d.ts @@ -129,6 +129,7 @@ export interface PerformanceBinding { samplePerIteration: boolean, ): InternalPerformanceBinding.ELDHistogram; markBootstrapComplete(): void; - uvMetricsInfo(): [number, number, number]; + uvMetricsInfo(): void; + uvMetricsBuffer: Int32Array; now(): number; } From d4cb9166248a17363ebc4c43171b4849f88fc88d Mon Sep 17 00:00:00 2001 From: SRAVANI GUNDEPALLI <88590399+sravani1510@users.noreply.github.com> Date: Tue, 15 Sep 2026 10:56:45 +0530 Subject: [PATCH 119/217] test: skip test-vfs-real-provider-watch.js on IBM i Signed-off-by: Gundepalli Sravani PR-URL: https://github.com/nodejs/node/pull/65987 Refs: https://github.com/nodejs/node/issues/52640 Reviewed-By: Richard Lau Reviewed-By: Luigi Pinca --- benchmark/fs/bench-watch-recursive.js | 5 +++++ test/parallel/test-vfs-real-provider-watch.js | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/benchmark/fs/bench-watch-recursive.js b/benchmark/fs/bench-watch-recursive.js index 45ed0871718e..9fdb81d83043 100644 --- a/benchmark/fs/bench-watch-recursive.js +++ b/benchmark/fs/bench-watch-recursive.js @@ -4,6 +4,11 @@ // On Linux and other platforms without a native recursive watcher this is // implemented in JavaScript on top of per-directory watchers. +if (process.platform === 'os400') { + console.log('Skipping: IBMi does not support `fs.watch()`'); + process.exit(0); +} + const common = require('../common'); const fs = require('fs'); const path = require('path'); diff --git a/test/parallel/test-vfs-real-provider-watch.js b/test/parallel/test-vfs-real-provider-watch.js index f4218fa408ee..056744c5fd0c 100644 --- a/test/parallel/test-vfs-real-provider-watch.js +++ b/test/parallel/test-vfs-real-provider-watch.js @@ -4,6 +4,10 @@ // watch / promises.watch / watchFile through RealFSProvider. const common = require('../common'); + +if (common.isIBMi) + common.skip('IBMi does not support `fs.watch()`'); + const tmpdir = require('../common/tmpdir'); const assert = require('assert'); const fs = require('fs'); From 287ece3f22aff0ab0304144be20cecd57d00f2df Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Tue, 15 Sep 2026 10:03:38 +0200 Subject: [PATCH 120/217] tools: pass author to commit message validator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filip Skokan PR-URL: https://github.com/nodejs/node/pull/66012 Reviewed-By: Colin Ihrig Reviewed-By: Antoine du Hamel Reviewed-By: Gürgün Dayıoğlu Reviewed-By: Luigi Pinca Reviewed-By: James M Snell Reviewed-By: Xuguang Mei --- .github/workflows/commit-lint.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/commit-lint.yml b/.github/workflows/commit-lint.yml index 037ea7e810e0..6b173d8bc978 100644 --- a/.github/workflows/commit-lint.yml +++ b/.github/workflows/commit-lint.yml @@ -41,7 +41,11 @@ jobs: '--no-validate-metadata', '--tap', '-', ], { cwd: process.env.RUNNER_TEMP, - input: Buffer.from(JSON.stringify([{ id: commit.sha, message: commit.commit.message }])), + input: Buffer.from(JSON.stringify([{ + id: commit.sha, + message: commit.commit.message, + author: commit.commit.author, + }])), silent: true, ignoreReturnCode: true, }); From 64fb33d791bf47631ad13868c47946cfa906dc1d Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Tue, 15 Sep 2026 10:33:52 +0200 Subject: [PATCH 121/217] ffi: load libraries from a mounted VFS The operating system's dynamic loader cannot open a library that lives in a mounted virtual file system: the reserved mount path has no real inode. Native addons already handle this in require(): the loader hands their bytes to process.dlopen(), which loads them from a private, self-cleaning image - an anonymous in-memory memfd on Linux. Make ffi.dlopen() and new DynamicLibrary() do the same transparently. Mirroring the fs handler integration, the VFS hook installer sets a library reader into node:ffi while at least one VFS is mounted and clears it when the last one unmounts; DynamicLibrary consults it before every load, so the dependency points from the VFS into ffi and ffi never loads any VFS code. The reader hands the library's bytes to the native constructor, which loads them from the same kind of image, released right after the load, while library.path keeps reporting the virtual path. Libraries on the real file system are unaffected and load directly, and pay only a null check while no VFS is mounted. The AddonImage materializer moves from an anonymous namespace in node_binding.cc to node_binding.h so that node_ffi.cc can reuse it. On Windows the image is now written and closed before the load, because the loader shares read alone and a retained writable delete-on-close handle failed the load with ERROR_SHARING_VIOLATION; since a mapped image cannot be unlinked there, it is kept with the module it loaded as and both are released at process exit. On POSIX the image still never outlives the constructor call, so nothing is left for dlclose() to clean up. Also fix the VFS dlopen hook forwarding a missing flags argument as undefined, which process.dlopen() coerces to 0 - not a valid dlopen(2) mode - so loading any addon from the real file system failed with EINVAL while a VFS was mounted. Co-authored-by: Philipp Dunkel Signed-off-by: Matteo Collina PR-URL: https://github.com/nodejs/node/pull/65909 Reviewed-By: Paolo Insogna Reviewed-By: James M Snell --- doc/api/ffi.md | 25 +++ doc/api/vfs.md | 8 + lib/ffi.js | 34 +++- lib/internal/ffi/vfs.js | 27 +++ lib/internal/vfs/setup.js | 32 ++++ src/node_binding.cc | 177 +++++++++--------- src/node_binding.h | 56 ++++++ src/node_ffi.cc | 38 +++- test/ffi/test-ffi-vfs.js | 94 ++++++++++ .../test-dlopen-binary-image-cleanup.js | 90 +++++++++ test/parallel/test-vfs-addon.js | 8 + 11 files changed, 501 insertions(+), 88 deletions(-) create mode 100644 lib/internal/ffi/vfs.js create mode 100644 test/ffi/test-ffi-vfs.js create mode 100644 test/parallel/test-dlopen-binary-image-cleanup.js diff --git a/doc/api/ffi.md b/doc/api/ffi.md index ba7f866cebff..e3ecdf93626e 100644 --- a/doc/api/ffi.md +++ b/doc/api/ffi.md @@ -210,6 +210,11 @@ const path = `libsqlite3.${suffix}`; * `path` {string|null} Path to a dynamic library, or `null` to resolve symbols @@ -221,6 +226,13 @@ Loads a dynamic library and resolves the requested function definitions. On Windows passing `null` is not supported. +A `path` inside a mounted [virtual file system][] is supported: the +operating system's dynamic loader cannot open a virtual path, so the +library's bytes are read from the VFS and loaded from a private, +self-cleaning temporary image instead, while `lib.path` keeps reporting +the virtual path. Libraries on the real file system are unaffected and +load directly. + When `definitions` is omitted, `functions` is returned as an empty object until symbols are resolved explicitly. @@ -302,6 +314,14 @@ Represents a loaded dynamic library. ### `new DynamicLibrary(path)` + + * `path` {string|null} Path to a dynamic library, or `null` to resolve symbols from the current process image. @@ -309,6 +329,9 @@ Loads the dynamic library without resolving any functions eagerly. On Windows passing `null` is not supported. +A `path` inside a mounted [virtual file system][] loads the same way as +with [`ffi.dlopen()`][]. + ```cjs const { DynamicLibrary, suffix } = require('node:ffi'); @@ -798,7 +821,9 @@ and keep callback and pointer lifetimes explicit on the native side. [Permission Model]: permissions.md#permission-model [`--allow-ffi`]: cli.md#--allow-ffi +[`ffi.dlopen()`]: #ffidlopenpath-definitions [`ffi.toBuffer(pointer, length, copy)`]: #ffitobufferpointer-length-copy [`library.functions`]: #libraryfunctions [`using`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/using [type names]: #type-names +[virtual file system]: vfs.md diff --git a/doc/api/vfs.md b/doc/api/vfs.md index 9be346439af5..8172a7cf7e8d 100644 --- a/doc/api/vfs.md +++ b/doc/api/vfs.md @@ -425,6 +425,12 @@ addon's bytes are read from the VFS and loaded from a private, self-cleaning temporary image instead. Addons on the real file system are unaffected and load directly. +Shared libraries opened through [`ffi.dlopen()`][] (or +[`new ffi.DynamicLibrary()`][]) work the same way: a library path inside a +mounted VFS is detected, its bytes are read from the VFS, and the library is +loaded from a private, self-cleaning image while `library.path` keeps +reporting the virtual path. Libraries on the real file system load directly. + ## Use with Single Executable Applications When running as a [Single Executable Application][] built with @@ -634,9 +640,11 @@ fields use synthetic but stable values: [`VirtualFileSystem`]: #class-virtualfilesystem [`VirtualProvider`]: #class-virtualprovider [`ZipProvider`]: #class-zipprovider +[`ffi.dlopen()`]: ffi.md#ffidlopenpath-definitions [`fs.BigIntStats`]: fs.md#class-fsstats [`fs.Stats`]: fs.md#class-fsstats [`import.meta.resolve()`]: esm.md#importmetaresolvespecifier +[`new ffi.DynamicLibrary()`]: ffi.md#new-dynamiclibrarypath [`node:fs`]: fs.md [`require()`]: modules.md#requireid [`require.resolve()`]: modules.md#requireresolverequest-options diff --git a/lib/ffi.js b/lib/ffi.js index cbd188793200..e2ef9efbe552 100644 --- a/lib/ffi.js +++ b/lib/ffi.js @@ -9,6 +9,7 @@ const { ObjectGetOwnPropertyDescriptor, ObjectKeys, ObjectPrototypeToString, + ReflectConstruct, SafeWeakMap, SafeWeakRef, SymbolDispose, @@ -38,7 +39,7 @@ const { emitExperimentalWarning('FFI'); const { - DynamicLibrary, + DynamicLibrary: NativeDynamicLibrary, getInt8, getUint8, getInt16, @@ -119,6 +120,37 @@ function wrapFFIFunction(rawFn, owner) { return wrapped; } +const { getVfsLibraryReader } = require('internal/ffi/vfs'); + +// A thin constructor in front of the native class so that a library inside +// a mounted virtual file system loads transparently: its bytes are read +// from the VFS and handed to the native constructor, which loads them from +// a private, self-cleaning image - the same way require() handles a native +// addon in a VFS. The reader is installed by the VFS while it is mounted +// (see internal/ffi/vfs), so no VFS code is ever loaded from here. The +// wrapper shares the native prototype, so instances and instanceof behave +// as if the native class were exposed directly. +function DynamicLibrary(path) { + if (new.target === undefined) { + // Let the native constructor produce its usual error. + return FunctionPrototypeCall(NativeDynamicLibrary, this, path); + } + const readVirtualLibrary = getVfsLibraryReader(); + const binary = + readVirtualLibrary === null || typeof path !== 'string' ? + undefined : readVirtualLibrary(path); + return ReflectConstruct(NativeDynamicLibrary, + binary === undefined ? [path] : [path, binary], + new.target); +} +DynamicLibrary.prototype = NativeDynamicLibrary.prototype; +ObjectDefineProperty(DynamicLibrary.prototype, 'constructor', { + __proto__: null, + configurable: true, + value: DynamicLibrary, + writable: true, +}); + const rawGetFunction = DynamicLibrary.prototype.getFunction; const rawGetFunctions = DynamicLibrary.prototype.getFunctions; const rawClose = DynamicLibrary.prototype.close; diff --git a/lib/internal/ffi/vfs.js b/lib/internal/ffi/vfs.js new file mode 100644 index 000000000000..eb6342ec0452 --- /dev/null +++ b/lib/internal/ffi/vfs.js @@ -0,0 +1,27 @@ +'use strict'; + +// Seam between node:ffi and the virtual file system, mirroring the fs +// handler integration in internal/fs/utils: the VFS hook installer sets a +// library reader while at least one VFS is mounted and clears it when the +// last one unmounts, and DynamicLibrary consults it before every load. The +// dependency points from the VFS into ffi: ffi never loads any VFS code, +// and pays only a null check while no VFS is mounted. + +// When reader is null, no VFS is active (zero overhead). Otherwise it is +// (path) => Buffer|undefined: the library's bytes for a path inside a +// mounted VFS, or undefined for a path the dynamic loader should open +// itself. +let vfsLibraryReader = null; + +function setVfsLibraryReader(reader) { + vfsLibraryReader = reader; +} + +function getVfsLibraryReader() { + return vfsLibraryReader; +} + +module.exports = { + getVfsLibraryReader, + setVfsLibraryReader, +}; diff --git a/lib/internal/vfs/setup.js b/lib/internal/vfs/setup.js index 3e183e422831..3d624d5af293 100644 --- a/lib/internal/vfs/setup.js +++ b/lib/internal/vfs/setup.js @@ -968,10 +968,38 @@ function installAddonLoader() { const { dlopenBinary } = internalBinding('process_methods'); return dlopenBinary(module, filename, flags, readFileSync(filename)); } + // Do not forward a missing flags argument as `undefined`: + // process.dlopen() coerces it to 0, which is not a valid dlopen(2) + // mode, instead of applying the default flags. + if (flags === undefined) return originalDlopen(module, filename); return originalDlopen(module, filename, flags); }; } +/** + * Reads the bytes of a file that lives in a mounted VFS. Returns undefined + * for a path outside the reserved VFS root - the caller should open the + * path itself - and throws ENOENT for a path under the root that no + * mounted VFS serves, since no real file can exist there. Installed into + * node:ffi while hooks are installed, so DynamicLibrary can load a + * VFS-resident library from a private image, the same way the module + * loader handles a native addon in a VFS. + * @param {string} pathStr The path of the library + * @returns {Buffer|undefined} The library's bytes, or undefined + */ +function readVirtualBinary(pathStr) { + const normalized = normalizeMountedPath(pathStr); + if (!StringPrototypeStartsWith(normalized, normalizedVfsRootPrefix)) { + return undefined; + } + const layerId = getLayerIdFromPath(normalized); + const vfs = layerId === -1 ? undefined : activeVFSLayers.get(layerId); + if (vfs === undefined || !vfs.shouldHandleNormalized(normalized)) { + throw createENOENT('open', pathStr); + } + return vfs.readFileSync(normalized); +} + /** * Install all VFS hooks: module loader overrides and fs handlers. */ @@ -981,6 +1009,8 @@ function installHooks() { normalizedVfsRootPrefix = getNormalizedVfsRoot() + sep; installModuleLoaderOverrides(); installAddonLoader(); + const { setVfsLibraryReader } = require('internal/ffi/vfs'); + setVfsLibraryReader(readVirtualBinary); vfsHandlerObj = createVfsHandlers(); setVfsHandlers(vfsHandlerObj); hooksInstalled = true; @@ -998,6 +1028,8 @@ function uninstallHooks() { setLoaderOverrides(); setVfsHandlers(null); vfsHandlerObj = undefined; + const { setVfsLibraryReader } = require('internal/ffi/vfs'); + setVfsLibraryReader(null); process.dlopen = originalDlopen; hooksInstalled = false; } diff --git a/src/node_binding.cc b/src/node_binding.cc index 0437f59ca60e..52d388434873 100644 --- a/src/node_binding.cc +++ b/src/node_binding.cc @@ -8,7 +8,9 @@ #include "permission/permission.h" #include "util.h" +#include #include +#include #include #ifdef _WIN32 @@ -17,7 +19,6 @@ #include #include #include -#include #if defined(__linux__) #include #include @@ -474,64 +475,60 @@ int NodeMemfdCreate(const char* name, unsigned int flags) { return static_cast(syscall(SYS_memfd_create, name, flags)); } #endif // __linux__ +#else // _WIN32 + +// Windows refuses to unlink a file that backs a mapped image section: neither +// delete-on-close, nor DeleteFile(), nor a POSIX-semantics disposition can +// remove it while the DLL is loaded. A materialized image therefore has to +// outlive its load, and the only moment it can go is once the module is +// unloaded again. Node keeps addons loaded for the life of the process, so +// that moment is process exit: each image is kept here with the module it was +// loaded as, and released together at exit. +struct RetainedAddonImage { + HMODULE module; + std::wstring path; +}; +Mutex g_retained_addon_images_mutex; +std::vector* g_retained_addon_images = nullptr; + +// Unloads the images this process materialized -- and only those; addons loaded +// from a real path are left alone -- so that each file can finally be deleted. +// This has to happen after everything that might still call into an addon, so +// it is registered during static initialisation below: atexit() runs handlers +// last-registered-first, so registering before main() puts this behind every +// handler that is registered while running. +void ReleaseRetainedAddonImages() { + Mutex::ScopedLock lock(g_retained_addon_images_mutex); + if (g_retained_addon_images == nullptr) return; + for (auto it = g_retained_addon_images->rbegin(); + it != g_retained_addon_images->rend(); + ++it) { + // Deleting first doubles as the test for whether the image is still + // mapped, because that is the only thing that can stop it: an FFI library + // the caller already close()d is gone by now, and unloading it a second + // time through a stale module handle would be wrong. + if (DeleteFileW(it->path.c_str())) continue; + if (it->module != nullptr) FreeLibrary(it->module); + DeleteFileW(it->path.c_str()); + } + g_retained_addon_images->clear(); +} + +// Arms the hook before main() rather than at the first load; see above. +const struct RetainedAddonImageExitHook { + RetainedAddonImageExitHook() { atexit(ReleaseRetainedAddonImages); } +} g_retained_addon_image_exit_hook; + #endif // !_WIN32 -// Materializes native-addon bytes into a form dlopen()/LoadLibrary() can load, -// with the smallest, most private on-disk footprint each platform allows: -// Linux: an anonymous in-memory memfd, loaded via /proc/self/fd/N - -// the bytes never touch the filesystem. -// other POSIX: a 0700 mkdtemp() directory plus an O_EXCL|O_NOFOLLOW file, -// unlink()ed right after the load (the mapping keeps it alive). -// Windows: a temp file opened FILE_FLAG_DELETE_ON_CLOSE; its handle is -// retained for the process lifetime so the file is removed -// automatically once the process (and the loaded DLL) exit. -// Used for an addon that lives somewhere dlopen() cannot open by path, such as -// a virtual file system. -class AddonImage { - public: - AddonImage() = default; - ~AddonImage(); - AddonImage(const AddonImage&) = delete; - AddonImage& operator=(const AddonImage&) = delete; - - // The directory a temporary image would be written to, with a trailing - // separator; empty when it cannot be determined. Names the resource for the - // file-system permission check. - static std::string TempDir(); - - // On success sets path() to a real, loadable path for `data`. - bool Materialize(const char* data, size_t len); - const std::string& path() const { return path_; } - const std::string& errmsg() const { return errmsg_; } - - // Call exactly once, right after DLib::Open(); `opened` says whether the load - // succeeded. Releases the transient resources that are no longer needed (a - // successful load holds its own mapping): on POSIX closes the memfd or - // unlinks the temp file; on Windows retains the delete-on-close handle for - // the process lifetime when opened, or closes it (deleting the file) on - // failure. - void AfterOpen(bool opened); +} // namespace - private: - std::string path_; - std::string errmsg_; - bool consumed_ = false; -#ifdef _WIN32 - HANDLE handle_ = INVALID_HANDLE_VALUE; -#else - bool MaterializeTempFile(const char* data, size_t len); - int fd_ = -1; - std::string temp_dir_; // non-empty only for the temp-file (non-memfd) path -#endif -}; +// AddonImage is declared in node_binding.h so that the other loader of +// dynamically shared objects, node_ffi.cc, can reuse it; see the header for +// the platform-by-platform description. #ifdef _WIN32 -// Delete-on-close handles kept alive until process exit so their temp files -// outlive the loaded DLLs and are removed once the process ends. -Mutex g_retained_addon_handles_mutex; -std::vector* g_retained_addon_handles = nullptr; - // static std::string AddonImage::TempDir() { wchar_t dir[MAX_PATH + 1]; @@ -558,18 +555,22 @@ bool AddonImage::Materialize(const char* data, size_t len) { errmsg_ = "could not create a temporary file name"; return false; } - // Reopen the just-created file delete-on-close, sharing delete so the loader - // can map it while it is delete-pending; the file is removed when this handle - // and the loader's section are both released (i.e. at process exit). - handle_ = CreateFileW(file, - GENERIC_READ | GENERIC_WRITE, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - nullptr, - CREATE_ALWAYS, - FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE, - nullptr); - if (handle_ == INVALID_HANDLE_VALUE) { + // Write the image and close it again: nothing may still hold the file open + // when the loader gets to it. Sharing is checked in both directions, and the + // loader opens a DLL for read and execute while sharing read alone, so any + // handle of ours holding write access fails the load with + // ERROR_SHARING_VIOLATION however permissive this side's share mode is. + HANDLE writer = + CreateFileW(file, + GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + CREATE_ALWAYS, + FILE_ATTRIBUTE_TEMPORARY, + nullptr); + if (writer == INVALID_HANDLE_VALUE) { errmsg_ = "could not create a temporary file for the native addon"; + DeleteFileW(file); return false; } size_t off = 0; @@ -577,48 +578,52 @@ bool AddonImage::Materialize(const char* data, size_t len) { DWORD chunk = len - off > MAXDWORD ? MAXDWORD : static_cast(len - off); DWORD written = 0; - if (!WriteFile(handle_, data + off, chunk, &written, nullptr)) { + if (!WriteFile(writer, data + off, chunk, &written, nullptr)) { errmsg_ = "could not write the native addon to a temporary file"; - CloseHandle(handle_); - handle_ = INVALID_HANDLE_VALUE; + CloseHandle(writer); + DeleteFileW(file); return false; } off += written; } + CloseHandle(writer); + int utf8_len = WideCharToMultiByte(CP_UTF8, 0, file, -1, nullptr, 0, nullptr, nullptr); if (utf8_len <= 0) { errmsg_ = "could not encode the temporary file path"; - CloseHandle(handle_); - handle_ = INVALID_HANDLE_VALUE; + DeleteFileW(file); return false; } path_.resize(utf8_len - 1); WideCharToMultiByte( CP_UTF8, 0, file, -1, path_.data(), utf8_len, nullptr, nullptr); + wpath_ = file; return true; } -void AddonImage::AfterOpen(bool opened) { +void AddonImage::AfterOpen(bool opened, void* module) { consumed_ = true; - if (handle_ == INVALID_HANDLE_VALUE) return; + if (wpath_.empty()) return; if (!opened) { - CloseHandle(handle_); // delete-on-close removes the file - handle_ = INVALID_HANDLE_VALUE; + DeleteFileW(wpath_.c_str()); // nothing mapped it, so it can go now + wpath_.clear(); return; } - Mutex::ScopedLock lock(g_retained_addon_handles_mutex); - if (g_retained_addon_handles == nullptr) { - g_retained_addon_handles = new std::vector(); + // The load mapped it, so it has to stay until that module is unloaded again. + Mutex::ScopedLock lock(g_retained_addon_images_mutex); + if (g_retained_addon_images == nullptr) { + g_retained_addon_images = new std::vector(); } - g_retained_addon_handles->push_back(handle_); - handle_ = INVALID_HANDLE_VALUE; + g_retained_addon_images->push_back( + {static_cast(module), std::move(wpath_)}); + wpath_.clear(); } AddonImage::~AddonImage() { - // Materialized but Open() was never reached (e.g. an exception in between): - // closing the delete-on-close handle removes the file. - if (!consumed_ && handle_ != INVALID_HANDLE_VALUE) CloseHandle(handle_); + // Materialized but the load was never reached (e.g. an exception in + // between): nothing mapped the file, so remove it now. + if (!consumed_ && !wpath_.empty()) DeleteFileW(wpath_.c_str()); } #else // !_WIN32 @@ -700,10 +705,12 @@ bool AddonImage::MaterializeTempFile(const char* data, size_t len) { return true; } -void AddonImage::AfterOpen(bool opened) { +void AddonImage::AfterOpen(bool opened, void* module) { consumed_ = true; - // The right cleanup is the same whether or not the load worked. + // The right cleanup is the same whether or not the load worked, and the + // module never has to be unloaded: the name is already gone by now. (void)opened; + (void)module; // memfd: the load's mapping (or nothing, on failure) owns it from here. if (fd_ != -1) { close(fd_); @@ -730,8 +737,6 @@ AddonImage::~AddonImage() { #endif // _WIN32 -} // namespace - // Shared by process.dlopen() and the internal dlopenBinary(). `allow_binary` // says whether args[3] may carry the addon's bytes; it is false for // process.dlopen(), whose signature stays (module, filename[, flags]). @@ -814,7 +819,7 @@ static void DLOpenImpl(const FunctionCallbackInfo& args, Mutex::ScopedLock lock(dlib_load_mutex); const bool is_opened = dlib->Open(); - image.AfterOpen(is_opened); + image.AfterOpen(is_opened, is_opened ? dlib->handle_ : nullptr); // Objects containing v14 or later modules will have registered themselves // on the pending list. Activate all of them now. At present, only one diff --git a/src/node_binding.h b/src/node_binding.h index c200cc0d0c8a..0f19475ab48e 100644 --- a/src/node_binding.h +++ b/src/node_binding.h @@ -7,6 +7,8 @@ #include #endif +#include + #include "node.h" #include "node_api.h" #include "quic/guard.h" @@ -170,6 +172,60 @@ void GetLinkedBinding(const v8::FunctionCallbackInfo& args); void DLOpen(const v8::FunctionCallbackInfo& args); void DLOpenBinary(const v8::FunctionCallbackInfo& args); +// Materializes the bytes of a dynamically shared object into a form +// dlopen()/LoadLibrary() can load, with the smallest, most private on-disk +// footprint each platform allows: +// Linux: an anonymous in-memory memfd, loaded via /proc/self/fd/N - +// the bytes never touch the filesystem. +// other POSIX: a 0700 mkdtemp() directory plus an O_EXCL|O_NOFOLLOW file, +// unlink()ed right after the load (the mapping keeps it alive). +// Windows: a temp file, written and closed before the load because the +// loader shares read alone. It cannot be unlinked while its +// image is mapped, so it is kept with the module it loaded as +// and both are released at process exit. +// Used for a native addon or an FFI library that lives somewhere the dynamic +// loader cannot open by path, such as a virtual file system. Call exactly one +// of Materialize()+AfterOpen() around the load; a destroyed image that never +// reached AfterOpen() cleans up after itself. +class AddonImage { + public: + AddonImage() = default; + ~AddonImage(); + AddonImage(const AddonImage&) = delete; + AddonImage& operator=(const AddonImage&) = delete; + + // The directory a temporary image would be written to, with a trailing + // separator; empty when it cannot be determined. Names the resource for the + // file-system permission check. + static std::string TempDir(); + + // On success sets path() to a real, loadable path for `data`. + bool Materialize(const char* data, size_t len); + const std::string& path() const { return path_; } + const std::string& errmsg() const { return errmsg_; } + + // Call exactly once, right after the load; `opened` says whether the load + // succeeded and `module` is the module handle it produced. Releases what is + // no longer needed: on POSIX closes the memfd or unlinks the temp file, which + // a successful load keeps alive through its own mapping. Windows cannot + // unlink a mapped image, so there the file is removed at once only when the + // load failed; otherwise it is kept, with `module`, until process exit, where + // the module is unloaded and the file finally deleted. + void AfterOpen(bool opened, void* module); + + private: + std::string path_; + std::string errmsg_; + bool consumed_ = false; +#ifdef _WIN32 + std::wstring wpath_; // the path of the image, to delete it again at exit +#else + bool MaterializeTempFile(const char* data, size_t len); + int fd_ = -1; + std::string temp_dir_; +#endif +}; + } // namespace binding } // namespace node diff --git a/src/node_ffi.cc b/src/node_ffi.cc index 88327f1f1c47..a596fab7c683 100644 --- a/src/node_ffi.cc +++ b/src/node_ffi.cc @@ -10,6 +10,7 @@ #include "ffi/data.h" #include "ffi/fast.h" #include "ffi/types.h" +#include "node_binding.h" #include "node_errors.h" namespace node { @@ -525,9 +526,44 @@ void DynamicLibrary::New(const FunctionCallbackInfo& args) { library_path = lib->path_.c_str(); } + // On the internal path args[1] carries the library's bytes, for a library + // that lives somewhere the dynamic loader cannot open by path (a virtual + // file system). Materialize them into a private, self-cleaning image - the + // same mechanism process.dlopen() uses for such native addons - and load + // that, while still reporting the library's own path in `library.path` and + // any error. + binding::AddonImage image; + if (args.Length() > 1 && !args[1]->IsUndefined()) { + if (!args[1]->IsArrayBufferView()) { + THROW_ERR_INVALID_ARG_TYPE( + env, "Library binary must be a Buffer, TypedArray, or DataView"); + return; + } + // Loading from bytes materializes them into an image in the temporary + // directory, so this needs write access there on top of the FFI + // permission checked above. The check does not depend on whether the + // image actually reaches the file system on this platform (Linux uses an + // anonymous memfd): what a program must be granted should not vary by + // platform. + THROW_IF_INSUFFICIENT_PERMISSIONS( + env, + permission::PermissionScope::kFileSystemWrite, + binding::AddonImage::TempDir()); + ArrayBufferViewContents binary(args[1]); + if (!image.Materialize(binary.data(), binary.length())) { + THROW_ERR_FFI_CALL_FAILED( + env, "dlopen failed: %s: %s", image.errmsg().c_str(), library_path); + return; + } + library_path = image.path().c_str(); + } + CHECK(lib->is_closed()); // Open the library - if (uv_dlopen(library_path, &lib->lib_) != 0) { + const bool opened = uv_dlopen(library_path, &lib->lib_) == 0; + image.AfterOpen(opened, + opened ? static_cast(lib->lib_.handle) : nullptr); + if (!opened) { THROW_ERR_FFI_CALL_FAILED(env, "dlopen failed: %s", uv_dlerror(&lib->lib_)); return; } diff --git a/test/ffi/test-ffi-vfs.js b/test/ffi/test-ffi-vfs.js new file mode 100644 index 000000000000..884bd9f0d5da --- /dev/null +++ b/test/ffi/test-ffi-vfs.js @@ -0,0 +1,94 @@ +// Flags: --experimental-vfs +'use strict'; +const common = require('../common'); +common.skipIfFFIMissing(); +const assert = require('node:assert'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { test } = require('node:test'); +const ffi = require('node:ffi'); +const vfs = require('node:vfs'); +const { fixtureSymbols, libraryPath } = require('./ffi-test-common'); + +// A library inside a mounted VFS loads transparently: the dynamic loader +// cannot open the reserved mount path, so its bytes are read from the VFS +// and loaded from a private, self-cleaning image - the same way require() +// handles a native addon in a VFS. +const libraryName = path.basename(libraryPath); +const myVfs = vfs.create(); +myVfs.writeFileSync(`/${libraryName}`, fs.readFileSync(libraryPath)); +const mountPoint = myVfs.mount(); +const virtualPath = path.join(mountPoint, libraryName); + +test('ffi.dlopen() loads a library from a mounted VFS', () => { + const before = new Set(fs.readdirSync(os.tmpdir())); + const { lib, functions } = ffi.dlopen(virtualPath, { + add_i32: fixtureSymbols.add_i32, + }); + + try { + assert.ok(lib instanceof ffi.DynamicLibrary); + // The library reports its own (virtual) path, not the image's. + assert.strictEqual(lib.path, virtualPath); + assert.strictEqual(functions.add_i32(2, 40), 42); + } finally { + lib.close(); + } + + // On Linux the image is an in-memory memfd that never touches the + // filesystem; on other POSIX it is unlinked right after loading. + // (Windows keeps a delete-on-close file until exit, so skip there.) + if (!common.isWindows) { + const leaked = fs.readdirSync(os.tmpdir()) + .filter((f) => f.startsWith('node-addon') && !before.has(f)); + assert.deepStrictEqual(leaked, [], `image not cleaned up: ${leaked}`); + } +}); + +test('new ffi.DynamicLibrary() loads from a mounted VFS', () => { + const lib = new ffi.DynamicLibrary(virtualPath); + + try { + assert.strictEqual(lib.path, virtualPath); + const addU8 = lib.getFunction('add_u8', fixtureSymbols.add_u8); + assert.strictEqual(addU8(19, 23), 42); + } finally { + lib.close(); + } +}); + +test('a missing library inside the VFS throws ENOENT', () => { + assert.throws(() => { + ffi.dlopen(path.join(mountPoint, 'no-such-library.so')); + }, { code: 'ENOENT' }); +}); + +test('libraries on the real file system still load directly', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + add_i32: fixtureSymbols.add_i32, + }); + + try { + assert.strictEqual(lib.path, libraryPath); + assert.strictEqual(functions.add_i32(-1, 2), 1); + } finally { + lib.close(); + } +}); + +test('a library loaded from a VFS outlives the mount', () => { + const otherVfs = vfs.create(); + otherVfs.writeFileSync(`/${libraryName}`, fs.readFileSync(libraryPath)); + const otherMount = otherVfs.mount(); + const { lib, functions } = ffi.dlopen(path.join(otherMount, libraryName), { + add_i32: fixtureSymbols.add_i32, + }); + + try { + otherVfs.unmount(); + assert.strictEqual(functions.add_i32(20, 22), 42); + } finally { + lib.close(); + } +}); diff --git a/test/parallel/test-dlopen-binary-image-cleanup.js b/test/parallel/test-dlopen-binary-image-cleanup.js new file mode 100644 index 000000000000..dabaeabaa0cc --- /dev/null +++ b/test/parallel/test-dlopen-binary-image-cleanup.js @@ -0,0 +1,90 @@ +// Flags: --expose-internals +'use strict'; + +// Loading an addon from bytes materializes them into a private image so the +// dynamic loader has a real path to open. That image is transient and must not +// outlive the process that loaded it. How it is held differs by platform, so +// this checks both halves of the contract: +// +// Linux: an anonymous memfd loaded through /proc/self/fd - nothing ever +// reaches the filesystem, and AfterOpen() closes the descriptor +// once the load owns its mapping, so repeated loads must not +// accumulate open descriptors. +// other POSIX: a mkdtemp() directory unlinked and rmdir()ed right after the +// load, so nothing is left even while the process runs. +// Windows: the loader maps the file by path for the DLL's lifetime, so +// the image has to stay put; it is retained with the module it +// loaded as, and both are released at process exit. + +const common = require('../common'); +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const { spawnSync } = require('child_process'); +const tmpdir = require('../common/tmpdir'); + +const addonPath = path.join( + __dirname, '..', 'addons', 'hello-world', 'build', 'Release', 'binding.node'); +if (!fs.existsSync(addonPath)) common.skip('the hello-world addon is not built'); + +tmpdir.refresh(); + +// Where a temp-file image would land: GetTempPathW() reads TMP/TEMP and +// TempDir() reads TMPDIR, so pointing all three at a directory this test owns +// keeps any image the child writes somewhere it can inspect afterwards. Linux +// normally uses a memfd and never writes here at all. +const imageDir = tmpdir.resolve('addon-images'); +fs.mkdirSync(imageDir, { recursive: true }); + +const child = ` + const fs = require('fs'); + const { internalBinding } = require('internal/test/binding'); + const { dlopenBinary } = internalBinding('process_methods'); + const bytes = fs.readFileSync(${JSON.stringify(addonPath)}); + // A path that does not exist on disk, as a VFS-resident addon would be, so + // the load can only come from the bytes and their materialized image. + const virtualPath = ${JSON.stringify(path.join(addonPath, '..', 'nowhere', 'binding.node'))}; + + // On Linux the image is a descriptor rather than a file, so count them: each + // load must hand its fd to the mapping and close it, leaving no growth. + const fdDir = '/proc/self/fd'; + const countFds = () => { + try { return fs.readdirSync(fdDir).length; } catch { return -1; } + }; + const before = countFds(); + + // Load repeatedly: each load materializes its own image, so a leak of an + // image, a descriptor or a retained handle shows up as growth. + for (let i = 0; i < 5; i++) { + const m = { exports: {} }; + // flags undefined: keep the default dlopen(2) mode - 0 is not valid + // everywhere (glibc rejects it with EINVAL). + dlopenBinary(m, virtualPath, undefined, bytes); + if (m.exports.hello() !== 'world') throw new Error('addon did not load'); + } + + const after = countFds(); + if (before !== -1 && after > before) { + throw new Error(\`descriptor leak: \${before} -> \${after} after 5 loads\`); + } + process.exit(0); +`; + +const res = spawnSync(process.execPath, ['--expose-internals', '-e', child], { + env: { ...process.env, TMPDIR: imageDir, TMP: imageDir, TEMP: imageDir }, + encoding: 'utf8', +}); + +// The load itself must succeed. On Windows a retained writable handle makes the +// loader fail with ERROR_SHARING_VIOLATION ("The process cannot access the file +// because it is being used by another process"). +assert.strictEqual(res.status, 0, `child failed:\n${res.stderr}`); + +// Nothing an image left behind may outlive the process that created it. Match +// the shapes the two on-disk paths produce rather than requiring the directory +// to be empty, so an unrelated temp file cannot fail this. +const leftovers = fs.readdirSync(imageDir).filter( + (name) => /^nod.*\.tmp$/i.test(name) || name.startsWith('node-addon-')); +assert.deepStrictEqual( + leftovers, [], + `materialized addon image outlived the process that loaded it: ${leftovers}`); diff --git a/test/parallel/test-vfs-addon.js b/test/parallel/test-vfs-addon.js index 1c138a187d0a..7ea8fdbf7943 100644 --- a/test/parallel/test-vfs-addon.js +++ b/test/parallel/test-vfs-addon.js @@ -34,4 +34,12 @@ if (process.platform !== 'win32') { assert.deepStrictEqual(leaked, [], `addon temp not cleaned up: ${leaked}`); } +// Regression check: while a VFS is mounted, process.dlopen() of a +// real-file-system addon without a flags argument must keep the default +// flags rather than forwarding `undefined`, which coerces to 0 - not a +// valid dlopen(2) mode. +const realMod = { exports: {} }; +process.dlopen(realMod, addonPath); +assert.strictEqual(realMod.exports.hello(), 'world'); + myVfs.unmount(); From 89297305c09bbe8d9198681d657793e66d4e47ac Mon Sep 17 00:00:00 2001 From: greenhead Date: Sun, 13 Sep 2026 09:05:04 +0900 Subject: [PATCH 122/217] test: fix stderr Buffer assertion in exec encoding test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: greenhead PR-URL: https://github.com/nodejs/node/pull/66008 Refs: https://github.com/nodejs/node/pull/10919 Reviewed-By: Luigi Pinca Reviewed-By: James M Snell Reviewed-By: Xuguang Mei Reviewed-By: Ulises Gascón --- test/parallel/test-child-process-exec-encoding.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/parallel/test-child-process-exec-encoding.js b/test/parallel/test-child-process-exec-encoding.js index 21ab207fca8c..78a9b55b9dfc 100644 --- a/test/parallel/test-child-process-exec-encoding.js +++ b/test/parallel/test-child-process-exec-encoding.js @@ -41,7 +41,7 @@ if (process.argv[2] === 'child') { [undefined, null, 'buffer', 'invalid'].forEach((encoding) => { run({ encoding }, common.mustCall((stdout, stderr) => { assert(stdout instanceof Buffer); - assert(stdout instanceof Buffer); + assert(stderr instanceof Buffer); assert.strictEqual(stdout.toString(), expectedStdout); assert.strictEqual(stderr.toString(), expectedStderr); })); From 61a98e6512afd66befadb125bec9c4edb360b0fe Mon Sep 17 00:00:00 2001 From: Tim Perry <1526883+pimterry@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:29:34 +0200 Subject: [PATCH 123/217] quic: fix readable stream truncation on stop-sending, abort & timeout Signed-off-by: Tim Perry PR-URL: https://github.com/nodejs/node/pull/63967 Reviewed-By: James M Snell --- doc/api/quic.md | 24 +++ lib/internal/blob.js | 6 +- lib/internal/errors.js | 2 - lib/internal/quic/quic.js | 130 +++++++++---- lib/internal/quic/state.js | 35 +++- src/quic/streams.cc | 46 +++-- src/quic/streams.h | 12 +- test/common/quic.mjs | 79 ++++++++ .../test-quic-stream-iteration-destroyed.mjs | 37 ---- .../test-quic-stream-iteration-reset.mjs | 64 ------ .../test-quic-stream-setbody-errors.mjs | 8 +- ...st-quic-stream-truncated-reads-destroy.mjs | 72 +++++++ ...st-quic-stream-truncated-reads-timeout.mjs | 35 ++++ .../test-quic-stream-truncated-reads.mjs | 184 ++++++++++++++++++ 14 files changed, 556 insertions(+), 178 deletions(-) delete mode 100644 test/parallel/test-quic-stream-iteration-destroyed.mjs delete mode 100644 test/parallel/test-quic-stream-iteration-reset.mjs create mode 100644 test/parallel/test-quic-stream-truncated-reads-destroy.mjs create mode 100644 test/parallel/test-quic-stream-truncated-reads-timeout.mjs create mode 100644 test/parallel/test-quic-stream-truncated-reads.mjs diff --git a/doc/api/quic.md b/doc/api/quic.md index f49323a02e5e..c63ca25554c0 100644 --- a/doc/api/quic.md +++ b/doc/api/quic.md @@ -3310,6 +3310,30 @@ value, PING frames will be sent automatically to keep the connection alive before the idle timeout fires. The value should be less than the effective idle timeout (`maxIdleTimeout` transport parameter) to be useful. +#### `sessionOptions.truncatedReads` + +* Type: {string} One of `'error'` or `'ignore'`. +* **Default:** `'error'` + +Controls how reading a stream reports a truncated read. A stream's read side +can end without receiving a QUIC FIN, meaning the peer never signalled that +the whole stream had been sent and the data received may be incomplete. This +selects how the stream's async iterator reports this: + +* `'error'` - The default. Peers are expected to always send a FIN to end + their data explicitly, and so any truncation is an error. The iterator yields + the data that did arrive and then throws, so an incomplete stream can never + be mistaken for a complete one. Incomplete streams will either throw a + `ERR_QUIC_STREAM_RESET` carrying the peer's error code, a connection error, + or `ERR_QUIC_STREAM_ABORTED` for other cases. + +* `'ignore'` - The truncation itself is ignored: only a stream or connection + error is reported, and any clean abort/cancellation or similar simply ends + the stream. A non-zero peer reset, non-zero local stop-sending or connection + error still fails, but a truncation with no error at all (an idle timeout, + a graceful close, or a plain `stopSending()`) ends the read cleanly with the + data received. This matches `stream.closed`, which rejects only on an error. + #### `sessionOptions.verifyPeer` (client only) * Type: {string} One of `'strict'`, `'auto'`, or `'manual'`. diff --git a/lib/internal/blob.js b/lib/internal/blob.js index 316518ca401b..61cc0cbf22df 100644 --- a/lib/internal/blob.js +++ b/lib/internal/blob.js @@ -625,7 +625,7 @@ const kMaxBatchChunks = 16; const kDefaultMaxBatchBytes = 65536; async function* createBlobReaderIterable(reader, options = kEmptyObject) { - const { getReadError, maxBatchBytes = kDefaultMaxBatchBytes } = options; + const { maxBatchBytes = kDefaultMaxBatchBytes } = options; let wakeup = PromiseWithResolvers(); let immediate; reader.setWakeup(() => { @@ -658,9 +658,7 @@ async function* createBlobReaderIterable(reader, options = kEmptyObject) { break; } if (pullResult.status < 0) { - error = typeof getReadError === 'function' ? - getReadError(pullResult.status) : - new ERR_INVALID_STATE('The reader is not readable'); + error = new ERR_INVALID_STATE('The reader is not readable'); break; } if (pullResult.status === 2) { diff --git a/lib/internal/errors.js b/lib/internal/errors.js index b9b1490c31ee..9332e85773f8 100644 --- a/lib/internal/errors.js +++ b/lib/internal/errors.js @@ -1715,8 +1715,6 @@ E('ERR_QUIC_CONNECTION_FAILED', 'QUIC connection failed', Error); E('ERR_QUIC_ENDPOINT_CLOSED', 'QUIC endpoint closed: %s (%d)', Error); E('ERR_QUIC_OPEN_STREAM_FAILED', 'Failed to open QUIC stream', Error); E('ERR_QUIC_STREAM_ABORTED', '%s', Error); -E('ERR_QUIC_STREAM_RESET', - 'The QUIC stream was reset by the peer with error code %d', Error); E('ERR_QUIC_VERSION_NEGOTIATION_ERROR', 'The QUIC session requires version negotiation', Error); E('ERR_REQUIRE_ASYNC_MODULE', function(filename, parent, locations) { let message = 'require() cannot be used on an ESM graph with top-level await. Use import() instead.'; diff --git a/lib/internal/quic/quic.js b/lib/internal/quic/quic.js index 33e419002495..2ee2529b1c81 100644 --- a/lib/internal/quic/quic.js +++ b/lib/internal/quic/quic.js @@ -13,7 +13,6 @@ const { ErrorCaptureStackTrace, FunctionPrototypeBind, FunctionPrototypeCall, - Number, ObjectDefineProperties, ObjectKeys, PromisePrototypeThen, @@ -117,7 +116,6 @@ const { ERR_QUIC_ENDPOINT_CLOSED, ERR_QUIC_OPEN_STREAM_FAILED, ERR_QUIC_STREAM_ABORTED, - ERR_QUIC_STREAM_RESET, ERR_QUIC_VERSION_NEGOTIATION_ERROR, }, } = require('internal/errors'); @@ -401,6 +399,7 @@ const endpointRegistry = new SafeSet(); * @property {number} [minVersion] The minimum acceptable QUIC version * @property {'use'|'ignore'|'default'} [preferredAddressPolicy] The preferred address policy * @property {'strict'|'auto'|'manual'} [verifyPeer='auto'] Peer certificate verification policy (client only) + * @property {'error'|'ignore'} [truncatedReads] Truncated read policy * @property {ApplicationOptions} [application] The application options * @property {TransportParams} [transportParams] The transport parameters * @property {string} [servername] The server name identifier (client only) @@ -1600,6 +1599,9 @@ class QuicStream { state: undefined, stats: undefined, pendingClose: undefined, + destroyError: undefined, + stopSendingCode: undefined, + truncatedReads: undefined, reader: undefined, destroying: false, iteratorLocked: false, @@ -1648,9 +1650,10 @@ class QuicStream { * @param {object} handle * @param {QuicSession} session * @param {number} direction - * @param {boolean} [isLocal] + * @param {boolean} isLocal + * @param {'error'|'ignore'} truncatedReads */ - constructor(privateSymbol, handle, session, direction, isLocal) { + constructor(privateSymbol, handle, session, direction, isLocal, truncatedReads) { assertPrivateSymbol(privateSymbol); this.#handle = handle; @@ -1659,6 +1662,7 @@ class QuicStream { inner.session = session; inner.direction = direction; inner.isLocal = isLocal; + inner.truncatedReads = truncatedReads; inner.state = new QuicStreamState( kPrivateConstructor, handle.state, handle.stateByteOffset); @@ -1690,34 +1694,49 @@ class QuicStream { inner.iteratorLocked = true; inner.reader ??= this.#handle?.getReader(); - // Non-readable stream (outbound-only unidirectional, or closed) - if (!inner.reader) return; - - yield* createBlobReaderIterable(inner.reader, { - getReadError: () => { - // The read side ends for one of three reasons: - // * Clean FIN received from the peer (state.finReceived - // === true). The iterator stops without calling this; - // fall through to the generic state error if it does. - // * Peer sent us a RESET_STREAM. The C++ side records the - // code in state.resetCode regardless of whether the JS - // onreset handler was attached. state.finReceived stays - // false because no FIN was seen. - // * We aborted locally via stream.resetStream() or - // stream.stopSending(). Both paths run EndReadable in - // C++, setting state.readEnded without setting - // state.finReceived. There is no peer code to surface. - if (inner.state.readEnded && !inner.state.finReceived) { - const peerResetCode = inner.state.resetCode; - if (peerResetCode !== undefined && peerResetCode > 0n) { - return new ERR_QUIC_STREAM_RESET(Number(peerResetCode)); - } - return new ERR_QUIC_STREAM_ABORTED( - 'Stream aborted before FIN was received'); - } - return new ERR_INVALID_STATE('The stream is not readable'); - }, - }); + // No reader means either an outbound-only unidirectional stream, or a + // stream already destroyed (data gone, but truncation must still be + // checked below). + if (inner.reader) { + yield* createBlobReaderIterable(inner.reader); + } + + if (inner.state.readEnded && !inner.state.finReceived) { + // The readable has been truncated - ended with no clean FIN. We expose + // this in different ways depending on the truncatedReads option. + + // If we cancelled ourselves, check our own stop-sending code (not the + // result mirrored by the remote peer) + if (inner.stopSendingCode > 0n) { + throw new QuicError('Stream aborted before FIN was received', + { __proto__: null, + errorCode: inner.stopSendingCode }); + } + + // Non-zero reset is always an error: + const peerResetCode = inner.state.resetCode; + if (peerResetCode > 0n) { + throw new QuicError( + `The QUIC stream was reset by the peer with error code ${peerResetCode}`, + { __proto__: null, + code: 'ERR_QUIC_STREAM_RESET', + errorCode: peerResetCode }); + } + + // If stream teardown has started (stats.destroyedAt is set) then a + // close event confirming a final error/clean close will settle + // imminently (might be settled already). We await here to rethrow + // any errors if the connection has failed. + if (this.destroyed || this.stats.destroyedAt !== 0n) { + await this.closed; + } + + // Clean abort is truncation, but not necessarily an error: + if (inner.truncatedReads === 'error') { + throw new QuicError('Stream aborted before FIN was received', + { __proto__: null, errorCode: peerResetCode ?? 0n }); + } + } } /** @@ -2099,9 +2118,11 @@ class QuicStream { if (error !== undefined && typeof inner.onerror === 'function') { invokeOnerror(inner.onerror, error); } - const handle = this.#handle; - this[kFinishClose](error); - handle.destroy(); + // handle.destroy() kicks off all the cleanup internals, eventually + // including [kFinishClose] which needs this original destroy error: + inner.destroyError = error; + this.#handle.destroy(); + this[kFinishClose](error); // A no-op, unless destroy() failed } /** @@ -2519,7 +2540,9 @@ class QuicStream { stopSending(code = 0n) { assertIsQuicStream(this); if (this.destroyed) return; - this.#handle.stopSending(BigInt(code)); + const abortCode = BigInt(code); + this.#inner.stopSendingCode = abortCode; + this.#handle.stopSending(abortCode); } /** @@ -2612,6 +2635,9 @@ class QuicStream { if (this.destroyed) { return inner.pendingClose.promise; } + // Prefer an error staged by destroy() (the original object the caller + // passed) over the error delivered by the native close callback. + error = inner.destroyError ?? error; if (error !== undefined) { inner.pendingClose.reject(error); } else { @@ -2649,6 +2675,7 @@ class QuicStream { inner.session = undefined; inner.pendingClose.reject = undefined; inner.pendingClose.resolve = undefined; + inner.destroyError = undefined; inner.onblocked = undefined; inner.onreset = undefined; inner.onstopsending = undefined; @@ -2859,6 +2886,7 @@ class QuicSession { // because server-side cert validation is handled by rejectUnauthorized // at the C++ level. verifyPeer: 'manual', + truncatedReads: 'error', handshakeInfo: undefined, /** @type {QuicSessionPath|undefined} */ path: undefined, @@ -2892,8 +2920,9 @@ class QuicSession { * @param {symbol} privateSymbol * @param {object} handle * @param {QuicEndpoint} endpoint + * @param {{ truncatedReads?: 'error'|'ignore' }} [options] */ - constructor(privateSymbol, handle, endpoint) { + constructor(privateSymbol, handle, endpoint, options = kEmptyObject) { // Instances of QuicSession can only be created internally. assertPrivateSymbol(privateSymbol); @@ -2902,6 +2931,8 @@ class QuicSession { const inner = this.#inner; inner.endpoint = endpoint; + const { truncatedReads } = options; + if (truncatedReads !== undefined) inner.truncatedReads = truncatedReads; // Move any qlog entries that arrived before the wrapper existed. if (handle._pendingQlog !== undefined) { inner.pendingQlog = handle._pendingQlog; @@ -3447,7 +3478,8 @@ class QuicSession { } const stream = new QuicStream( - kPrivateConstructor, handle, this, direction, true /* isLocal */); + kPrivateConstructor, handle, this, direction, true /* isLocal */, + inner.truncatedReads); inner.streams.add(stream); if (typeof this.#inner.onerror === 'function') { markPromiseAsHandled(stream.closed); @@ -4232,7 +4264,7 @@ class QuicSession { [kNewStream](handle, direction) { const inner = this.#inner; const stream = new QuicStream(kPrivateConstructor, handle, this, direction, - false /* isLocal */); + false /* isLocal */, inner.truncatedReads); // Set the default byte budget for received streams. stream.budget = kDefaultBudget; @@ -4349,6 +4381,7 @@ class QuicEndpoint { sessions: new SafeSet(), stat: undefined, stats: undefined, + truncatedReads: undefined, onsession: undefined, sessionCallbacks: undefined, }; @@ -4481,8 +4514,8 @@ class QuicEndpoint { }; } - #newSession(handle) { - const session = new QuicSession(kPrivateConstructor, handle, this); + #newSession(handle, options) { + const session = new QuicSession(kPrivateConstructor, handle, this, options); this.#inner.sessions.add(session); // Set default pending datagram queue size. session.maxPendingDatagrams = kDefaultMaxPendingDatagrams; @@ -4656,9 +4689,13 @@ class QuicEndpoint { ontrailers, oninfo, onwanttrailers, + // Stored on the endpoint and applied to each incoming session. + truncatedReads, ...rest } = options; + inner.truncatedReads = truncatedReads; + // Store session and stream callbacks to apply to each new incoming session. inner.sessionCallbacks = { __proto__: null, @@ -4700,6 +4737,7 @@ class QuicEndpoint { validateObject(options, 'options'); const { sessionTicket, + truncatedReads, ...rest } = options; @@ -4708,7 +4746,7 @@ class QuicEndpoint { if (handle === undefined) { throw new ERR_QUIC_CONNECTION_FAILED(); } - const session = this.#newSession(handle); + const session = this.#newSession(handle, { __proto__: null, truncatedReads }); // Set callbacks before any async work to avoid missing events // that fire during or immediately after the handshake. applyCallbacks(session, options); @@ -4942,7 +4980,8 @@ class QuicEndpoint { const inner = this.#inner; assert(typeof inner.onsession === 'function', 'onsession callback not specified'); - const session = this.#newSession(handle); + const session = this.#newSession(handle, + { __proto__: null, truncatedReads: inner.truncatedReads }); // Apply session callbacks stored at listen time before notifying // the onsession callback, to avoid missing events that fire // during or immediately after the handshake. @@ -5434,6 +5473,7 @@ function processSessionOptions(options, config = kEmptyObject) { maxDatagramSendAttempts = 5, streamIdleTimeout, verifyPeer = 'auto', + truncatedReads = 'error', // HTTP/3 application-specific options. Nested under `application` // to separate protocol-specific settings from transport-level ones. application = kEmptyObject, @@ -5485,6 +5525,9 @@ function processSessionOptions(options, config = kEmptyObject) { validateOneOf(verifyPeer, 'options.verifyPeer', ['strict', 'auto', 'manual']); + validateOneOf(truncatedReads, 'options.truncatedReads', + ['error', 'ignore']); + validateInteger(drainingPeriodMultiplier, 'options.drainingPeriodMultiplier', 3, 255); @@ -5547,6 +5590,7 @@ function processSessionOptions(options, config = kEmptyObject) { verifyHostname: verifyPeer !== 'manual', }, verifyPeer, + truncatedReads, qlog, maxPayloadSize, unacknowledgedPacketThreshold, diff --git a/lib/internal/quic/state.js b/lib/internal/quic/state.js index 21db0f0e3d59..0a47e2e7adc8 100644 --- a/lib/internal/quic/state.js +++ b/lib/internal/quic/state.js @@ -8,7 +8,9 @@ const { DataView, DataViewPrototypeGetBigInt64, DataViewPrototypeGetBigUint64, + DataViewPrototypeGetBuffer, DataViewPrototypeGetByteLength, + DataViewPrototypeGetByteOffset, DataViewPrototypeGetUint16, DataViewPrototypeGetUint32, DataViewPrototypeGetUint8, @@ -17,6 +19,9 @@ const { DataViewPrototypeSetUint8, JSONStringify, Number, + TypedArrayPrototypeGetBuffer, + TypedArrayPrototypeSlice, + Uint8Array, } = primordials; const { @@ -107,6 +112,7 @@ const { IDX_STATE_STREAM_WRITE_DESIRED_SIZE, IDX_STATE_STREAM_BUDGET, IDX_STATE_STREAM_RESET_CODE, + IDX_STATE_STREAM_SIZE, } = internalBinding('quic'); assert(IDX_STATE_SESSION_LISTENER_FLAGS !== undefined); @@ -723,8 +729,8 @@ class QuicStreamState { #handle; /** @type {number} */ #offset = 0; - /** @type {bigint|undefined} */ - #id = undefined; + /** @type {boolean} */ + #disconnected = false; /** * @param {symbol} privateSymbol @@ -746,7 +752,7 @@ class QuicStreamState { /** @type {bigint} */ get id() { const handle = this.#handle; - if (handle === undefined) return this.#id; + if (handle === undefined) return undefined; return DataViewPrototypeGetBigInt64(handle, this.#offset + IDX_STATE_STREAM_ID, kIsLittleEndian); } @@ -943,7 +949,7 @@ class QuicStreamState { } [kInspect](depth, options) { - if (this.#handle === undefined || + if (this.#disconnected || this.#handle === undefined || DataViewPrototypeGetByteLength(this.#handle) === 0) { return 'QuicStreamState { }'; } @@ -998,9 +1004,24 @@ class QuicStreamState { } [kFinishClose]() { - // Cache the stream ID since the buffer will be zeroed out and the ID will be lost. - this.#id = this.id; - this.#handle = undefined; + // Copy the final values out of the shared buffer (zeroed when the + // underlying stream goes away) so getters keep reporting the final + // state, the same way QuicStreamStats does. + const handle = this.#handle; + if (handle === undefined || + DataViewPrototypeGetByteLength(handle) < + this.#offset + IDX_STATE_STREAM_SIZE) { + this.#handle = undefined; + this.#disconnected = true; + return; + } + const copy = TypedArrayPrototypeSlice(new Uint8Array( + DataViewPrototypeGetBuffer(handle), + DataViewPrototypeGetByteOffset(handle) + this.#offset, + IDX_STATE_STREAM_SIZE)); + this.#handle = new DataView(TypedArrayPrototypeGetBuffer(copy)); + this.#offset = 0; + this.#disconnected = true; } } diff --git a/src/quic/streams.cc b/src/quic/streams.cc index 0a5a783711ba..aae72740f871 100644 --- a/src/quic/streams.cc +++ b/src/quic/streams.cc @@ -1008,6 +1008,9 @@ void Stream::InitPerContext(Realm* realm, Local target) { #undef V NODE_DEFINE_CONSTANT(target, IDX_STATS_STREAM_COUNT); + + constexpr auto IDX_STATE_STREAM_SIZE = sizeof(Stream::State); + NODE_DEFINE_CONSTANT(target, IDX_STATE_STREAM_SIZE); } Stream* Stream::From(void* stream_user_data) { @@ -1321,13 +1324,6 @@ BaseObjectPtr Stream::get_reader() { return reader; } -void Stream::set_final_size(uint64_t final_size) { - DCHECK_IMPLIES(state()->fin_received == 1, - final_size <= STAT_GET(Stats, final_size)); - state()->fin_received = 1; - STAT_SET(Stats, final_size, final_size); -} - void Stream::set_outbound(std::shared_ptr source) { if (!source || !is_writable()) return; Debug(this, "Setting the outbound data source"); @@ -1471,7 +1467,7 @@ void Stream::FlushAccumulation() { return; } // Should be unreachable: append() only fails once the queue has been capped, - // EndReadable() flushes before capping, and ReceiveData() accumulates + // CapReadable() flushes before capping, and ReceiveData() accumulates // nothing once read_ended is set. Reaching here means received stream data // is being dropped on the floor, so say so and at least do not also leak // the flow control credit for it. @@ -1536,13 +1532,26 @@ void Stream::EndWritable() { state()->write_ended = 1; } -void Stream::EndReadable(std::optional maybe_final_size) { +void Stream::FinishReadable() { + if (!is_readable()) return; + state()->fin_received = 1; + CapReadable(std::nullopt); +} + +void Stream::TruncateReadable(std::optional maybe_final_size) { if (!is_readable()) return; + CapReadable(maybe_final_size); +} + +void Stream::CapReadable(std::optional maybe_final_size) { + DCHECK(is_readable()); state()->read_ended = 1; // Flush any accumulated data before capping so the reader can see it. FlushAccumulation(); - set_final_size(maybe_final_size.value_or(STAT_GET(Stats, bytes_received))); - inbound_->cap(STAT_GET(Stats, final_size)); + const uint64_t final_size = + maybe_final_size.value_or(STAT_GET(Stats, bytes_received)); + STAT_SET(Stats, final_size, final_size); + inbound_->cap(final_size); // Notify the JS reader so it can see EOS. The subsequent pull observes // the now-capped DataQueue and returns EOS. if (reader_) reader_->NotifyPull(); @@ -1570,14 +1579,15 @@ void Stream::Destroy(QuicError error) { // End the writable before marking as destroyed. EndWritable(); - // Also end the readable side if it isn't already. - EndReadable(); + // Also end the readable side if it isn't already. If not already ended, + // this will eventually surface as an error, since the data is truncated. + TruncateReadable(); // We are going to release our reference to the outbound_ queue here. outbound_.reset(); application_state_.reset(); - // EndReadable() above already flushed accumulated data. Just release + // TruncateReadable() above already flushed accumulated data. Just release // the ring buffer memory. recv_accumulator_.reset(); @@ -1632,7 +1642,7 @@ void Stream::ReceiveData(const uint8_t* data, // end the readable side if this is the last bit of data we've received. Debug(this, "Receiving %zu bytes of data", len); if (state()->read_ended == 1 || len == 0) { - if (flags.fin) EndReadable(); + if (flags.fin) FinishReadable(); // Nothing will ever consume these bytes, so return the connection-level // credit ngtcp2 charged for them. The stream window is deliberately left // alone: there is no point inviting more data onto a stream we have @@ -1715,7 +1725,7 @@ void Stream::ReceiveData(const uint8_t* data, if (flags.fin) { FlushAccumulation(); - EndReadable(); + FinishReadable(); } else if (reader_ && was_empty) { // Notify the reader once when the accumulator transitions from empty // to non-empty. This wakes the reader exactly once per accumulation @@ -1746,7 +1756,7 @@ void Stream::ReceiveStreamReset(uint64_t final_size, QuicError error) { final_size, error); state()->reset_code = error.code(); - EndReadable(final_size); + TruncateReadable(final_size); EmitReset(error); } @@ -1769,7 +1779,7 @@ void Stream::DoStreamReset(error_code code) { } void Stream::SendStopSending(error_code code) { - EndReadable(); + TruncateReadable(); if (!is_pending()) { // If the stream is a local unidirectional there's nothing to do here. diff --git a/src/quic/streams.h b/src/quic/streams.h index 182237c6334d..38e8f2ae978e 100644 --- a/src/quic/streams.h +++ b/src/quic/streams.h @@ -325,7 +325,12 @@ class Stream final : public AsyncWrap, void UpdateWriteDesiredSize(); void EndWritable(); - void EndReadable(std::optional maybe_final_size = std::nullopt); + // The read side ended cleanly with a peer FIN: the content is complete. + void FinishReadable(); + // The read side ended without a FIN (a reset, a local abort, or the session + // being torn down) so the content is truncated. + void TruncateReadable( + std::optional maybe_final_size = std::nullopt); void EntryRead(size_t amount) override; void BeforePull() override; @@ -416,10 +421,13 @@ class Stream final : public AsyncWrap, // consumer or dropped before reaching one. void CreditConsumedBytes(uint64_t amount); + // Common tail of FinishReadable()/TruncateReadable(): marks the read side + // ended, caps it at the final size, and notifies the reader. + void CapReadable(std::optional maybe_final_size); + // Gets a reader for the data received for this stream from the peer, BaseObjectPtr get_reader(); - void set_final_size(uint64_t amount); void set_outbound(std::shared_ptr source); // Streaming outbound support diff --git a/test/common/quic.mjs b/test/common/quic.mjs index dc4b094cf900..72effd8b6528 100644 --- a/test/common/quic.mjs +++ b/test/common/quic.mjs @@ -7,6 +7,7 @@ // listen/connect that apply default options suitable for most tests. import * as fixtures from '../common/fixtures.mjs'; +import { setTimeout } from 'node:timers/promises'; const { createPrivateKey } = await import('node:crypto'); const quic = await import('node:quic'); @@ -98,6 +99,81 @@ function hashBytes(buf) { return h >>> 0; } +/** + * Write `size` bytes to the stream and wait until the peer has + * acknowledged them. + */ +async function writeAndAwaitAck(stream, size) { + stream.writer.write(new Uint8Array(size).fill(7)); + while (stream.stats.maxOffsetAcknowledged < BigInt(size)) await setTimeout(5); +} + +/** + * Send `size` bytes and then stall forever without a FIN + * @yields {Uint8Array} + */ +async function* stallingBody(size) { + yield new Uint8Array(size).fill(7); + await new Promise(() => {}); +} + +/** + * Do a full single stream-read scenario, with the given client options and + * server body, and two hooks available to tweak behaviour, returning the + * details of the result after completion. + * @param {Function} serverBody + * @param {object} [options] + * @param {object} [options.clientOptions] Options forwarded to connect(). + * @param {Function} [options.beforeIterate] + * @param {Function} [options.onFirstChunk] + * @returns {Promise<{received: number, threw: any, closedError: any}>} + * Bytes received, the iteration error if any, and the client stream's + * closed rejection if any. + */ +async function readStream(serverBody, options = {}) { + const { clientOptions, beforeIterate, onFirstChunk } = options; + + const serverEndpoint = await listen((session) => { + session.closed.catch(() => {}); + session.onstream = (stream) => { + stream.closed.catch(() => {}); + serverBody(stream, session); + }; + }); + + const session = await connect(serverEndpoint.address, clientOptions); + await session.opened; + session.closed.catch(() => {}); + + const stream = await session.createBidirectionalStream(); + await stream.writer.write(new Uint8Array([1])); + + let closedError; + const closedSettled = stream.closed.catch((err) => { closedError = err; }); + + await beforeIterate?.({ stream, session }); + + let received = 0; + let threw; + let firstChunk = true; + try { + for await (const chunk of stream) { + for (const c of chunk) received += c.byteLength; + if (firstChunk) { + firstChunk = false; + await onFirstChunk?.({ stream, session }); + } + } + } catch (err) { + threw = err; + } + + session.close(); + await closedSettled; + await serverEndpoint.close(); + return { received, threw, closedError }; +} + export { key, cert, @@ -105,4 +181,7 @@ export { connect, makePayload, hashBytes, + readStream, + stallingBody, + writeAndAwaitAck, }; diff --git a/test/parallel/test-quic-stream-iteration-destroyed.mjs b/test/parallel/test-quic-stream-iteration-destroyed.mjs deleted file mode 100644 index 585be6d4a813..000000000000 --- a/test/parallel/test-quic-stream-iteration-destroyed.mjs +++ /dev/null @@ -1,37 +0,0 @@ -// Flags: --experimental-quic --no-warnings - -// Test: destroyed stream returns finished iterator. - -import { hasQuic, skip, mustCall } from '../common/index.mjs'; -import * as assert from 'node:assert'; - -if (!hasQuic) { - skip('QUIC is not enabled'); -} - -const { listen, connect } = await import('../common/quic.mjs'); - -const encoder = new TextEncoder(); - -const serverEndpoint = await listen(mustCall(async (serverSession) => { - await serverSession.closed; -})); - -const clientSession = await connect(serverEndpoint.address); -await clientSession.opened; - -const stream = await clientSession.createBidirectionalStream({ - body: encoder.encode('destroy test'), -}); - -// Destroy the stream immediately. -stream.destroy(); - -// Iterating a destroyed stream should immediately finish. -const iter = stream[Symbol.asyncIterator](); -const { done } = await iter.next(); -assert.strictEqual(done, true); - -await stream.closed; -await clientSession.close(); -await serverEndpoint.destroy(); diff --git a/test/parallel/test-quic-stream-iteration-reset.mjs b/test/parallel/test-quic-stream-iteration-reset.mjs deleted file mode 100644 index 6df571bd9ea7..000000000000 --- a/test/parallel/test-quic-stream-iteration-reset.mjs +++ /dev/null @@ -1,64 +0,0 @@ -// Flags: --experimental-quic --experimental-stream-iter --no-warnings - -// Test: peer RESET_STREAM causes iterator to error. -// When the server resets the stream, the client's async iterator -// should throw or return early. - -import { hasQuic, skip, mustCall } from '../common/index.mjs'; -import * as assert from 'node:assert'; - -if (!hasQuic) { - skip('QUIC is not enabled'); -} - -const { listen, connect } = await import('../common/quic.mjs'); - -const encoder = new TextEncoder(); - -const serverReady = Promise.withResolvers(); - -const serverEndpoint = await listen(mustCall((serverSession) => { - serverSession.onstream = mustCall(async (stream) => { - // Reset the stream from the server side. - stream.resetStream(42n); - await assert.rejects(stream.closed, mustCall((err) => { - assert.ok(err); - return true; - })); - serverReady.resolve(); - await serverSession.closed; - }); -}), { transportParams: { maxIdleTimeout: 1 } }); - -const clientSession = await connect(serverEndpoint.address, { - transportParams: { maxIdleTimeout: 1 }, -}); -await clientSession.opened; - -const stream = await clientSession.createBidirectionalStream({ - body: encoder.encode('will be reset by server'), -}); - -// Set up the closed handler before the reset to avoid unhandled rejection. -const closedPromise = assert.rejects(stream.closed, mustCall((err) => { - assert.ok(err); - return true; -})); - -await serverReady.promise; - -// The async iterator should either throw or return early when the -// peer resets the readable side. -try { - for await (const batch of stream) { - // May receive some data before the reset arrives. - assert.ok(Array.isArray(batch)); - } -} catch { - // The iterator may throw when the reset arrives mid-iteration. -} - -// Either way, the stream should close. -await closedPromise; -await clientSession.closed; -await serverEndpoint.close(); diff --git a/test/parallel/test-quic-stream-setbody-errors.mjs b/test/parallel/test-quic-stream-setbody-errors.mjs index a15f2347171e..da045d764c2f 100644 --- a/test/parallel/test-quic-stream-setbody-errors.mjs +++ b/test/parallel/test-quic-stream-setbody-errors.mjs @@ -53,7 +53,13 @@ await clientSession.opened; message: /writer already accessed/, }); - for await (const _ of stream) { /* drain */ } // eslint-disable-line no-unused-vars + // The server handles only the first stream and then closes its session, so + // this stream is never answered and never receives a FIN. Reading it + // therefore surfaces the truncation rather than ending cleanly. + await assert.rejects((async () => { + // eslint-disable-next-line no-unused-vars + for await (const _ of stream) { /* drain */ } + })(), { code: 'ERR_QUIC_STREAM_ABORTED' }); await stream.closed; } diff --git a/test/parallel/test-quic-stream-truncated-reads-destroy.mjs b/test/parallel/test-quic-stream-truncated-reads-destroy.mjs new file mode 100644 index 000000000000..01eddd3b0f0f --- /dev/null +++ b/test/parallel/test-quic-stream-truncated-reads-destroy.mjs @@ -0,0 +1,72 @@ +// Flags: --experimental-quic --no-warnings + +// Test: truncation is still reported when the stream is torn down locally +// before the iterator finishes draining. The stream's final read state and +// close error are persisted at close, so a consumer slower than the teardown +// (or one that only starts reading after it) still observes the truncation +// rather than a clean end that would make an incomplete stream look complete. + +import { hasQuic, skip } from '../common/index.mjs'; +import { setTimeout } from 'node:timers/promises'; +import assert from 'node:assert'; + +if (!hasQuic) { + skip('QUIC is not enabled'); +} + +const { readStream, stallingBody } = await import('../common/quic.mjs'); + +// The server sends 1000 bytes then stalls without a FIN, so the client's +// read side can only end by truncation. +const serve = (stream) => { stream.setBody(stallingBody(1000)); }; + +// The session is destroyed with an error while the consumer is mid-iteration: +// the iterator delivers the data that arrived, then throws that same error +// object. +{ + const boom = new Error('boom'); + const { received, threw } = await readStream(serve, { + onFirstChunk: ({ session }) => session.destroy(boom), + }); + assert.ok(received > 0); + assert.strictEqual(threw, boom); +} + +// The session is destroyed without an error mid-iteration: an errorless +// truncation, reported as aborted under the default policy. +{ + const { received, threw } = await readStream(serve, { + onFirstChunk: ({ session }) => session.destroy(), + }); + assert.ok(received > 0); + assert.strictEqual(threw?.code, 'ERR_QUIC_STREAM_ABORTED'); + assert.strictEqual(threw.errorCode, 0n); +} + +// The session is destroyed before iteration even starts: the buffered data +// is gone, but the truncation is still reported rather than a clean empty +// end. +{ + const { received, threw } = await readStream(serve, { + beforeIterate: async ({ stream, session }) => { + while (stream.stats.bytesReceived === 0n) await setTimeout(5); + session.destroy(); + }, + }); + assert.strictEqual(received, 0); + assert.strictEqual(threw?.code, 'ERR_QUIC_STREAM_ABORTED'); +} + +// Destroying the stream itself (rather than the session) before iterating +// reports the truncation the same way, with closed resolving cleanly. +{ + const { received, threw, closedError } = await readStream(serve, { + beforeIterate: async ({ stream }) => { + while (stream.stats.bytesReceived === 0n) await setTimeout(5); + stream.destroy(); + }, + }); + assert.strictEqual(received, 0); + assert.strictEqual(threw?.code, 'ERR_QUIC_STREAM_ABORTED'); + assert.strictEqual(closedError, undefined); +} diff --git a/test/parallel/test-quic-stream-truncated-reads-timeout.mjs b/test/parallel/test-quic-stream-truncated-reads-timeout.mjs new file mode 100644 index 000000000000..2ff19f463c3c --- /dev/null +++ b/test/parallel/test-quic-stream-truncated-reads-timeout.mjs @@ -0,0 +1,35 @@ +// Flags: --experimental-quic --no-warnings + +// Test: a readable truncated by the connection idle timeout delivers the +// data it received, then ends per the truncatedReads policy: an error under +// the default (so an incomplete stream can never look complete), and a clean +// end under 'ignore' (an idle timeout carries no error). + +import { hasQuic, skip } from '../common/index.mjs'; +import assert from 'node:assert'; + +if (!hasQuic) { + skip('QUIC is not enabled'); +} + +const { readStream, stallingBody } = await import('../common/quic.mjs'); + +// The server sends 1000 bytes then stalls without a FIN, so the short (1s) +// connection idle timeout is the only thing that ends the read side. +const serve = (stream) => { stream.setBody(stallingBody(1000)); }; +const transportParams = { maxIdleTimeout: 1 }; + +{ + const { received, threw } = + await readStream(serve, { clientOptions: { transportParams } }); + assert.strictEqual(received, 1000); + assert.strictEqual(threw?.code, 'ERR_QUIC_STREAM_ABORTED'); + assert.strictEqual(threw.errorCode, 0n); +} +{ + const { received, threw } = await readStream(serve, { + clientOptions: { transportParams, truncatedReads: 'ignore' }, + }); + assert.strictEqual(received, 1000); + assert.strictEqual(threw, undefined); +} diff --git a/test/parallel/test-quic-stream-truncated-reads.mjs b/test/parallel/test-quic-stream-truncated-reads.mjs new file mode 100644 index 000000000000..e5c381a8d679 --- /dev/null +++ b/test/parallel/test-quic-stream-truncated-reads.mjs @@ -0,0 +1,184 @@ +// Flags: --experimental-quic --no-warnings + +// Test: a stream read that ends without a FIN is a truncation. What the read +// reports is decided in this order: +// +// 1. the peer reset the stream with a non-zero code: ERR_QUIC_STREAM_RESET +// carrying that code, under either policy; +// 2. the stream was already being torn down and its closed promise rejected: +// that same error, under either policy; +// 3. otherwise the truncation carries no error of its own and the +// truncatedReads policy decides - 'error' (the default) throws +// ERR_QUIC_STREAM_ABORTED so an incomplete stream can never look +// complete, 'ignore' ends the read cleanly. +// +// This file covers rules 1 and 3 on a live connection: peer resets and local +// aborts. Rule 2 needs the stream to already be tearing down when the reader +// resumes, which is covered by the sibling +// test-quic-stream-truncated-reads-destroy test, and truncation by idle +// timeout by test-quic-stream-truncated-reads-timeout. + +import { hasQuic, skip } from '../common/index.mjs'; +import assert from 'node:assert'; + +if (!hasQuic) { + skip('QUIC is not enabled'); +} + +const { connect, listen, readStream, stallingBody, writeAndAwaitAck } = + await import('../common/quic.mjs'); + +// Sends 1000 bytes, waits for them to land, then resets with the given code. +const resetWith = (code) => async (stream) => { + await writeAndAwaitAck(stream, 1000); + stream.resetStream(code); +}; + +// Sends 1000 bytes and then stalls without a FIN, so only the client's own +// abort can end the read. +const stall = (stream) => { stream.setBody(stallingBody(1000)); }; + +// A peer reset with code 0 is a clean abort: a truncation, but not an error. +// Rule 3 - the default reports it, 'ignore' treats it as a clean end. +{ + const { received, threw } = await readStream(resetWith(0n)); + assert.strictEqual(received, 1000); + assert.strictEqual(threw?.code, 'ERR_QUIC_STREAM_ABORTED'); + assert.strictEqual(threw.errorCode, 0n); +} +{ + const { received, threw } = + await readStream(resetWith(0n), { clientOptions: { truncatedReads: 'ignore' } }); + assert.strictEqual(received, 1000); + assert.strictEqual(threw, undefined); +} + +// A peer reset with a nonzero code is rule 1: an error under either policy, +// carrying the peer's code. It also rejects the closed promise on both sides. +{ + const serverClosed = Promise.withResolvers(); + const { received, threw, closedError } = await readStream(async (stream) => { + await writeAndAwaitAck(stream, 1000); + stream.resetStream(42n); + // Our own reset closes the server-side stream with an error too. + serverClosed.resolve(stream.closed.then(() => undefined, (err) => err)); + }); + assert.strictEqual(received, 1000); + assert.strictEqual(threw?.code, 'ERR_QUIC_STREAM_RESET'); + assert.strictEqual(threw.errorCode, 42n); + assert.strictEqual(threw.message, + 'The QUIC stream was reset by the peer with error code 42'); + assert.strictEqual(closedError?.code, 'ERR_QUIC_APPLICATION_ERROR'); + assert.strictEqual(closedError.errorCode, 42n); + const serverClosedError = await serverClosed.promise; + assert.strictEqual(serverClosedError?.code, 'ERR_QUIC_APPLICATION_ERROR'); + assert.strictEqual(serverClosedError.errorCode, 42n); +} +{ + const { received, threw } = + await readStream(resetWith(42n), { clientOptions: { truncatedReads: 'ignore' } }); + assert.strictEqual(received, 1000); + assert.strictEqual(threw?.code, 'ERR_QUIC_STREAM_RESET'); + assert.strictEqual(threw.errorCode, 42n); +} + +// Aborting our own read with stopSending() is rule 3, not rule 1: we asked for +// the truncation, so it is not an error the peer inflicted on us, and the code +// we send does not come back as one. The read still stops short, so the +// default policy reports it and 'ignore' does not. +{ + const { received, threw, closedError } = await readStream(stall, { + onFirstChunk: ({ stream }) => stream.stopSending(0n), + }); + assert.ok(received > 0); + assert.strictEqual(threw?.code, 'ERR_QUIC_STREAM_ABORTED'); + assert.strictEqual(threw.errorCode, 0n); + assert.strictEqual(closedError, undefined); +} +{ + const { received, threw } = await readStream(stall, { + clientOptions: { truncatedReads: 'ignore' }, + onFirstChunk: ({ stream }) => stream.stopSending(0n), + }); + assert.ok(received > 0); + assert.strictEqual(threw, undefined); +} + +// A nonzero stopSending code is an error this end raised, so it is reported +// under either policy and carries that code. The peer answers our +// STOP_SENDING with a RESET_STREAM echoing it, which is what rejects closed - +// but the read reports our own abort rather than attributing it to the peer, +// and does so whether or not that answer has arrived yet. +for (const truncatedReads of ['error', 'ignore']) { + for (const awaitEcho of [false, true]) { + const peerReset = Promise.withResolvers(); + const { received, threw, closedError } = await readStream(stall, { + clientOptions: { truncatedReads }, + beforeIterate: ({ stream }) => { stream.onreset = () => peerReset.resolve(); }, + onFirstChunk: async ({ stream }) => { + stream.stopSending(7n); + // For the 2nd pass, wait until we receive the corresponding reset: + if (awaitEcho) await peerReset.promise; + }, + }); + assert.ok(received > 0); + assert.strictEqual(threw?.code, 'ERR_QUIC_STREAM_ABORTED'); + assert.strictEqual(threw.errorCode, 7n); + assert.strictEqual(closedError?.code, 'ERR_QUIC_APPLICATION_ERROR'); + assert.strictEqual(closedError.errorCode, 7n); + } +} + +// A peer abruptly destroying its session truncates the read too. That +// currently reaches us as an implicit reset, so which rule applies depends on +// the code the peer's teardown puts on the wire - not part of this contract, +// and deliberately not pinned here. What must hold either way is the default +// policy's promise: the incomplete stream never looks complete. +{ + const { received, threw } = await readStream(async (stream, session) => { + await writeAndAwaitAck(stream, 1000); + session.destroy(new Error('connection boom')); + }); + assert.strictEqual(received, 1000); + assert.ok(threw); +} + +// Check the option in the server case as well: +{ + const serverRead = Promise.withResolvers(); + const serverEndpoint = await listen((session) => { + session.closed.catch(() => {}); + session.onstream = async (stream) => { + stream.closed.catch(() => {}); + let received = 0; + try { + for await (const chunk of stream) { + for (const c of chunk) received += c.byteLength; + } + serverRead.resolve({ received, threw: undefined }); + } catch (err) { + serverRead.resolve({ received, threw: err }); + } + }; + }, { truncatedReads: 'ignore' }); + + const session = await connect(serverEndpoint.address); + await session.opened; + session.closed.catch(() => {}); + + const stream = await session.createBidirectionalStream(); + stream.closed.catch(() => {}); + await writeAndAwaitAck(stream, 100); + stream.resetStream(0n); + + const { threw } = await serverRead.promise; + assert.strictEqual(threw, undefined); + + session.close(); + await serverEndpoint.close(); +} + +// The option is validated. +await assert.rejects(connect('127.0.0.1:1234', { truncatedReads: 'nope' }), { + code: 'ERR_INVALID_ARG_VALUE', +}); From 66f182f5651b1b9bd441207e17fa01de1f8353af Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Sun, 13 Sep 2026 11:58:50 +0200 Subject: [PATCH 124/217] tools: make checkout credential use explicit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disable credential persistence for CodeQL. Signed-off-by: Filip Skokan Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/66013 Reviewed-By: Antoine du Hamel Reviewed-By: James M Snell Reviewed-By: Xuguang Mei Reviewed-By: Ulises Gascón --- .github/workflows/codeql.yml | 2 ++ .github/workflows/commit-queue.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0b83c888ecdc..10a9e299145b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -25,6 +25,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/commit-queue.yml b/.github/workflows/commit-queue.yml index 600dadc17bc4..8f29a390c678 100644 --- a/.github/workflows/commit-queue.yml +++ b/.github/workflows/commit-queue.yml @@ -188,6 +188,7 @@ jobs: # to be set here because `checkout` configures GitHub authentication # for push as well. token: ${{ secrets.GH_USER_TOKEN }} + persist-credentials: true - name: Start the Commit Queue if: steps.get_mergeable_prs.outputs.numbers != '' From d5880dcb058e5bad7f520c84faefd9b155cc81f0 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Sun, 13 Sep 2026 11:58:50 +0200 Subject: [PATCH 125/217] tools: correct Slack action version comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filip Skokan Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/66013 Reviewed-By: Antoine du Hamel Reviewed-By: James M Snell Reviewed-By: Xuguang Mei Reviewed-By: Ulises Gascón --- .github/workflows/notify-on-push.yml | 4 ++-- .github/workflows/notify-on-review-wanted.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/notify-on-push.yml b/.github/workflows/notify-on-push.yml index 16bd91bccd2a..25421b8447d8 100644 --- a/.github/workflows/notify-on-push.yml +++ b/.github/workflows/notify-on-push.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-24.04-arm steps: - name: Slack Notification - uses: rtCamp/action-slack-notify@33ca3be66c6f378fe1610fd1d5258632dbed5e58 # 2.4.0 + uses: rtCamp/action-slack-notify@33ca3be66c6f378fe1610fd1d5258632dbed5e58 # v2.4.0 env: SLACK_COLOR: '#DE512A' SLACK_ICON: https://github.com/nodejs.png?size=48 @@ -50,7 +50,7 @@ jobs: COMMITS: ${{ toJSON(github.event.commits) }} - name: Slack Notification if: ${{ failure() && steps.commit-check.conclusion == 'failure' && github.repository == 'nodejs/node' }} - uses: rtCamp/action-slack-notify@33ca3be66c6f378fe1610fd1d5258632dbed5e58 # 2.4.0 + uses: rtCamp/action-slack-notify@33ca3be66c6f378fe1610fd1d5258632dbed5e58 # v2.4.0 env: SLACK_COLOR: '#DE512A' SLACK_ICON: https://github.com/nodejs.png?size=48 diff --git a/.github/workflows/notify-on-review-wanted.yml b/.github/workflows/notify-on-review-wanted.yml index 2f1f3af8139b..effc6c209eb1 100644 --- a/.github/workflows/notify-on-review-wanted.yml +++ b/.github/workflows/notify-on-review-wanted.yml @@ -34,7 +34,7 @@ jobs: fi - name: Slack Notification - uses: rtCamp/action-slack-notify@33ca3be66c6f378fe1610fd1d5258632dbed5e58 # 2.4.0 + uses: rtCamp/action-slack-notify@33ca3be66c6f378fe1610fd1d5258632dbed5e58 # v2.4.0 env: MSG_MINIMAL: actions url SLACK_COLOR: '#3d85c6' From caf0ee0d7b42b2eeef5a73b775bdc4e029dfc154 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Sun, 13 Sep 2026 11:58:51 +0200 Subject: [PATCH 126/217] tools: use self-repository references MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolve local actions and reusable workflows from the running workflow commit with $/ references, independent of checkout paths. Remove the tarball job's action-only checkout and the WPT action checkout/copy workaround, which are no longer needed. Signed-off-by: Filip Skokan Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/66013 Reviewed-By: Antoine du Hamel Reviewed-By: James M Snell Reviewed-By: Xuguang Mei Reviewed-By: Ulises Gascón --- .github/workflows/build-tarball.yml | 7 +----- .../workflows/coverage-linux-without-intl.yml | 2 +- .github/workflows/coverage-linux.yml | 2 +- .github/workflows/daily-wpt-fyi.yml | 22 ++----------------- .github/workflows/stress-test.yml | 2 +- .github/workflows/test-internet.yml | 2 +- .github/workflows/test-linux-perfetto.yml | 2 +- .github/workflows/test-linux-quic.yml | 2 +- .github/workflows/test-linux.yml | 2 +- .github/workflows/test-shared.yml | 4 ++-- 10 files changed, 12 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build-tarball.yml b/.github/workflows/build-tarball.yml index e173d4a544e0..045bd3ae93c2 100644 --- a/.github/workflows/build-tarball.yml +++ b/.github/workflows/build-tarball.yml @@ -104,13 +104,8 @@ jobs: SCCACHE_GHA_ENABLED: ${{ github.base_ref == 'main' || github.ref_name == 'main' }} SCCACHE_IDLE_TIMEOUT: '0' steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - sparse-checkout: .github/actions/install-clang - sparse-checkout-cone-mode: false - name: Install Clang ${{ env.CLANG_VERSION }} - uses: ./.github/actions/install-clang + uses: $/.github/actions/install-clang with: clang-version: ${{ env.CLANG_VERSION }} - name: Set up Python ${{ env.PYTHON_VERSION }} diff --git a/.github/workflows/coverage-linux-without-intl.yml b/.github/workflows/coverage-linux-without-intl.yml index 7a73d69b70ec..a98492ec668e 100644 --- a/.github/workflows/coverage-linux-without-intl.yml +++ b/.github/workflows/coverage-linux-without-intl.yml @@ -54,7 +54,7 @@ jobs: with: persist-credentials: false - name: Install Clang ${{ env.CLANG_VERSION }} - uses: ./.github/actions/install-clang + uses: $/.github/actions/install-clang with: clang-version: ${{ env.CLANG_VERSION }} - name: Set up Python ${{ env.PYTHON_VERSION }} diff --git a/.github/workflows/coverage-linux.yml b/.github/workflows/coverage-linux.yml index 79c6eaaebc91..4ba373b790a2 100644 --- a/.github/workflows/coverage-linux.yml +++ b/.github/workflows/coverage-linux.yml @@ -54,7 +54,7 @@ jobs: with: persist-credentials: false - name: Install Clang ${{ env.CLANG_VERSION }} - uses: ./.github/actions/install-clang + uses: $/.github/actions/install-clang with: clang-version: ${{ env.CLANG_VERSION }} - name: Set up Python ${{ env.PYTHON_VERSION }} diff --git a/.github/workflows/daily-wpt-fyi.yml b/.github/workflows/daily-wpt-fyi.yml index 043de271faa9..8901ecbcee42 100644 --- a/.github/workflows/daily-wpt-fyi.yml +++ b/.github/workflows/daily-wpt-fyi.yml @@ -105,33 +105,15 @@ jobs: else echo "UNDICI_WPT=legacy" >> $GITHUB_ENV fi - # Checkout composite actions from the default branch since the - # version-specific checkout above overwrites .github/actions/ - - name: Checkout undici WPT actions - if: ${{ env.WPT_REPORT != '' }} - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - sparse-checkout: | - .github/actions/undici-wpt-current - .github/actions/undici-wpt-legacy - sparse-checkout-cone-mode: false - persist-credentials: false - path: _wpt_actions - clean: false - - name: Place undici WPT actions - if: ${{ env.WPT_REPORT != '' }} - run: | - mkdir -p .github/actions - cp -r _wpt_actions/.github/actions/undici-wpt-* .github/actions/ - name: Run undici WPT (current) if: ${{ env.UNDICI_WPT == 'current' }} - uses: ./.github/actions/undici-wpt-current + uses: $/.github/actions/undici-wpt-current with: undici-version: ${{ env.UNDICI_VERSION }} wpt-report: ${{ env.WPT_REPORT }} - name: Run undici WPT (legacy) if: ${{ env.UNDICI_WPT == 'legacy' }} - uses: ./.github/actions/undici-wpt-legacy + uses: $/.github/actions/undici-wpt-legacy with: undici-version: ${{ env.UNDICI_VERSION }} wpt-report: ${{ env.WPT_REPORT }} diff --git a/.github/workflows/stress-test.yml b/.github/workflows/stress-test.yml index 1c93d4ba3dda..17393eb5713b 100644 --- a/.github/workflows/stress-test.yml +++ b/.github/workflows/stress-test.yml @@ -60,7 +60,7 @@ jobs: path: node - name: Install Clang ${{ env.CLANG_VERSION }} if: runner.os == 'Linux' - uses: ./node/.github/actions/install-clang + uses: $/.github/actions/install-clang with: clang-version: ${{ env.CLANG_VERSION }} - name: Set up Xcode ${{ env.XCODE_VERSION }} diff --git a/.github/workflows/test-internet.yml b/.github/workflows/test-internet.yml index 08677618b3d6..43d71f5f3165 100644 --- a/.github/workflows/test-internet.yml +++ b/.github/workflows/test-internet.yml @@ -51,7 +51,7 @@ jobs: with: persist-credentials: false - name: Install Clang ${{ env.CLANG_VERSION }} - uses: ./.github/actions/install-clang + uses: $/.github/actions/install-clang with: clang-version: ${{ env.CLANG_VERSION }} - name: Set up Python ${{ env.PYTHON_VERSION }} diff --git a/.github/workflows/test-linux-perfetto.yml b/.github/workflows/test-linux-perfetto.yml index 70a92202c1c7..ce31704f8166 100644 --- a/.github/workflows/test-linux-perfetto.yml +++ b/.github/workflows/test-linux-perfetto.yml @@ -42,7 +42,7 @@ jobs: persist-credentials: false path: node - name: Install Clang ${{ env.CLANG_VERSION }} - uses: ./node/.github/actions/install-clang + uses: $/.github/actions/install-clang with: clang-version: ${{ env.CLANG_VERSION }} - name: Install Rust ${{ env.RUSTC_VERSION }} diff --git a/.github/workflows/test-linux-quic.yml b/.github/workflows/test-linux-quic.yml index 70aa8d9b04d1..0eded996fefa 100644 --- a/.github/workflows/test-linux-quic.yml +++ b/.github/workflows/test-linux-quic.yml @@ -52,7 +52,7 @@ jobs: persist-credentials: false path: node - name: Install Clang ${{ env.CLANG_VERSION }} - uses: ./node/.github/actions/install-clang + uses: $/.github/actions/install-clang with: clang-version: ${{ env.CLANG_VERSION }} - name: Install Rust ${{ env.RUSTC_VERSION }} diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 408371f3313c..74dc45ae8c0b 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -65,7 +65,7 @@ jobs: persist-credentials: false path: node - name: Install Clang ${{ env.CLANG_VERSION }} - uses: ./node/.github/actions/install-clang + uses: $/.github/actions/install-clang with: clang-version: ${{ env.CLANG_VERSION }} - name: Install Rust ${{ env.RUSTC_VERSION }} diff --git a/.github/workflows/test-shared.yml b/.github/workflows/test-shared.yml index f7483103016f..be3c22face09 100644 --- a/.github/workflows/test-shared.yml +++ b/.github/workflows/test-shared.yml @@ -159,7 +159,7 @@ jobs: - runner: macos-latest system: aarch64-darwin name: '${{ matrix.system }}: with shared libraries${{ matrix.perfetto && '' and perfetto'' || '''' }}' - uses: ./.github/workflows/build-shared.yml + uses: $/.github/workflows/build-shared.yml with: runner: ${{ matrix.runner }} with-sccache: ${{ github.base_ref == 'main' || github.ref_name == 'main' }} @@ -250,7 +250,7 @@ jobs: matrix: openssl: ${{ fromJSON(needs.build-aarch64-linux-v8.outputs.matrix) }} name: 'aarch64-linux: with shared ${{ matrix.openssl.name }}' - uses: ./.github/workflows/build-shared.yml + uses: $/.github/workflows/build-shared.yml with: runner: ubuntu-24.04-arm v8-nar: ${{ needs.build-aarch64-linux-v8.outputs.local-cache && 'libv8-aarch64-linux.nar' }} From a5b1659d5bb8ff44e333311f74b63e1da4ee47b5 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Sun, 13 Sep 2026 11:58:51 +0200 Subject: [PATCH 127/217] tools: avoid workflow shell interpolation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass input, output, and version values through environment variables instead of expanding them into shell source. Split benchmark categories and PR numbers into arrays to retain separate arguments. Signed-off-by: Filip Skokan Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/66013 Reviewed-By: Antoine du Hamel Reviewed-By: James M Snell Reviewed-By: Xuguang Mei Reviewed-By: Ulises Gascón --- .github/workflows/auto-start-ci.yml | 4 +++- .github/workflows/benchmark.yml | 17 ++++++++++++----- .github/workflows/commit-queue.yml | 4 +++- .github/workflows/daily-wpt-fyi.yml | 7 +++++-- .github/workflows/timezone-update.yml | 4 ++-- .github/workflows/tools.yml | 2 +- 6 files changed, 26 insertions(+), 12 deletions(-) diff --git a/.github/workflows/auto-start-ci.yml b/.github/workflows/auto-start-ci.yml index 8c01b0592c2e..dc827d62ba71 100644 --- a/.github/workflows/auto-start-ci.yml +++ b/.github/workflows/auto-start-ci.yml @@ -72,7 +72,9 @@ jobs: - name: Start the CI run: | + read -r -a numbers <<< "$PULL_REQUESTS" curl -fsSL "https://github.com/${GITHUB_REPOSITORY}/raw/${GITHUB_SHA}/tools/actions/start-ci.sh" \ - | sh -s -- ${{ needs.get-prs-for-ci.outputs.numbers }} + | sh -s -- "${numbers[@]}" env: GH_TOKEN: ${{ github.token }} + PULL_REQUESTS: ${{ needs.get-prs-for-ci.outputs.numbers }} diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 924feb42725e..7f10917e238a 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -47,7 +47,9 @@ jobs: steps: - name: Mark token input as sensitive if: inputs.token != '' - run: echo "::add-mask::${{ inputs.token }}" + run: printf '::add-mask::%s\n' "$COMMENT_TOKEN" + env: + COMMENT_TOKEN: ${{ inputs.token }} - name: Add link to the current run id: comment run: | @@ -155,7 +157,7 @@ jobs: run: | nix-shell \ -I nixpkgs=./tools/nix/pkgs.nix \ - --pure --keep FILTER --keep LC_ALL --keep LANG \ + --pure --keep CATEGORIES --keep FILTER --keep RUNS --keep LC_ALL --keep LANG \ --arg loadJSBuiltinsDynamically false \ --arg ccache 'null' \ --arg icu 'null' \ @@ -163,11 +165,12 @@ jobs: --arg devTools '[]' \ --run ' set -o pipefail + read -r -a categories <<< "$CATEGORIES" ./base_node benchmark/compare.js \ --filter "$FILTER" \ - --runs ${{ inputs.runs }} \ + --runs "$RUNS" \ --old ./base_node --new ./node \ - -- ${{ inputs.category }} \ + -- "${categories[@]}" \ | tee /dev/stderr \ > ${{ matrix.system }}.csv echo "> [!WARNING] " @@ -185,7 +188,9 @@ jobs: echo "> using a dedicated machine, e.g. Jenkins CI." ' | tee /dev/stderr >> "$GITHUB_STEP_SUMMARY" env: + CATEGORIES: ${{ inputs.category }} FILTER: ${{ inputs.filter }} + RUNS: ${{ inputs.runs }} - name: Upload raw benchmark results uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -246,7 +251,9 @@ jobs: ' | tee /dev/stderr ${{ inputs.post-comment && 'body.txt' || '' }} >> "$GITHUB_STEP_SUMMARY" - name: Mark token input as sensitive if: inputs.token != '' - run: echo "::add-mask::${{ inputs.token }}" + run: printf '::add-mask::%s\n' "$COMMENT_TOKEN" + env: + COMMENT_TOKEN: ${{ inputs.token }} - name: Edit comment if: inputs.post-comment run: | diff --git a/.github/workflows/commit-queue.yml b/.github/workflows/commit-queue.yml index 8f29a390c678..83f921838b30 100644 --- a/.github/workflows/commit-queue.yml +++ b/.github/workflows/commit-queue.yml @@ -196,6 +196,8 @@ jobs: git config --local user.email "github-bot@iojs.org" git config --local user.name "Node.js GitHub Bot" ncu-config set token "$GH_TOKEN" - ./tools/actions/commit-queue.sh ${{ steps.get_mergeable_prs.outputs.numbers }} + read -r -a numbers <<< "$PULL_REQUESTS" + ./tools/actions/commit-queue.sh "${numbers[@]}" env: GH_TOKEN: ${{ secrets.GH_USER_TOKEN }} + PULL_REQUESTS: ${{ steps.get_mergeable_prs.outputs.numbers }} diff --git a/.github/workflows/daily-wpt-fyi.yml b/.github/workflows/daily-wpt-fyi.yml index 8901ecbcee42..d15f21010cd9 100644 --- a/.github/workflows/daily-wpt-fyi.yml +++ b/.github/workflows/daily-wpt-fyi.yml @@ -122,7 +122,9 @@ jobs: - name: Clone report for upload if: ${{ env.WPT_REPORT != '' }} working-directory: out/wpt - run: cp wptreport.json wptreport-${{ steps.setup-node.outputs.node-version }}.json + run: cp wptreport.json "wptreport-$NODE_VERSION.json" + env: + NODE_VERSION: ${{ steps.setup-node.outputs.node-version }} - name: Upload GitHub Actions artifact if: ${{ env.WPT_REPORT != '' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -133,12 +135,13 @@ jobs: - name: Upload WPT Report to wpt.fyi API if: ${{ env.WPT_REPORT != '' }} env: + NODE_VERSION: ${{ steps.setup-node.outputs.node-version }} WPT_FYI_USERNAME: ${{ vars.WPT_FYI_USERNAME }} WPT_FYI_PASSWORD: ${{ secrets.WPT_FYI_PASSWORD }} working-directory: out/wpt run: | gzip wptreport.json - echo "## Node.js ${{ steps.setup-node.outputs.node-version }}" >> $GITHUB_STEP_SUMMARY + echo "## Node.js $NODE_VERSION" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "WPT Revision: [\`${WPT_REVISION:0:7}\`](https://github.com/web-platform-tests/wpt/commits/$WPT_REVISION)" >> $GITHUB_STEP_SUMMARY for WPT_FYI_ENDPOINT in "https://wpt.fyi/api/results/upload" "https://staging.wpt.fyi/api/results/upload" diff --git a/.github/workflows/timezone-update.yml b/.github/workflows/timezone-update.yml index 481c20da4e05..abe4516db4a7 100644 --- a/.github/workflows/timezone-update.yml +++ b/.github/workflows/timezone-update.yml @@ -40,14 +40,14 @@ jobs: - name: Compare versions run: | - echo "Comparing current version ${{ env.current_version }} to new version ${{ env.new_version }}" + echo "Comparing current version $current_version to new version $new_version" - run: ./tools/dep_updaters/update-timezone.mjs if: ${{ env.new_version != env.current_version }} - name: Update the expected timezone version in test if: ${{ env.new_version != env.current_version }} - run: echo "${{ env.new_version }}" > test/fixtures/tz-version.txt + run: printf '%s\n' "$new_version" > test/fixtures/tz-version.txt - name: Open Pull Request if: ${{ env.new_version != env.current_version }} diff --git a/.github/workflows/tools.yml b/.github/workflows/tools.yml index 097902102685..fe2db7ac95c4 100644 --- a/.github/workflows/tools.yml +++ b/.github/workflows/tools.yml @@ -338,7 +338,7 @@ jobs: - name: Generate commit message if not set if: env.COMMIT_MSG == '' && (github.event_name == 'schedule' || inputs.id == 'all' || inputs.id == matrix.id) run: | - echo "COMMIT_MSG=${{ matrix.subsystem }}: update ${{ matrix.id }} to ${{ env.NEW_VERSION }}" >> "$GITHUB_ENV" + echo "COMMIT_MSG=${{ matrix.subsystem }}: update ${{ matrix.id }} to $NEW_VERSION" >> "$GITHUB_ENV" - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 if: github.event_name == 'schedule' || inputs.id == 'all' || inputs.id == matrix.id # Creates a PR or update the Action's existing PR, or From 3a69cc1bf489f37ebed37256fb594b4811c0bf82 Mon Sep 17 00:00:00 2001 From: John Finnerty Date: Wed, 16 Sep 2026 00:36:03 +1200 Subject: [PATCH 128/217] doc: clarify QUIC async write backpressure Signed-off-by: John Finnerty <297514060+johnfinnerty-nz@users.noreply.github.com> PR-URL: https://github.com/nodejs/node/pull/65947 Reviewed-By: James M Snell Reviewed-By: Xuguang Mei --- doc/api/quic.md | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/doc/api/quic.md b/doc/api/quic.md index c63ca25554c0..6f347dfec976 100644 --- a/doc/api/quic.md +++ b/doc/api/quic.md @@ -321,10 +321,17 @@ There are two ways to write data to a stream: up front or can be expressed as an iterable. * **Writer** — access [`stream.writer`][] to push data incrementally. The writer exposes synchronous methods (`writeSync()`, `writevSync()`, - `endSync()`) that return immediately, as well as async equivalents - (`write()`, `writev()`, `end()`) that wait for drain when backpressured. + `endSync()`) that return immediately, as well as asynchronous counterparts + (`write()`, `writev()`, `end()`). The asynchronous `write()` and `writev()` + methods use the stream/iter strict backpressure policy: when the write buffer + is full, they reject with `ERR_INVALID_STATE` instead of waiting for capacity. + If a drain is already pending, `end()` waits for it before closing. Check + `writer.canWrite` before writing. To wait for capacity, use `ondrain()` from + `node:stream/iter`, then retry the write. The stream's `onblocked` callback + reports that transport flow control has blocked progress, but does not + signal that writer capacity is available again. `writeSync()` returns `false` when the write buffer is full; the caller - should wait for drain before retrying. + should wait with `ondrain()` before retrying. These two approaches are mutually exclusive for a given stream. @@ -2353,12 +2360,16 @@ The Writer has the following methods: * `writeSync(chunk)` — Synchronous write. Returns `true` if accepted, `false` if flow-controlled. Data is NOT accepted on `false`. -* `write(chunk[, options])` — Async write with drain wait. `options.signal` - is checked at entry but not observed during the write. +* `write(chunk[, options])` — Async write. Rejects with `ERR_INVALID_STATE` + when the stream is flow-controlled rather than waiting for capacity. + `options.signal` is checked at entry but not observed during the write. * `writevSync(chunks)` — Synchronous vectored write. All-or-nothing. -* `writev(chunks[, options])` — Async vectored write. +* `writev(chunks[, options])` — Async vectored write. Rejects with + `ERR_INVALID_STATE` when the stream is flow-controlled rather than waiting + for capacity. * `endSync()` — Synchronous close. Returns total bytes or `-1`. -* `end([options])` — Async close. +* `end([options])` — Async close. If a drain is already pending, waits for it + before closing. * `fail(reason)` — Errors the stream (sends `RESET_STREAM` to peer). When `reason` is a [`QuicError`][], its [`error.errorCode`][] is used as the wire code on the resulting `RESET_STREAM` frame; otherwise @@ -2368,7 +2379,20 @@ The Writer has the following methods: See [`stream.destroy()`][] for a full-stream abort that also resets the readable side via `STOP_SENDING`. * `canWrite` — `true` if writes will be accepted, `false` if at capacity, - or `null` if closed/errored. + or `null` if closed/errored. When `writeSync()` returns `false`, use + `ondrain()` from `node:stream/iter` to wait before retrying. If `ondrain()` + returns `null`, no drain wait is available and the write should not be + retried. + +```mjs +import { ondrain } from 'node:stream/iter'; + +while (!writer.writeSync(chunk)) { + const drain = ondrain(writer); + if (drain === null) break; + await drain; +} +``` The bytes from each `writeSync()` / `writevSync()` / `write()` / `writev()` input chunk are copied into an internal buffer, so the caller's source From e8edeffe2d67b6b952f3f938f76187664a708c48 Mon Sep 17 00:00:00 2001 From: "Node.js GitHub Bot" Date: Tue, 15 Sep 2026 10:00:45 -0400 Subject: [PATCH 129/217] deps: update googletest to 8eff9e336692fc95961e096564f1044c600b881d PR-URL: https://github.com/nodejs/node/pull/66009 Reviewed-By: Filip Skokan Reviewed-By: Colin Ihrig Reviewed-By: Luigi Pinca --- .../include/gtest/gtest-death-test.h | 52 +--- .../include/gtest/internal/gtest-port.h | 32 +-- deps/googletest/src/gtest-internal-inl.h | 21 +- deps/googletest/src/gtest-port.cc | 243 ++---------------- deps/googletest/src/gtest.cc | 2 +- 5 files changed, 43 insertions(+), 307 deletions(-) diff --git a/deps/googletest/include/gtest/gtest-death-test.h b/deps/googletest/include/gtest/gtest-death-test.h index afd7b3a4685a..337313ea2093 100644 --- a/deps/googletest/include/gtest/gtest-death-test.h +++ b/deps/googletest/include/gtest/gtest-death-test.h @@ -105,54 +105,10 @@ GTEST_API_ bool InDeathTestChild(); // // On the regular expressions used in death tests: // -// On POSIX-compliant systems (*nix), we use the library, -// which uses the POSIX extended regex syntax. -// -// On other platforms (e.g. Windows or Mac), we only support a simple regex -// syntax implemented as part of Google Test. This limited -// implementation should be enough most of the time when writing -// death tests; though it lacks many features you can find in PCRE -// or POSIX extended regex syntax. For example, we don't support -// union ("x|y"), grouping ("(xy)"), brackets ("[xy]"), and -// repetition count ("x{5,7}"), among others. -// -// Below is the syntax that we do support. We chose it to be a -// subset of both PCRE and POSIX extended regex, so it's easy to -// learn wherever you come from. In the following: 'A' denotes a -// literal character, period (.), or a single \\ escape sequence; -// 'x' and 'y' denote regular expressions; 'm' and 'n' are for -// natural numbers. -// -// c matches any literal character c -// \\d matches any decimal digit -// \\D matches any character that's not a decimal digit -// \\f matches \f -// \\n matches \n -// \\r matches \r -// \\s matches any ASCII whitespace, including \n -// \\S matches any character that's not a whitespace -// \\t matches \t -// \\v matches \v -// \\w matches any letter, _, or decimal digit -// \\W matches any character that \\w doesn't match -// \\c matches any literal character c, which must be a punctuation -// . matches any single character except \n -// A? matches 0 or 1 occurrences of A -// A* matches 0 or many occurrences of A -// A+ matches 1 or many occurrences of A -// ^ matches the beginning of a string (not that of each line) -// $ matches the end of a string (not that of each line) -// xy matches x followed by y -// -// If you accidentally use PCRE or POSIX extended regex features -// not implemented by us, you will get a run-time failure. In that -// case, please try to rewrite your regular expression within the -// above syntax. -// -// This implementation is *not* meant to be as highly tuned or robust -// as a compiled regex library, but should perform well enough for a -// death test, which already incurs significant overhead by launching -// a child process. +// Depending on the platform, this may use RE2, the POSIX library, +// the C++11 standard library's engine with ECMAScript syntax, or +// another similar engine. Regular expressions should be simple and portable +// enough to work across the engines of interest. // // Known caveats: // diff --git a/deps/googletest/include/gtest/internal/gtest-port.h b/deps/googletest/include/gtest/internal/gtest-port.h index 154be3c16029..3b2947b852e7 100644 --- a/deps/googletest/include/gtest/internal/gtest-port.h +++ b/deps/googletest/include/gtest/internal/gtest-port.h @@ -176,7 +176,7 @@ // GTEST_USES_POSIX_RE - enhanced POSIX regex is used. Do not confuse with // GTEST_HAS_POSIX_RE (see above) which users can // define themselves. -// GTEST_USES_SIMPLE_RE - our own simple regex is used; +// GTEST_USES_STD_RE - std::regex from the C++ standard library is used; // the above RE\b(s) are mutually exclusive. // GTEST_HAS_ABSL - Google Test is compiled with Abseil. @@ -438,8 +438,9 @@ typedef struct _RTL_CRITICAL_SECTION GTEST_CRITICAL_SECTION; #include // NOLINT #define GTEST_USES_POSIX_RE 1 #else -// Use our own simple regex implementation. -#define GTEST_USES_SIMPLE_RE 1 +// Use std::regex from the C++ standard library. +#include // NOLINT +#define GTEST_USES_STD_RE 1 #endif #ifndef GTEST_HAS_EXCEPTIONS @@ -992,12 +993,11 @@ class GTEST_API_ [[nodiscard]] RE { RE2 regex_; }; -#elif defined(GTEST_USES_POSIX_RE) || defined(GTEST_USES_SIMPLE_RE) +#elif defined(GTEST_USES_POSIX_RE) || defined(GTEST_USES_STD_RE) GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ /* class A needs to have dll-interface to be used by clients of class B */) -// A simple C++ wrapper for . It uses the POSIX Extended -// Regular Expression syntax. +// A simple C++ wrapper for or . class GTEST_API_ [[nodiscard]] RE { public: // A copy constructor is required by the Standard to initialize object @@ -1037,9 +1037,9 @@ class GTEST_API_ [[nodiscard]] RE { regex_t full_regex_; // For FullMatch(). regex_t partial_regex_; // For PartialMatch(). -#else // GTEST_USES_SIMPLE_RE +#else // GTEST_USES_STD_RE - std::string full_pattern_; // For FullMatch(); + std::regex regex_; #endif }; @@ -1755,14 +1755,16 @@ class [[nodiscard]] MutexBase { #define GTEST_DECLARE_STATIC_MUTEX_(mutex) \ extern ::testing::internal::MutexBase mutex +#if defined(PTHREAD_NULL) +#define GTEST_INTERNAL_PTHREAD_NULL PTHREAD_NULL +#else +#define GTEST_INTERNAL_PTHREAD_NULL (pthread_t{}) +#endif + // Defines and statically (i.e. at link time) initializes a static mutex. -// The initialization list here does not explicitly initialize each field, -// instead relying on default initialization for the unspecified fields. In -// particular, the owner_ field (a pthread_t) is not explicitly initialized. -// This allows initialization to work whether pthread_t is a scalar or struct. -// The flag -Wmissing-field-initializers must not be specified for this to work. -#define GTEST_DEFINE_STATIC_MUTEX_(mutex) \ - ::testing::internal::MutexBase mutex = {PTHREAD_MUTEX_INITIALIZER, false, 0} +#define GTEST_DEFINE_STATIC_MUTEX_(mutex) \ + ::testing::internal::MutexBase mutex = {PTHREAD_MUTEX_INITIALIZER, false, \ + GTEST_INTERNAL_PTHREAD_NULL} // The Mutex class can only be used for mutexes created at runtime. It // shares its API with MutexBase otherwise. diff --git a/deps/googletest/src/gtest-internal-inl.h b/deps/googletest/src/gtest-internal-inl.h index 4bebca1bc656..5a6332a755ae 100644 --- a/deps/googletest/src/gtest-internal-inl.h +++ b/deps/googletest/src/gtest-internal-inl.h @@ -980,26 +980,7 @@ inline UnitTestImpl* GetUnitTestImpl() { return UnitTest::GetInstance()->impl(); } -#ifdef GTEST_USES_SIMPLE_RE - -// Internal helper functions for implementing the simple regular -// expression matcher. -GTEST_API_ bool IsInSet(char ch, const char* str); -GTEST_API_ bool IsAsciiDigit(char ch); -GTEST_API_ bool IsAsciiPunct(char ch); -GTEST_API_ bool IsRepeat(char ch); -GTEST_API_ bool IsAsciiWhiteSpace(char ch); -GTEST_API_ bool IsAsciiWordChar(char ch); -GTEST_API_ bool IsValidEscape(char ch); -GTEST_API_ bool AtomMatchesChar(bool escaped, char pattern, char ch); -GTEST_API_ bool ValidateRegex(const char* regex); -GTEST_API_ bool MatchRegexAtHead(const char* regex, const char* str); -GTEST_API_ bool MatchRepetitionAndRegexAtHead(bool escaped, char ch, - char repeat, const char* regex, - const char* str); -GTEST_API_ bool MatchRegexAnywhere(const char* regex, const char* str); - -#endif // GTEST_USES_SIMPLE_RE + // Parses the command line for Google Test flags, without initializing // other parts of Google Test. diff --git a/deps/googletest/src/gtest-port.cc b/deps/googletest/src/gtest-port.cc index be5b16e76d3e..68f77f71247b 100644 --- a/deps/googletest/src/gtest-port.cc +++ b/deps/googletest/src/gtest-port.cc @@ -766,249 +766,46 @@ void RE::Init(const char* regex) { delete[] full_pattern; } -#elif defined(GTEST_USES_SIMPLE_RE) - -// Returns true if and only if ch appears anywhere in str (excluding the -// terminating '\0' character). -bool IsInSet(char ch, const char* str) { - return ch != '\0' && strchr(str, ch) != nullptr; -} - -// Returns true if and only if ch belongs to the given classification. -// Unlike similar functions in , these aren't affected by the -// current locale. -bool IsAsciiDigit(char ch) { return '0' <= ch && ch <= '9'; } -bool IsAsciiPunct(char ch) { - return IsInSet(ch, "^-!\"#$%&'()*+,./:;<=>?@[\\]_`{|}~"); -} -bool IsRepeat(char ch) { return IsInSet(ch, "?*+"); } -bool IsAsciiWhiteSpace(char ch) { return IsInSet(ch, " \f\n\r\t\v"); } -bool IsAsciiWordChar(char ch) { - return ('a' <= ch && ch <= 'z') || ('A' <= ch && ch <= 'Z') || - ('0' <= ch && ch <= '9') || ch == '_'; -} - -// Returns true if and only if "\\c" is a supported escape sequence. -bool IsValidEscape(char c) { - return (IsAsciiPunct(c) || IsInSet(c, "dDfnrsStvwW")); -} - -// Returns true if and only if the given atom (specified by escaped and -// pattern) matches ch. The result is undefined if the atom is invalid. -bool AtomMatchesChar(bool escaped, char pattern_char, char ch) { - if (escaped) { // "\\p" where p is pattern_char. - switch (pattern_char) { - case 'd': - return IsAsciiDigit(ch); - case 'D': - return !IsAsciiDigit(ch); - case 'f': - return ch == '\f'; - case 'n': - return ch == '\n'; - case 'r': - return ch == '\r'; - case 's': - return IsAsciiWhiteSpace(ch); - case 'S': - return !IsAsciiWhiteSpace(ch); - case 't': - return ch == '\t'; - case 'v': - return ch == '\v'; - case 'w': - return IsAsciiWordChar(ch); - case 'W': - return !IsAsciiWordChar(ch); - } - return IsAsciiPunct(pattern_char) && pattern_char == ch; - } - - return (pattern_char == '.' && ch != '\n') || pattern_char == ch; -} - -// Helper function used by ValidateRegex() to format error messages. -static std::string FormatRegexSyntaxError(const char* regex, int index) { - return (Message() << "Syntax error at index " << index - << " in simple regular expression \"" << regex << "\": ") - .GetString(); -} - -// Generates non-fatal failures and returns false if regex is invalid; -// otherwise returns true. -bool ValidateRegex(const char* regex) { - if (regex == nullptr) { - ADD_FAILURE() << "NULL is not a valid simple regular expression."; - return false; - } - - bool is_valid = true; - - // True if and only if ?, *, or + can follow the previous atom. - bool prev_repeatable = false; - for (int i = 0; regex[i]; i++) { - if (regex[i] == '\\') { // An escape sequence - i++; - if (regex[i] == '\0') { - ADD_FAILURE() << FormatRegexSyntaxError(regex, i - 1) - << "'\\' cannot appear at the end."; - return false; - } - - if (!IsValidEscape(regex[i])) { - ADD_FAILURE() << FormatRegexSyntaxError(regex, i - 1) - << "invalid escape sequence \"\\" << regex[i] << "\"."; - is_valid = false; - } - prev_repeatable = true; - } else { // Not an escape sequence. - const char ch = regex[i]; - - if (ch == '^' && i > 0) { - ADD_FAILURE() << FormatRegexSyntaxError(regex, i) - << "'^' can only appear at the beginning."; - is_valid = false; - } else if (ch == '$' && regex[i + 1] != '\0') { - ADD_FAILURE() << FormatRegexSyntaxError(regex, i) - << "'$' can only appear at the end."; - is_valid = false; - } else if (IsInSet(ch, "()[]{}|")) { - ADD_FAILURE() << FormatRegexSyntaxError(regex, i) << "'" << ch - << "' is unsupported."; - is_valid = false; - } else if (IsRepeat(ch) && !prev_repeatable) { - ADD_FAILURE() << FormatRegexSyntaxError(regex, i) << "'" << ch - << "' can only follow a repeatable token."; - is_valid = false; - } - - prev_repeatable = !IsInSet(ch, "^$?*+"); - } - } - - return is_valid; -} - -// Matches a repeated regex atom followed by a valid simple regular -// expression. The regex atom is defined as c if escaped is false, -// or \c otherwise. repeat is the repetition meta character (?, *, -// or +). The behavior is undefined if str contains too many -// characters to be indexable by size_t, in which case the test will -// probably time out anyway. We are fine with this limitation as -// std::string has it too. -bool MatchRepetitionAndRegexAtHead(bool escaped, char c, char repeat, - const char* regex, const char* str) { - const size_t min_count = (repeat == '+') ? 1 : 0; - const size_t max_count = (repeat == '?') ? 1 : static_cast(-1) - 1; - // We cannot call numeric_limits::max() as it conflicts with the - // max() macro on Windows. - - for (size_t i = 0; i <= max_count; ++i) { - // We know that the atom matches each of the first i characters in str. - if (i >= min_count && MatchRegexAtHead(regex, str + i)) { - // We have enough matches at the head, and the tail matches too. - // Since we only care about *whether* the pattern matches str - // (as opposed to *how* it matches), there is no need to find a - // greedy match. - return true; - } - if (str[i] == '\0' || !AtomMatchesChar(escaped, c, str[i])) return false; - } - return false; -} - -// Returns true if and only if regex matches a prefix of str. regex must -// be a valid simple regular expression and not start with "^", or the -// result is undefined. -bool MatchRegexAtHead(const char* regex, const char* str) { - if (*regex == '\0') // An empty regex matches a prefix of anything. - return true; - - // "$" only matches the end of a string. Note that regex being - // valid guarantees that there's nothing after "$" in it. - if (*regex == '$') return *str == '\0'; - - // Is the first thing in regex an escape sequence? - const bool escaped = *regex == '\\'; - if (escaped) ++regex; - if (IsRepeat(regex[1])) { - // MatchRepetitionAndRegexAtHead() calls MatchRegexAtHead(), so - // here's an indirect recursion. It terminates as the regex gets - // shorter in each recursion. - return MatchRepetitionAndRegexAtHead(escaped, regex[0], regex[1], regex + 2, - str); - } else { - // regex isn't empty, isn't "$", and doesn't start with a - // repetition. We match the first atom of regex with the first - // character of str and recurse. - return (*str != '\0') && AtomMatchesChar(escaped, *regex, *str) && - MatchRegexAtHead(regex + 1, str + 1); - } -} - -// Returns true if and only if regex matches any substring of str. regex must -// be a valid simple regular expression, or the result is undefined. -// -// The algorithm is recursive, but the recursion depth doesn't exceed -// the regex length, so we won't need to worry about running out of -// stack space normally. In rare cases the time complexity can be -// exponential with respect to the regex length + the string length, -// but usually it's must faster (often close to linear). -bool MatchRegexAnywhere(const char* regex, const char* str) { - if (regex == nullptr || str == nullptr) return false; - - if (*regex == '^') return MatchRegexAtHead(regex + 1, str); - - // A successful match can be anywhere in str. - do { - if (MatchRegexAtHead(regex, str)) return true; - } while (*str++ != '\0'); - return false; -} - -// Implements the RE class. +#elif defined(GTEST_USES_STD_RE) RE::~RE() = default; // Returns true if and only if regular expression re matches the entire str. bool RE::FullMatch(const char* str, const RE& re) { - return re.is_valid_ && MatchRegexAnywhere(re.full_pattern_.c_str(), str); + if (!re.is_valid_ || str == nullptr) return false; + return std::regex_match(str, re.regex_); } // Returns true if and only if regular expression re matches a substring of // str (including str itself). bool RE::PartialMatch(const char* str, const RE& re) { - return re.is_valid_ && MatchRegexAnywhere(re.pattern_.c_str(), str); + if (!re.is_valid_ || str == nullptr) return false; + return std::regex_search(str, re.regex_); } // Initializes an RE from its string representation. void RE::Init(const char* regex) { - full_pattern_.clear(); - pattern_.clear(); + pattern_ = regex == nullptr ? "" : regex; + is_valid_ = false; - if (regex != nullptr) { - pattern_ = regex; - } - - is_valid_ = ValidateRegex(regex); - if (!is_valid_) { - // No need to calculate the full pattern when the regex is invalid. + if (regex == nullptr) { + ADD_FAILURE() << "NULL is not a valid regular expression."; return; } - // Reserves enough bytes to hold the regular expression used for a - // full match: we need space to prepend a '^' and append a '$'. - full_pattern_.reserve(pattern_.size() + 2); - - if (pattern_.empty() || pattern_.front() != '^') { - full_pattern_.push_back('^'); // Makes sure full_pattern_ starts with '^'. +#if GTEST_HAS_EXCEPTIONS + try { + regex_ = std::regex(regex, std::regex_constants::ECMAScript); + } catch (const std::regex_error& e) { + ADD_FAILURE() << "Regular expression \"" << regex + << "\" is not a valid regular expression: " << e.what(); + return; } +#else + regex_ = std::regex(regex, std::regex_constants::ECMAScript); +#endif - full_pattern_.append(pattern_); - - if (pattern_.empty() || pattern_.back() != '$') { - full_pattern_.push_back('$'); // Makes sure full_pattern_ ends with '$'. - } + is_valid_ = true; } #endif // GTEST_USES_POSIX_RE diff --git a/deps/googletest/src/gtest.cc b/deps/googletest/src/gtest.cc index 47c60da22916..3772f18552dc 100644 --- a/deps/googletest/src/gtest.cc +++ b/deps/googletest/src/gtest.cc @@ -4648,7 +4648,7 @@ std::string JsonUnitTestResultPrinter::EscapeJson(const std::string& str) { m << "\\r"; break; default: - if (ch < ' ') { + if (static_cast(ch) < ' ' || ch == '\x7F') { m << "\\u00" << String::FormatByte(static_cast(ch)); } else { m << ch; From 27c62235d06857fa888d2a42a384036a3b8d6731 Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Tue, 15 Sep 2026 18:06:21 +0200 Subject: [PATCH 130/217] src: ffi: create fast-call metadata Symbols lazily The FFI fast-call wrappers key per-function metadata on raw FFI functions using two per-isolate Symbols (kFastArguments / kFastBufferInvoke) that were declared in src/env_properties.h. Everything in env_properties.h is allocated while the startup snapshot is built, so each Symbol advances the isolate's identity-hash RNG before Object.prototype / Function.prototype receive their snapshot identity hashes. In the snapshot produced for Node 26.4.0+ this shifted those hashes so a function map (a function whose `length` was redefined) and a plain-object map (an object literal with an accessor) collide in V8's 64-slot NormalizedMapCache. Every store into such objects then misses the inline cache, and the repro reported in the linked issue is roughly 7x slower. Create the two Symbols lazily in the FFI binding's Initialize, on the first run of internalBinding('ffi') at runtime, instead of declaring them in env_properties.h. They are therefore not allocated during snapshot serialization and no longer bias the snapshot's prototype identity hashes. Their export, property layout, and the fast-call feature behavior are unchanged. Refs: https://github.com/nodejs/node/issues/66011 Signed-off-by: Matteo Collina Assisted-by: Pi PR-URL: https://github.com/nodejs/node/pull/66015 Reviewed-By: Paolo Insogna Reviewed-By: Xuguang Mei Reviewed-By: James M Snell --- src/env-inl.h | 34 ++++++++++++++++++++++++++++++++++ src/env.h | 20 ++++++++++++++++++++ src/env_properties.h | 2 -- src/node_ffi.cc | 17 ++++++++++++++++- 4 files changed, 70 insertions(+), 3 deletions(-) diff --git a/src/env-inl.h b/src/env-inl.h index e41afd25ffc3..a194751e8b31 100644 --- a/src/env-inl.h +++ b/src/env-inl.h @@ -866,6 +866,24 @@ void Environment::set_process_exit_handler( #undef V #undef VM + inline v8::Local IsolateData::ffi_fast_arguments_symbol() const { + return ffi_fast_arguments_symbol_.Get(isolate_); + } + inline void IsolateData::set_ffi_fast_arguments_symbol( + v8::Local value) { + CHECK(ffi_fast_arguments_symbol_.IsEmpty()); + ffi_fast_arguments_symbol_.Set(isolate_, value); + } + inline v8::Local IsolateData::ffi_fast_buffer_invoke_symbol() + const { + return ffi_fast_buffer_invoke_symbol_.Get(isolate_); + } + inline void IsolateData::set_ffi_fast_buffer_invoke_symbol( + v8::Local value) { + CHECK(ffi_fast_buffer_invoke_symbol_.IsEmpty()); + ffi_fast_buffer_invoke_symbol_.Set(isolate_, value); + } + #define VP(PropertyName, StringValue) V(v8::Private, PropertyName) #define VY(PropertyName, StringValue) V(v8::Symbol, PropertyName) #define VS(PropertyName, StringValue) V(v8::String, PropertyName) @@ -881,6 +899,22 @@ void Environment::set_process_exit_handler( #undef VY #undef VP + inline v8::Local Environment::ffi_fast_arguments_symbol() const { + return isolate_data()->ffi_fast_arguments_symbol(); + } + inline void Environment::set_ffi_fast_arguments_symbol( + v8::Local value) { + isolate_data()->set_ffi_fast_arguments_symbol(value); + } + inline v8::Local Environment::ffi_fast_buffer_invoke_symbol() + const { + return isolate_data()->ffi_fast_buffer_invoke_symbol(); + } + inline void Environment::set_ffi_fast_buffer_invoke_symbol( + v8::Local value) { + isolate_data()->set_ffi_fast_buffer_invoke_symbol(value); + } + #define V(Name, label, _, __) \ inline v8::Local Environment::Name##_permission_string() const { \ return isolate_data()->Name##_permission_string(); \ diff --git a/src/env.h b/src/env.h index 9149b91fe4bb..cb701b197df9 100644 --- a/src/env.h +++ b/src/env.h @@ -214,6 +214,17 @@ class NODE_EXTERN_PRIVATE IsolateData : public MemoryRetainer { inline v8::Local async_wrap_provider(int index) const; + // Symbols used by the FFI fast-call API to key per-function metadata on raw + // FFI functions. Kept out of env_properties.h so they are created lazily at + // runtime, not while the startup snapshot is built (allocating Symbols during + // serialization advances the isolate's identity-hash RNG, which can shift the + // snapshot hashes for Object.prototype/Function.prototype and make a function + // map and a plain-object map collide in V8's NormalizedMapCache). + inline v8::Local ffi_fast_arguments_symbol() const; + inline void set_ffi_fast_arguments_symbol(v8::Local value); + inline v8::Local ffi_fast_buffer_invoke_symbol() const; + inline void set_ffi_fast_buffer_invoke_symbol(v8::Local value); + size_t max_young_gen_size = 1; std::unordered_map> static_str_map; @@ -254,6 +265,9 @@ class NODE_EXTERN_PRIVATE IsolateData : public MemoryRetainer { PERMISSIONS(V) #undef V + v8::Eternal ffi_fast_arguments_symbol_; + v8::Eternal ffi_fast_buffer_invoke_symbol_; + // Keep a list of all Persistent strings used for AsyncWrap Provider types. std::array, AsyncWrap::PROVIDERS_LENGTH> async_wrap_providers_; @@ -963,6 +977,12 @@ class Environment final : public MemoryRetainer { #undef VY #undef VP + // Runtime-created FFI fast-call API Symbols (see IsolateData). + inline v8::Local ffi_fast_arguments_symbol() const; + inline void set_ffi_fast_arguments_symbol(v8::Local value); + inline v8::Local ffi_fast_buffer_invoke_symbol() const; + inline void set_ffi_fast_buffer_invoke_symbol(v8::Local value); + #define V(Name, label, _, __) \ inline v8::Local Name##_permission_string() const; PERMISSIONS(V) diff --git a/src/env_properties.h b/src/env_properties.h index bfb981d1c89b..70572221d020 100644 --- a/src/env_properties.h +++ b/src/env_properties.h @@ -50,8 +50,6 @@ V(ffi_sb_invoke_slow_symbol, "ffi_sb_invoke_slow_symbol") \ V(ffi_sb_arguments_symbol, "ffi_sb_arguments_symbol") \ V(ffi_sb_return_symbol, "ffi_sb_return_symbol") \ - V(ffi_fast_arguments_symbol, "ffi_fast_arguments_symbol") \ - V(ffi_fast_buffer_invoke_symbol, "ffi_fast_buffer_invoke_symbol") \ V(constructor_key_symbol, "constructor_key_symbol") \ V(handle_onclose_symbol, "handle_onclose") \ V(no_message_symbol, "no_message_symbol") \ diff --git a/src/node_ffi.cc b/src/node_ffi.cc index a596fab7c683..bde8b7c57e6c 100644 --- a/src/node_ffi.cc +++ b/src/node_ffi.cc @@ -1419,7 +1419,22 @@ static void Initialize(Local target, env->ffi_sb_return_symbol()) .Check(); // Fast API wrappers use separate metadata Symbols so pointer-conversion - // routing does not depend on SharedBuffer internals. + // routing does not depend on SharedBuffer internals. These are created here + // (at runtime, on first `internalBinding('ffi')`) instead of being declared + // in env_properties.h, so they are not allocated while the startup snapshot + // is being built. Allocating Symbols during snapshot serialization advances + // the isolate's identity-hash RNG and shifts the identity hashes baked into + // the snapshot for Object.prototype / Function.prototype, which can make a + // function map and a plain-object map collide in V8's NormalizedMapCache. + if (env->ffi_fast_arguments_symbol().IsEmpty()) { + env->set_ffi_fast_arguments_symbol(v8::Symbol::New( + isolate, FIXED_ONE_BYTE_STRING(isolate, "ffi_fast_arguments_symbol"))); + } + if (env->ffi_fast_buffer_invoke_symbol().IsEmpty()) { + env->set_ffi_fast_buffer_invoke_symbol(v8::Symbol::New( + isolate, + FIXED_ONE_BYTE_STRING(isolate, "ffi_fast_buffer_invoke_symbol"))); + } target ->Set(context, FIXED_ONE_BYTE_STRING(isolate, "kFastArguments"), From 1bbc5488a5db91d86a2f9ecb8dc1e5e2d27c0a8c Mon Sep 17 00:00:00 2001 From: Philipp Dunkel Date: Tue, 15 Sep 2026 18:06:32 +0200 Subject: [PATCH 131/217] vfs: close the fs hook gaps for mounted paths Several `node:fs` entry points behave differently for a mounted path than for a real one, because of how the call reaches the VFS hooks. Make them behave as they do for a real path: * Add the `watchFile`, `unwatchFile` and `promisesWatch` handlers, backed by the provider's stat watcher and async watcher; those calls threw a TypeError before. Have `watch` refuse a path that does not exist with ENOENT instead of handing back a watcher that polls forever and keeps the process alive. * Convert timestamps and validate arguments before the hook runs in `utimes`, `lutimes` and `readdir` (sync, callback and promise forms), so a mounted path gets the same ERR_INVALID_ARG_* errors and the same seconds-since-epoch numbers as a real one. * Pass the mode and times through to the `fchmod` and `futimes` hooks and route them to the handle's entry, so descriptor operations take effect like their path forms instead of being no-ops; the memory handle validates the way a FileHandle would since one calls it directly. * Treat a `mkdtemp` prefix as text rather than a path when it ends in a separator, so the directory is created inside the intended parent. * Map the first directory a recursive `mkdir` created back under the mount point instead of returning the provider-relative path. * Make disposing an already closed virtual `Dir` a no-op, as on the native `Dir`, instead of rejecting with ERR_DIR_CLOSED. test-vfs-fs-hook-gaps adds a test per gap, stating the real-fs outcome as the expectation. The existing file handle test asserted that `chmod()` and `utimes()` without arguments were no-ops; they now validate and apply, so it exercises that instead. Signed-off-by: Philipp Dunkel PR-URL: https://github.com/nodejs/node/pull/65852 Reviewed-By: James M Snell Reviewed-By: Matteo Collina Reviewed-By: Trivikram Kamat Reviewed-By: Filip Skokan --- lib/fs.js | 75 ++++++------- lib/internal/fs/promises.js | 31 +++--- lib/internal/vfs/dir.js | 7 +- lib/internal/vfs/file_handle.js | 54 ++++++++- lib/internal/vfs/file_system.js | 48 +++++--- lib/internal/vfs/setup.js | 56 +++++++++- test/parallel/test-vfs-file-handle.js | 11 +- test/parallel/test-vfs-fs-matches-real-fs.js | 111 +++++++++++++++++++ 8 files changed, 304 insertions(+), 89 deletions(-) create mode 100644 test/parallel/test-vfs-fs-matches-real-fs.js diff --git a/lib/fs.js b/lib/fs.js index d53120af9bb3..c013b8732746 100644 --- a/lib/fs.js +++ b/lib/fs.js @@ -1864,9 +1864,6 @@ function readdir(path, options, callback) { options = undefined; } - const h = vfsState.handlers; - if (h !== null && vfsResult(h.readdir(path, options), callback)) return; - callback = makeCallback(callback); options = getOptions(options); path = getValidatedPath(path); @@ -1874,6 +1871,9 @@ function readdir(path, options, callback) { validateBoolean(options.recursive, 'options.recursive'); } + const h = vfsState.handlers; + if (h !== null && vfsResult(h.readdir(path, options), callback)) return; + if (options.recursive) { readdirRecursive(path, options, callback); return; @@ -1910,17 +1910,18 @@ function readdir(path, options, callback) { * @returns {string | Buffer[] | Dirent[]} */ function readdirSync(path, options) { - const h = vfsState.handlers; - if (h !== null) { - const result = h.readdirSync(path, options); - if (result !== undefined) return result; - } options = getOptions(options); path = getValidatedPath(path); if (options.recursive != null) { validateBoolean(options.recursive, 'options.recursive'); } + const h = vfsState.handlers; + if (h !== null) { + const result = h.readdirSync(path, options); + if (result !== undefined) return result; + } + if (options.recursive) { return readdirSyncRecursive(path, options); } @@ -2423,7 +2424,7 @@ function fchmod(fd, mode, callback) { callback = makeCallback(callback); const h = vfsState.handlers; - if (h !== null && vfsVoid(h.fchmod(fd), callback)) return; + if (h !== null && vfsVoid(h.fchmod(fd, mode), callback)) return; if (permission.isEnabled()) { callback(new ERR_ACCESS_DENIED('fchmod API is disabled when Permission Model is enabled.')); @@ -2442,19 +2443,18 @@ function fchmod(fd, mode, callback) { * @returns {void} */ function fchmodSync(fd, mode) { + mode = parseFileMode(mode, 'mode'); + const h = vfsState.handlers; if (h !== null) { - const result = h.fchmodSync(fd); + const result = h.fchmodSync(fd, mode); if (result !== undefined) return; } if (permission.isEnabled()) { throw new ERR_ACCESS_DENIED('fchmod API is disabled when Permission Model is enabled.'); } - binding.fchmod( - fd, - parseFileMode(mode, 'mode'), - ); + binding.fchmod(fd, mode); } /** @@ -2693,18 +2693,15 @@ function chownSync(path, uid, gid) { function utimes(path, atime, mtime, callback) { callback = makeCallback(callback); path = getValidatedPath(path); + atime = toUnixTimestamp(atime); + mtime = toUnixTimestamp(mtime); const h = vfsState.handlers; if (h !== null && vfsVoid(h.utimes(path, atime, mtime), callback)) return; const req = new FSReqCallback(); req.oncomplete = callback; - binding.utimes( - path, - toUnixTimestamp(atime), - toUnixTimestamp(mtime), - req, - ); + binding.utimes(path, atime, mtime, req); } /** @@ -2717,6 +2714,8 @@ function utimes(path, atime, mtime, callback) { */ function utimesSync(path, atime, mtime) { path = getValidatedPath(path); + atime = toUnixTimestamp(atime); + mtime = toUnixTimestamp(mtime); const h = vfsState.handlers; if (h !== null) { @@ -2724,11 +2723,7 @@ function utimesSync(path, atime, mtime) { if (result !== undefined) return; } - binding.utimes( - path, - toUnixTimestamp(atime), - toUnixTimestamp(mtime), - ); + binding.utimes(path, atime, mtime); } /** @@ -2746,7 +2741,7 @@ function futimes(fd, atime, mtime, callback) { callback = makeCallback(callback); const h = vfsState.handlers; - if (h !== null && vfsVoid(h.futimes(fd), callback)) return; + if (h !== null && vfsVoid(h.futimes(fd, atime, mtime), callback)) return; if (permission.isEnabled()) { callback(new ERR_ACCESS_DENIED('futimes API is disabled when Permission Model is enabled.')); @@ -2768,9 +2763,12 @@ function futimes(fd, atime, mtime, callback) { * @returns {void} */ function futimesSync(fd, atime, mtime) { + atime = toUnixTimestamp(atime, 'atime'); + mtime = toUnixTimestamp(mtime, 'mtime'); + const h = vfsState.handlers; if (h !== null) { - const result = h.futimesSync(fd); + const result = h.futimesSync(fd, atime, mtime); if (result !== undefined) return; } @@ -2778,11 +2776,7 @@ function futimesSync(fd, atime, mtime) { throw new ERR_ACCESS_DENIED('futimes API is disabled when Permission Model is enabled.'); } - binding.futimes( - fd, - toUnixTimestamp(atime, 'atime'), - toUnixTimestamp(mtime, 'mtime'), - ); + binding.futimes(fd, atime, mtime); } /** @@ -2797,18 +2791,15 @@ function futimesSync(fd, atime, mtime) { function lutimes(path, atime, mtime, callback) { callback = makeCallback(callback); path = getValidatedPath(path); + atime = toUnixTimestamp(atime); + mtime = toUnixTimestamp(mtime); const h = vfsState.handlers; if (h !== null && vfsVoid(h.lutimes(path, atime, mtime), callback)) return; const req = new FSReqCallback(); req.oncomplete = callback; - binding.lutimes( - path, - toUnixTimestamp(atime), - toUnixTimestamp(mtime), - req, - ); + binding.lutimes(path, atime, mtime, req); } /** @@ -2821,6 +2812,8 @@ function lutimes(path, atime, mtime, callback) { */ function lutimesSync(path, atime, mtime) { path = getValidatedPath(path); + atime = toUnixTimestamp(atime); + mtime = toUnixTimestamp(mtime); const h = vfsState.handlers; if (h !== null) { @@ -2828,11 +2821,7 @@ function lutimesSync(path, atime, mtime) { if (result !== undefined) return; } - binding.lutimes( - path, - toUnixTimestamp(atime), - toUnixTimestamp(mtime), - ); + binding.lutimes(path, atime, mtime); } function writeAll(fd, isUserFd, buffer, offset, length, signal, flush, callback) { diff --git a/lib/internal/fs/promises.js b/lib/internal/fs/promises.js index 0e9a16ce83a9..4487ea778fb3 100644 --- a/lib/internal/fs/promises.js +++ b/lib/internal/fs/promises.js @@ -1725,17 +1725,18 @@ async function readdirRecursiveWithPermissionModel(basePath, options) { } async function readdir(path, options) { - const h = vfsState.handlers; - if (h !== null) { - const promise = h.readdir(path, options); - if (promise !== undefined) return await promise; - } options = getOptions(options); // Make shallow copy to prevent mutating options from affecting results options = copyObject(options); path = getValidatedPath(path); + + const h = vfsState.handlers; + if (h !== null) { + const promise = h.readdir(path, options); + if (promise !== undefined) return await promise; + } if (options.recursive) { return readdirRecursive(path, options); } @@ -2011,6 +2012,8 @@ async function chown(path, uid, gid) { async function utimes(path, atime, mtime) { path = getValidatedPath(path); + atime = toUnixTimestamp(atime); + mtime = toUnixTimestamp(mtime); const h = vfsState.handlers; if (h !== null) { @@ -2019,12 +2022,7 @@ async function utimes(path, atime, mtime) { } return await PromisePrototypeThen( - binding.utimes( - path, - toUnixTimestamp(atime), - toUnixTimestamp(mtime), - kUsePromises, - ), + binding.utimes(path, atime, mtime, kUsePromises), undefined, handleErrorFromBinding, ); @@ -2044,6 +2042,10 @@ async function futimes(handle, atime, mtime) { } async function lutimes(path, atime, mtime) { + path = getValidatedPath(path); + atime = toUnixTimestamp(atime); + mtime = toUnixTimestamp(mtime); + const h = vfsState.handlers; if (h !== null) { const promise = h.lutimes(path, atime, mtime); @@ -2051,12 +2053,7 @@ async function lutimes(path, atime, mtime) { } return await PromisePrototypeThen( - binding.lutimes( - getValidatedPath(path), - toUnixTimestamp(atime), - toUnixTimestamp(mtime), - kUsePromises, - ), + binding.lutimes(path, atime, mtime, kUsePromises), undefined, handleErrorFromBinding, ); diff --git a/lib/internal/vfs/dir.js b/lib/internal/vfs/dir.js index 803aeb404531..3b0a6140b1ee 100644 --- a/lib/internal/vfs/dir.js +++ b/lib/internal/vfs/dir.js @@ -94,10 +94,15 @@ class VirtualDir { this.closeSync(); } } + + async [SymbolAsyncDispose]() { + if (!this.#closed) { + this.closeSync(); + } + } } VirtualDir.prototype[SymbolAsyncIterator] = VirtualDir.prototype.entries; -VirtualDir.prototype[SymbolAsyncDispose] = VirtualDir.prototype.close; module.exports = { VirtualDir, diff --git a/lib/internal/vfs/file_handle.js b/lib/internal/vfs/file_handle.js index a65fe5f99386..4d28abde535e 100644 --- a/lib/internal/vfs/file_handle.js +++ b/lib/internal/vfs/file_handle.js @@ -20,6 +20,8 @@ const { const { createEBADF, } = require('internal/vfs/errors'); +const { stringToFlags, toUnixTimestamp } = require('internal/fs/utils'); +const { parseFileMode } = require('internal/validators'); // Private symbols const kPath = Symbol('kPath'); @@ -29,7 +31,6 @@ const kPosition = Symbol('kPosition'); const kClosed = Symbol('kClosed'); const kAccess = Symbol('kAccess'); -const { stringToFlags } = require('internal/fs/utils'); const { fs: { O_APPEND, O_CREAT, O_EXCL, O_RDONLY, O_RDWR, O_TRUNC, O_WRONLY }, } = internalBinding('constants'); @@ -288,10 +289,17 @@ class VirtualFileHandle { } /** - * No-op chmod - VFS files don't have real permissions. + * @param {number} mode The new permission bits + */ + chmodSync(mode) {} + + /** + * @param {number} mode The new permission bits * @returns {Promise} */ - async chmod() {} + async chmod(mode) { + this.chmodSync(mode); + } /** * No-op chown - VFS files don't have real ownership. @@ -300,10 +308,19 @@ class VirtualFileHandle { async chown() {} /** - * No-op utimes - timestamps are handled by the provider. + * @param {Date|number|string} atime The new access time + * @param {Date|number|string} mtime The new modification time + */ + utimesSync(atime, mtime) {} + + /** + * @param {Date|number|string} atime The new access time + * @param {Date|number|string} mtime The new modification time * @returns {Promise} */ - async utimes() {} + async utimes(atime, mtime) { + this.utimesSync(atime, mtime); + } /** * No-op datasync - VFS is in-memory. @@ -681,6 +698,33 @@ class MemoryFileHandle extends VirtualFileHandle { throw new ERR_INVALID_STATE('stats not available'); } + /** + * @param {number} mode The new permission bits + */ + chmodSync(mode) { + this.#checkClosed('fchmod'); + mode = parseFileMode(mode, 'mode'); + if (this.#entry) { + this.#entry.mode = (this.#entry.mode & ~0o7777) | (mode & 0o7777); + this.#entry.ctime = DateNow(); + } + } + + /** + * @param {Date|number|string} atime The new access time + * @param {Date|number|string} mtime The new modification time + */ + utimesSync(atime, mtime) { + this.#checkClosed('futimes'); + const atimeMs = toUnixTimestamp(atime, 'atime') * 1000; + const mtimeMs = toUnixTimestamp(mtime, 'mtime') * 1000; + if (this.#entry) { + this.#entry.atime = atimeMs; + this.#entry.mtime = mtimeMs; + this.#entry.ctime = DateNow(); + } + } + /** * Gets file stats. * @param {object} [options] Options diff --git a/lib/internal/vfs/file_system.js b/lib/internal/vfs/file_system.js index 574c076c426d..afb5fab3eb73 100644 --- a/lib/internal/vfs/file_system.js +++ b/lib/internal/vfs/file_system.js @@ -63,6 +63,17 @@ function normalizeMountedPath(inputPath) { return toNamespacedPath(resolvePath(inputPath)); } +const kTempChars = + 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; + +function randomSuffix() { + let suffix = ''; + for (let i = 0; i < 6; i++) { + suffix += kTempChars[(MathRandom() * kTempChars.length) | 0]; + } + return suffix; +} + let registerVFS; let deregisterVFS; @@ -359,7 +370,8 @@ class VirtualFileSystem { */ mkdirSync(dirPath, options) { const providerPath = this.#toProviderPath(dirPath); - return this[kProvider].mkdirSync(providerPath, options); + const created = this[kProvider].mkdirSync(providerPath, options); + return created === undefined ? undefined : this.#toMountedPath(created); } /** @@ -557,17 +569,25 @@ class VirtualFileSystem { * @returns {string} The full path of the created directory */ mkdtempSync(prefix) { - const providerPrefix = this.#toProviderPath(prefix); - const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; - let suffix = ''; - for (let i = 0; i < 6; i++) { - suffix += chars[(MathRandom() * chars.length) | 0]; - } - const dirPath = providerPrefix + suffix; + const dirPath = this.#toProviderPrefix(prefix) + randomSuffix(); this[kProvider].mkdirSync(dirPath); return this.#toMountedPath(dirPath); } + /** + * Converts a mkdtemp prefix to a provider-relative one, keeping a + * trailing separator. + * @param {string} prefix The mounted prefix + * @returns {string} + */ + #toProviderPrefix(prefix) { + const last = prefix[prefix.length - 1]; + const trailing = last === '/' || last === sep; + const providerPrefix = this.#toProviderPath(prefix); + if (!trailing) return providerPrefix; + return providerPrefix === '/' ? '/' : `${providerPrefix}/`; + } + /** * Opens a directory synchronously. * @param {string} dirPath The directory path @@ -1106,6 +1126,7 @@ class VirtualFileSystem { // Arrow functions capture `this` for private method access. const toProviderPath = (p) => this.#toProviderPath(p); + const toProviderPrefix = (p) => this.#toProviderPrefix(p); const toMountedPath = (p) => this.#toMountedPath(p); return ObjectFreeze({ @@ -1141,7 +1162,8 @@ class VirtualFileSystem { async mkdir(dirPath, options) { const providerPath = toProviderPath(dirPath); - return provider.mkdir(providerPath, options); + const created = await provider.mkdir(providerPath, options); + return created === undefined ? undefined : toMountedPath(created); }, async rmdir(dirPath) { @@ -1235,13 +1257,7 @@ class VirtualFileSystem { }, async mkdtemp(prefix) { - const providerPrefix = toProviderPath(prefix); - const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; - let suffix = ''; - for (let i = 0; i < 6; i++) { - suffix += chars[(MathRandom() * chars.length) | 0]; - } - const dirPath = providerPrefix + suffix; + const dirPath = toProviderPrefix(prefix) + randomSuffix(); await provider.mkdir(dirPath); return toMountedPath(dirPath); }, diff --git a/lib/internal/vfs/setup.js b/lib/internal/vfs/setup.js index 3d624d5af293..a589883bf4ad 100644 --- a/lib/internal/vfs/setup.js +++ b/lib/internal/vfs/setup.js @@ -534,9 +534,17 @@ function createVfsHandlers() { if (vfd) { vfd.entry.truncateSync(len); return true; } return undefined; }, - fchmodSync: noopFdSync, + fchmodSync(fd, mode) { + const vfd = getVirtualFd(fd); + if (vfd) { vfd.entry.chmodSync(mode); return true; } + return undefined; + }, fchownSync: noopFdSync, - futimesSync: noopFdSync, + futimesSync(fd, atime, mtime) { + const vfd = getVirtualFd(fd); + if (vfd) { vfd.entry.utimesSync(atime, mtime); return true; } + return undefined; + }, fdatasyncSync: noopFdSync, fsyncSync: noopFdSync, readvSync(fd, buffers, position) { @@ -593,9 +601,17 @@ function createVfsHandlers() { if (!vfd) return undefined; return vfd.entry.truncate(len).then(() => true); }, - fchmod: noopFd, + fchmod(fd, mode) { + const vfd = getVirtualFd(fd); + if (!vfd) return undefined; + return vfd.entry.chmod(mode).then(() => true); + }, fchown: noopFd, - futimes: noopFd, + futimes(fd, atime, mtime) { + const vfd = getVirtualFd(fd); + if (!vfd) return undefined; + return vfd.entry.utimes(atime, mtime).then(() => true); + }, fdatasync: noopFd, fsync: noopFd, @@ -628,10 +644,40 @@ function createVfsHandlers() { const pathStr = toPathStr(filename); if (pathStr !== null) { const r = findVFSForPath(pathStr); - if (r !== null) return r.vfs.watch(pathStr, options, listener); + if (r !== null) { + if (!r.vfs.existsSync(pathStr)) throw createENOENT('watch', pathStr); + return r.vfs.watch(pathStr, options, listener); + } } return undefined; }, + watchFile(filename, options, listener) { + const pathStr = toPathStr(filename); + if (pathStr === null) return undefined; + const r = findVFSForPath(pathStr); + if (r === null) return undefined; + if (options === null || typeof options !== 'object') { + listener = options; + options = kEmptyObject; + } + return r.vfs.watchFile(pathStr, options, listener); + }, + unwatchFile(filename, listener) { + const pathStr = toPathStr(filename); + if (pathStr === null) return undefined; + const r = findVFSForPath(pathStr); + if (r === null) return undefined; + r.vfs.unwatchFile(pathStr, listener); + return true; + }, + promisesWatch(filename, options) { + const pathStr = toPathStr(filename); + if (pathStr === null) return undefined; + const r = findVFSForPath(pathStr); + if (r === null) return undefined; + if (!r.vfs.existsSync(pathStr)) throw createENOENT('watch', pathStr); + return r.vfs.promises.watch(pathStr, options); + }, readdir(path, options) { const promise = vfsOp(path, (vfs, n) => vfs.promises.readdir(n, options)); diff --git a/test/parallel/test-vfs-file-handle.js b/test/parallel/test-vfs-file-handle.js index d9d919446b8e..4b86714fbc53 100644 --- a/test/parallel/test-vfs-file-handle.js +++ b/test/parallel/test-vfs-file-handle.js @@ -42,10 +42,17 @@ myVfs.writeFileSync('/file.txt', 'hello world'); assert.strictEqual(b1.toString(), 'hello'); assert.strictEqual(b2.toString(), ' world'); + // Metadata methods reach the entry the way fchmod(2)/futimes(2) do, and + // validate their arguments the way a FileHandle would. + await handle.chmod(0o600); + assert.strictEqual((await handle.stat()).mode & 0o777, 0o600); + await handle.utimes(1000, 2000); + assert.strictEqual((await handle.stat()).mtimeMs, 2000 * 1000); + await assert.rejects(handle.chmod(), { code: 'ERR_INVALID_ARG_TYPE' }); + await assert.rejects(handle.utimes(), { code: 'ERR_INVALID_ARG_TYPE' }); + // no-op metadata methods - await handle.chmod(); await handle.chown(); - await handle.utimes(); await handle.datasync(); await handle.sync(); diff --git a/test/parallel/test-vfs-fs-matches-real-fs.js b/test/parallel/test-vfs-fs-matches-real-fs.js new file mode 100644 index 000000000000..a500d796599d --- /dev/null +++ b/test/parallel/test-vfs-fs-matches-real-fs.js @@ -0,0 +1,111 @@ +// Flags: --experimental-vfs +'use strict'; + +// `node:fs` entry points route mounted paths through the VFS hooks. Where a +// hook is missing, runs before argument validation, or ignores the +// descriptor form of an operation, the same call behaves differently from a +// real path. Each case states the real-fs outcome as the expectation. Cases +// are independent so the runner reports each one. + +const common = require('../common'); +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const vfs = require('node:vfs'); +const { test } = require('node:test'); + +function mount(populate) { + const layer = vfs.create(); + populate?.(layer); + return layer.mount(); +} + +test('watchFile on a mounted path installs a stat watcher', () => { + const file = path.join(mount((l) => l.writeFileSync('/f', 'x')), 'f'); + fs.watchFile(file, { interval: 10 }, common.mustNotCall()); + fs.unwatchFile(file); +}); + +test('fs.promises.watch on a mounted directory yields events', async () => { + const dir = path.join(mount((l) => l.mkdirSync('/d')), 'd'); + const ac = new AbortController(); + const watcher = fs.promises.watch(dir, { signal: ac.signal }); + setTimeout(() => fs.writeFileSync(path.join(dir, 'x'), '1'), 20); + for await (const event of watcher) { + assert.strictEqual(event.filename, 'x'); + ac.abort(); + break; + } +}); + +test('watch on a missing mounted path throws ENOENT', () => { + const dir = mount(); + // Should the call return a watcher instead, it polls forever, so it is + // closed to let the process exit. + let watcher; + try { + assert.throws(() => { watcher = fs.watch(path.join(dir, 'nope')); }, + { code: 'ENOENT' }); + } finally { + watcher?.close(); + } +}); + +test('utimesSync accepts numeric strings as seconds', () => { + const file = path.join(mount((l) => l.writeFileSync('/f', 'x')), 'f'); + fs.utimesSync(file, '1000', '2000'); + assert.strictEqual(fs.statSync(file).mtimeMs, 2000 * 1000); +}); + +test('utimesSync rejects an invalid time argument', () => { + const file = path.join(mount((l) => l.writeFileSync('/f', 'x')), 'f'); + assert.throws(() => fs.utimesSync(file, {}, {}), { code: 'ERR_INVALID_ARG_TYPE' }); +}); + +test('readdirSync rejects an invalid encoding', () => { + const dir = mount(); + assert.throws(() => fs.readdirSync(dir, { encoding: 'nope' }), + { code: 'ERR_INVALID_ARG_VALUE' }); +}); + +test('futimesSync updates the timestamps through a descriptor', () => { + const file = path.join(mount((l) => l.writeFileSync('/f', 'x')), 'f'); + const fd = fs.openSync(file, 'r+'); + try { + fs.futimesSync(fd, 1000, 2000); + } finally { + fs.closeSync(fd); + } + assert.strictEqual(fs.statSync(file).mtimeMs, 2000 * 1000); +}); + +test('fchmodSync changes the mode through a descriptor', () => { + const file = path.join(mount((l) => l.writeFileSync('/f', 'x')), 'f'); + const fd = fs.openSync(file, 'r+'); + try { + fs.fchmodSync(fd, 0o600); + } finally { + fs.closeSync(fd); + } + assert.strictEqual(fs.statSync(file).mode & 0o777, 0o600); +}); + +test('mkdtempSync with a trailing separator creates the directory inside the prefix', () => { + const dir = path.join(mount((l) => l.mkdirSync('/dir')), 'dir'); + const created = fs.mkdtempSync(dir + path.sep); + assert.ok(created.startsWith(dir + path.sep), `${created} is not inside ${dir}`); + assert.strictEqual(fs.statSync(created).isDirectory(), true); +}); + +test('mkdirSync({ recursive: true }) returns the first directory created', () => { + const dir = mount(); + const created = fs.mkdirSync(path.join(dir, 'a', 'b'), { recursive: true }); + assert.strictEqual(created, path.join(dir, 'a')); +}); + +test('a closed Dir can be disposed asynchronously', async () => { + const dir = mount((l) => l.mkdirSync('/d')); + const handle = fs.opendirSync(dir); + handle.closeSync(); + await handle[Symbol.asyncDispose](); +}); From 8a27474044b0b70824b4cb793869fb9ba922ae4e Mon Sep 17 00:00:00 2001 From: Abhinandan Kumar <181508976+abhi128nandan@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:13:38 +0530 Subject: [PATCH 132/217] test: cover cpSync fast path timestamp preservation Add test coverage for the native cpSync fast path to ensure it matches the existing timestamp preservation behavior. Signed-off-by: Abhinandan Kumar PR-URL: https://github.com/nodejs/node/pull/65678 Reviewed-By: LiviaMedeiros --- ...est-fs-cp-sync-preserve-timestamps-dir.mjs | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/test/parallel/test-fs-cp-sync-preserve-timestamps-dir.mjs b/test/parallel/test-fs-cp-sync-preserve-timestamps-dir.mjs index 20d889075988..06d1c3521478 100644 --- a/test/parallel/test-fs-cp-sync-preserve-timestamps-dir.mjs +++ b/test/parallel/test-fs-cp-sync-preserve-timestamps-dir.mjs @@ -1,5 +1,6 @@ -// This tests that cpSync with a filter preserves directory timestamps -// when preserveTimestamps is true. +// This tests that cpSync preserves directory timestamps +// when preserveTimestamps is true, both on the JS fallback path (with filter) +// and the native fast path (without filter). import '../common/index.mjs'; import { nextdir } from '../common/fs.js'; import assert from 'node:assert'; @@ -40,3 +41,21 @@ assert.strictEqual(srcDirStat.mtime.getTime(), destDirStat.mtime.getTime()); const srcRootStat = statSync(src); const destRootStat = statSync(dest); assert.strictEqual(srcRootStat.mtime.getTime(), destRootStat.mtime.getTime()); + +// Copy with preserveTimestamps and NO filter (to exercise the native fast path). +const destFast = nextdir(); +cpSync(src, destFast, { + recursive: true, + preserveTimestamps: true, +}); + +// Verify file timestamps are preserved. +const destFastFileStat = statSync(join(destFast, 'subdir', 'file.txt')); +assert.strictEqual(srcFileStat.mtime.getTime(), destFastFileStat.mtime.getTime()); + +// Verify directory timestamps are preserved. +const destFastDirStat = statSync(join(destFast, 'subdir')); +assert.strictEqual(srcDirStat.mtime.getTime(), destFastDirStat.mtime.getTime()); + +const destFastRootStat = statSync(destFast); +assert.strictEqual(srcRootStat.mtime.getTime(), destFastRootStat.mtime.getTime()); From ee29c56993f3a9f58db80b1dfb4b52c27249fdad Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Tue, 15 Sep 2026 21:11:35 +0200 Subject: [PATCH 133/217] test: prevent parser reuse across close scenarios MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both cases replace parser cleanup methods. Faster socket cleanup can return a modified parser to the shared pool and close it before the other request uses it. Run the immediate and deferred close cases in separate test files so each gets its own process and parser pool. Preserve both cleanup paths and all call-count assertions. Signed-off-by: Filip Skokan Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/66017 Reviewed-By: James M Snell Reviewed-By: Xuguang Mei Reviewed-By: Ulises Gascón Reviewed-By: Luigi Pinca --- ...ver-connection-list-when-close-deferred.js | 35 ++++++++++++++++ ...-http-server-connection-list-when-close.js | 42 +++++-------------- 2 files changed, 45 insertions(+), 32 deletions(-) create mode 100644 test/parallel/test-http-server-connection-list-when-close-deferred.js diff --git a/test/parallel/test-http-server-connection-list-when-close-deferred.js b/test/parallel/test-http-server-connection-list-when-close-deferred.js new file mode 100644 index 000000000000..af87e3561618 --- /dev/null +++ b/test/parallel/test-http-server-connection-list-when-close-deferred.js @@ -0,0 +1,35 @@ +'use strict'; + +const common = require('../common'); +const http = require('http'); + +// Keep this case in a separate process from the immediate-close case so +// their modified parsers cannot be reused across cases. + +function request(server) { + http.get({ + agent: false, + port: server.address().port, + path: '/', + }, (res) => { + res.resume(); + }); +} + +const server = http.createServer(common.mustCallAtLeast((req, res) => { + // See `freeParser` in _http_common.js + const { parser } = req.socket; + parser.free = common.mustCall(() => { + setImmediate(common.mustCall(() => { + parser.close(); + })); + }); + req.socket.on('close', common.mustCall(() => { + setImmediate(common.mustCall(() => { + server.close(); + })); + })); + res.end('ok'); +})).listen(0, common.mustCall(() => { + request(server); +})); diff --git a/test/parallel/test-http-server-connection-list-when-close.js b/test/parallel/test-http-server-connection-list-when-close.js index 0c8308b63c53..305755b14eb3 100644 --- a/test/parallel/test-http-server-connection-list-when-close.js +++ b/test/parallel/test-http-server-connection-list-when-close.js @@ -13,36 +13,14 @@ function request(server) { }); } -{ - const server = http.createServer(common.mustCallAtLeast((req, res) => { - // Hack to not remove parser out of server.connectionList - // See `freeParser` in _http_common.js - req.socket.parser.free = common.mustCall(); - req.socket.on('close', common.mustCall(() => { - server.close(); - })); - res.end('ok'); - })).listen(0, common.mustCall(() => { - request(server); +const server = http.createServer(common.mustCallAtLeast((req, res) => { + // Hack to not remove parser out of server.connectionList + // See `freeParser` in _http_common.js + req.socket.parser.free = common.mustCall(); + req.socket.on('close', common.mustCall(() => { + server.close(); })); -} - -{ - const server = http.createServer(common.mustCallAtLeast((req, res) => { - // See `freeParser` in _http_common.js - const { parser } = req.socket; - parser.free = common.mustCall(() => { - setImmediate(common.mustCall(() => { - parser.close(); - })); - }); - req.socket.on('close', common.mustCall(() => { - setImmediate(common.mustCall(() => { - server.close(); - })); - })); - res.end('ok'); - })).listen(0, common.mustCall(() => { - request(server); - })); -} + res.end('ok'); +})).listen(0, common.mustCall(() => { + request(server); +})); From cce2795ad79082f491868a854a36fd7f245caf26 Mon Sep 17 00:00:00 2001 From: Yuya Inoue <65857152+inoway46@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:50:14 +0900 Subject: [PATCH 134/217] lib: fix AbortSignal.any() abort propagation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow composite signal sources at construction so abort state is propagated even without listeners, before sources can be collected. Keep the existing weak-reference finalization and timeout cleanup. Cover source collection before each state accessor, including nested composites, and cleanup of unreachable listener-less dependents. Fixes: https://github.com/nodejs/node/issues/65995 Refs: https://github.com/nodejs/node/issues/62363 Assisted-by: Codex Signed-off-by: inoway46 PR-URL: https://github.com/nodejs/node/pull/66014 Reviewed-By: Robert Nagy Reviewed-By: James M Snell Reviewed-By: Xuguang Mei Reviewed-By: Ulises Gascón Reviewed-By: Benjamin Gruenbaum --- lib/internal/abort_controller.js | 4 +++ .../test-abortsignal-any-source-gc.mjs | 31 +++++++++++++++++ .../test-abortsignal-drop-settled-signals.mjs | 34 ++++++++++++++++--- 3 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 test/parallel/test-abortsignal-any-source-gc.mjs diff --git a/lib/internal/abort_controller.js b/lib/internal/abort_controller.js index ef30d9619749..55be0b6f3669 100644 --- a/lib/internal/abort_controller.js +++ b/lib/internal/abort_controller.js @@ -381,6 +381,10 @@ class AbortSignal extends EventTarget { resultSignal[kTimeout] = true; } + // Preserve abort state even if sources are collected before this signal is + // observed. Following uses weak references so unused signals can be collected. + followCompositeSignal(resultSignal); + return resultSignal; } diff --git a/test/parallel/test-abortsignal-any-source-gc.mjs b/test/parallel/test-abortsignal-any-source-gc.mjs new file mode 100644 index 000000000000..06fd0c0e29f4 --- /dev/null +++ b/test/parallel/test-abortsignal-any-source-gc.mjs @@ -0,0 +1,31 @@ +// Flags: --expose-gc + +import '../common/index.mjs'; +import { gcUntil } from '../common/gc.js'; +import assert from 'node:assert/strict'; +import { it } from 'node:test'; + +for (const nested of [false, true]) { + for (const accessor of ['aborted', 'reason', 'throwIfAborted']) { + it(`preserves ${accessor} after source GC (nested: ${nested})`, async () => { + let controller = new AbortController(); + const sourceRef = new WeakRef(controller.signal); + let signal = AbortSignal.any([controller.signal]); + if (nested) signal = AbortSignal.any([signal]); + const reason = { message: 'stop' }; + + controller.abort(reason); + controller = null; + + // Do not observe the composite or attach a listener before source GC. + await gcUntil('source signal is collected', () => sourceRef.deref() === undefined); + + // Exercise each entry point before any other accessor can refresh state. + if (accessor === 'aborted') assert.strictEqual(signal.aborted, true); + if (accessor === 'reason') assert.strictEqual(signal.reason, reason); + assert.throws(() => signal.throwIfAborted(), (err) => err === reason); + assert.strictEqual(signal.aborted, true); + assert.strictEqual(signal.reason, reason); + }); + } +} diff --git a/test/parallel/test-abortsignal-drop-settled-signals.mjs b/test/parallel/test-abortsignal-drop-settled-signals.mjs index 224d65abc70f..d4c81a7165d4 100644 --- a/test/parallel/test-abortsignal-drop-settled-signals.mjs +++ b/test/parallel/test-abortsignal-drop-settled-signals.mjs @@ -122,16 +122,22 @@ describe('when there is a long-lived signal', () => { }, true); }); - it('does not keep retained dependent signals without listeners', (t, done) => { + it('propagates abort to retained dependent signals without listeners', (t, done) => { const ac = new AbortController(); const retainedSignals = []; - const kDependantSignals = Object.getOwnPropertySymbols(ac.signal).find( - (s) => s.toString() === 'Symbol(kDependantSignals)' - ); function run(iteration) { if (iteration > limit) { - t.assert.strictEqual(ac.signal[kDependantSignals]?.size ?? 0, 0); + const kDependantSignals = Object.getOwnPropertySymbols(ac.signal).find( + (s) => s.toString() === 'Symbol(kDependantSignals)' + ); + t.assert.strictEqual(ac.signal[kDependantSignals].size, limit); + ac.abort('stop'); + for (const signal of retainedSignals) { + t.assert.strictEqual(signal.aborted, true); + t.assert.strictEqual(signal.reason, 'stop'); + t.assert.throws(() => signal.throwIfAborted(), (err) => err === 'stop'); + } done(); return; } @@ -143,6 +149,24 @@ describe('when there is a long-lived signal', () => { run(1); }); + it('drops unreachable dependent signals without listeners', async () => { + const ac = new AbortController(); + const size = () => { + const sym = Object.getOwnPropertySymbols(ac.signal).find( + (s) => s.toString() === 'Symbol(kDependantSignals)' + ); + return ac.signal[sym]?.size ?? 0; + }; + + // Reuse a long-lived source across batches to catch accumulating WeakRefs. + for (let batch = 0; batch < 3; batch++) { + for (let i = 0; i < limit; i++) { + AbortSignal.any([ac.signal]); + } + await gcUntil('unreachable dependents are dropped', () => size() === 0); + } + }); + it('drops observed dependent signals once they are transitively aborted', async () => { const longLived = new AbortController(); const handler = () => {}; From 95110712b8c27408c58420e460e9a78db0811b86 Mon Sep 17 00:00:00 2001 From: Shelley Vohr Date: Wed, 2 Sep 2026 22:41:39 +0000 Subject: [PATCH 135/217] inspector: fix abort when two Environments own the inspector Two Environments with default flags alive at the same time (for example the embedding.md example run on two threads, or two `CommonEnvironmentSetup`s) aborted the process: `Agent::Start()` bound one file-level static `uv_async_t` to the current Environment's loop for every Environment with `kOwnsInspector`, which `kDefaultFlags` implies, and CHECKed that nobody else had. Environments created one after another did not abort, but each ran `StartDebugSignalHandler()` again, which re-initialized the semaphore the watchdog waits on and spawned another detached watchdog thread, leaking one thread per Environment. Give every Agent that asks for the debug signal handler its own async handle, keep those Agents in a mutex-protected list that the watchdog (or the Windows remote thread) walks, and set the watchdog up once per process while still unblocking SIGUSR1 on each Environment's thread. The handle is heap-allocated, closed by the cleanup hook or `~Agent()`, whichever runs first, and freed by its close callback. A SIGUSR1 now reaches every Environment that asked for the handler, and no longer starts the inspector of one that passed `kNoStartDebugSignalHandler`. Refs: https://github.com/nodejs/node/pull/25777 Signed-off-by: Shelley Vohr PR-URL: https://github.com/nodejs/node/pull/65877 Refs: https://github.com/nodejs/node/pull/44121 Reviewed-By: Matteo Collina --- src/inspector_agent.cc | 129 ++++++++++++++++++-------------- src/inspector_agent.h | 10 ++- test/cctest/test_environment.cc | 3 +- 3 files changed, 81 insertions(+), 61 deletions(-) diff --git a/src/inspector_agent.cc b/src/inspector_agent.cc index dcec24c16dba..cecc3ce19350 100644 --- a/src/inspector_agent.cc +++ b/src/inspector_agent.cc @@ -72,19 +72,18 @@ using v8_inspector::V8InspectorClient; #ifdef __POSIX__ static uv_sem_t start_io_thread_semaphore; #endif // __POSIX__ -static uv_async_t start_io_thread_async; -// This is just an additional check to make sure start_io_thread_async -// is not accidentally re-used or used when uninitialized. -static std::atomic_bool start_io_thread_async_initialized { false }; -// Protects the Agent* stored in start_io_thread_async.data. -static Mutex start_io_thread_async_mutex; - -// Called on the main thread. -void StartIoThreadAsyncCallback(uv_async_t* handle) { - static_cast(handle->data)->StartIoThread(); +// Agents that asked for the debug signal handler; SIGUSR1 (or the Windows +// remote thread) starts the io thread of each. The mutex also guards the +// once-per-process watchdog setup. +static Mutex start_io_thread_agents_mutex; +static std::vector start_io_thread_agents; +static bool debug_signal_handler_started = false; + +static void RequestIoThreadStartOnAgents() { + Mutex::ScopedLock lock(start_io_thread_agents_mutex); + for (Agent* agent : start_io_thread_agents) agent->RequestIoThreadStart(); } - #ifdef __POSIX__ static void StartIoThreadWakeup(int signo, siginfo_t* info, void* ucontext) { uv_sem_post(&start_io_thread_semaphore); @@ -94,16 +93,11 @@ inline void* StartIoThreadMain(void* unused) { uv_thread_setname("SignalInspector"); for (;;) { uv_sem_wait(&start_io_thread_semaphore); - Mutex::ScopedLock lock(start_io_thread_async_mutex); - - CHECK(start_io_thread_async_initialized); - Agent* agent = static_cast(start_io_thread_async.data); - if (agent != nullptr) - agent->RequestIoThreadStart(); + RequestIoThreadStartOnAgents(); } } -static int StartDebugSignalHandler() { +static int StartWatchdogThread() { // Start a watchdog thread for calling v8::Debug::DebugBreak() because // it's not safe to call directly from the signal handler, it can // deadlock with the thread it interrupts. @@ -138,14 +132,28 @@ static int StartDebugSignalHandler() { fprintf(stderr, "node[%u]: pthread_create: %s\n", uv_os_getpid(), strerror(err)); fflush(stderr); - // Leave SIGUSR1 blocked. We don't install a signal handler, - // receiving the signal would terminate the process. + uv_sem_destroy(&start_io_thread_semaphore); return -err; } RegisterSignalHandler(SIGUSR1, StartIoThreadWakeup); // Restore original mask CHECK_EQ(0, pthread_sigmask(SIG_SETMASK, &sigmask, nullptr)); - // Unblock SIGUSR1. A pending SIGUSR1 signal will now be delivered. + return 0; +} + +static int StartDebugSignalHandler() { + { + Mutex::ScopedLock lock(start_io_thread_agents_mutex); + if (!debug_signal_handler_started) { + // Leave SIGUSR1 blocked on failure. We don't install a signal handler, + // receiving the signal would terminate the process. + if (int err = StartWatchdogThread()) return err; + debug_signal_handler_started = true; + } + } + // Unblock SIGUSR1 on this thread; PlatformInit() left it blocked. A pending + // SIGUSR1 signal will now be delivered. + sigset_t sigmask; sigemptyset(&sigmask); sigaddset(&sigmask, SIGUSR1); CHECK_EQ(0, pthread_sigmask(SIG_UNBLOCK, &sigmask, nullptr)); @@ -156,11 +164,7 @@ static int StartDebugSignalHandler() { #ifdef _WIN32 DWORD WINAPI StartIoThreadProc(void* arg) { - Mutex::ScopedLock lock(start_io_thread_async_mutex); - CHECK(start_io_thread_async_initialized); - Agent* agent = static_cast(start_io_thread_async.data); - if (agent != nullptr) - agent->RequestIoThreadStart(); + RequestIoThreadStartOnAgents(); return 0; } @@ -170,6 +174,9 @@ static int GetDebugSignalHandlerMappingName(DWORD pid, wchar_t* buf, } static int StartDebugSignalHandler() { + Mutex::ScopedLock lock(start_io_thread_agents_mutex); + if (debug_signal_handler_started) return 0; + debug_signal_handler_started = true; wchar_t mapping_name[32]; HANDLE mapping_handle; DWORD pid; @@ -845,7 +852,21 @@ Agent::Agent(Environment* env) debug_options_(env->options()->debug_options()), host_port_(env->inspector_host_port()) {} -Agent::~Agent() = default; +Agent::~Agent() { + StopAcceptingIoThreadStarts(); +} + +void Agent::StopAcceptingIoThreadStarts() { + if (start_io_thread_async_ == nullptr) return; + { + Mutex::ScopedLock lock(start_io_thread_agents_mutex); + std::erase(start_io_thread_agents, this); + } + parent_env_->RemoveCleanupHook(StopAcceptingIoThreadStartsHook, this); + parent_env_->CloseHandle(start_io_thread_async_, + [](uv_async_t* handle) { delete handle; }); + start_io_thread_async_ = nullptr; +} bool Agent::Start(const std::string& path, const DebugOptions& options, @@ -857,33 +878,25 @@ bool Agent::Start(const std::string& path, host_port_ = host_port; client_ = std::make_shared(parent_env_, is_main); - if (parent_env_->owns_inspector()) { - Mutex::ScopedLock lock(start_io_thread_async_mutex); - CHECK_EQ(start_io_thread_async_initialized.exchange(true), false); - CHECK_EQ(0, uv_async_init(parent_env_->event_loop(), - &start_io_thread_async, - StartIoThreadAsyncCallback)); - uv_unref(reinterpret_cast(&start_io_thread_async)); - start_io_thread_async.data = this; - if (parent_env_->should_start_debug_signal_handler()) { - // Ignore failure, SIGUSR1 won't work, but that should not block node - // start. - StartDebugSignalHandler(); + if (parent_env_->owns_inspector() && + parent_env_->should_start_debug_signal_handler()) { + start_io_thread_async_ = new uv_async_t; + start_io_thread_async_->data = this; + CHECK_EQ(0, + uv_async_init(parent_env_->event_loop(), + start_io_thread_async_, + [](uv_async_t* handle) { + static_cast(handle->data)->StartIoThread(); + })); + uv_unref(reinterpret_cast(start_io_thread_async_)); + { + Mutex::ScopedLock lock(start_io_thread_agents_mutex); + start_io_thread_agents.push_back(this); } - - parent_env_->AddCleanupHook([](void* data) { - Environment* env = static_cast(data); - - { - Mutex::ScopedLock lock(start_io_thread_async_mutex); - start_io_thread_async.data = nullptr; - } - - // This is global, will never get freed - env->CloseHandle(&start_io_thread_async, [](uv_async_t*) { - CHECK(start_io_thread_async_initialized.exchange(false)); - }); - }, parent_env_); + parent_env_->AddCleanupHook(StopAcceptingIoThreadStartsHook, this); + // Ignore failure, SIGUSR1 won't work, but that should not block node + // start. + StartDebugSignalHandler(); } AtExit(parent_env_, [](void* env) { @@ -1162,6 +1175,10 @@ void Agent::AllAsyncTasksCanceled() { client_->AllAsyncTasksCanceled(); } +void Agent::StopAcceptingIoThreadStartsHook(void* agent) { + static_cast(agent)->StopAcceptingIoThreadStarts(); +} + void Agent::RequestIoThreadStart() { // We need to attempt to interrupt V8 flow (in case Node is running // continuous JS code) and to wake up libuv thread (in case Node is waiting @@ -1169,14 +1186,10 @@ void Agent::RequestIoThreadStart() { if (!options().allow_attaching_debugger) { return; } - CHECK(start_io_thread_async_initialized); - uv_async_send(&start_io_thread_async); parent_env_->RequestInterrupt([this](Environment*) { StartIoThread(); }); - - CHECK(start_io_thread_async_initialized); - uv_async_send(&start_io_thread_async); + uv_async_send(start_io_thread_async_); } void Agent::ContextCreated(Local context, const ContextInfo& info) { diff --git a/src/inspector_agent.h b/src/inspector_agent.h index 932e4e8dce89..5a1d1de26547 100644 --- a/src/inspector_agent.h +++ b/src/inspector_agent.h @@ -8,6 +8,7 @@ #endif #include "node_options.h" +#include "uv.h" #include "v8.h" #include @@ -117,7 +118,8 @@ class Agent { // Can only be called from the main thread. bool StartIoThread(); - // Calls StartIoThread() from off the main thread. + // Calls StartIoThread() from off the main thread. Only valid while the + // Environment owns the inspector and has not started cleanup. void RequestIoThreadStart(); const DebugOptions& options() { return debug_options_; } @@ -156,6 +158,12 @@ class Agent { bool async_hook_enabled_ = false; bool syncing_async_hook_state_ = false; + // Woken by the SIGUSR1 watchdog; closed by the cleanup hook or ~Agent(), + // whichever runs first, and freed by its close callback. + uv_async_t* start_io_thread_async_ = nullptr; + void StopAcceptingIoThreadStarts(); + static void StopAcceptingIoThreadStartsHook(void* agent); + bool network_tracking_enabled_ = false; bool pending_enable_network_tracking = false; bool pending_disable_network_tracking = false; diff --git a/test/cctest/test_environment.cc b/test/cctest/test_environment.cc index af9d4151a7a2..1d2c3ef0afbb 100644 --- a/test/cctest/test_environment.cc +++ b/test/cctest/test_environment.cc @@ -343,9 +343,8 @@ TEST_F(EnvironmentTest, RemoveEnvironmentCleanupHookDuringCleanup) { TEST_F(EnvironmentTest, MultipleEnvironmentsPerIsolate) { const v8::HandleScope handle_scope(isolate_); const Argv argv; - // Only one of the Environments can have default flags and own the inspector. Env env1 {handle_scope, argv}; - Env env2 {handle_scope, argv, node::EnvironmentFlags::kNoFlags}; + Env env2{handle_scope, argv}; AtExit(*env1, at_exit_callback1, nullptr); AtExit(*env2, at_exit_callback2, nullptr); From 2cc17dd67cdea79d5ce5fbd12ec487932015c250 Mon Sep 17 00:00:00 2001 From: Shelley Vohr Date: Thu, 3 Sep 2026 00:03:53 +0000 Subject: [PATCH 136/217] doc: note that default signal handling resets the signal mask `InitializeOncePerProcess()` without `kNoDefaultSignalHandling` calls `pthread_sigmask(SIG_SETMASK, ...)` with a set containing only SIGUSR1, which unblocks every signal the embedder had blocked on the calling thread. Say so in the flag's documentation. Refs: https://github.com/nodejs/node/pull/44121 Signed-off-by: Shelley Vohr PR-URL: https://github.com/nodejs/node/pull/65877 Refs: https://github.com/nodejs/node/pull/25777 Reviewed-By: Matteo Collina --- src/node.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/node.h b/src/node.h index 995d8281d029..c8f45a6b4c12 100644 --- a/src/node.h +++ b/src/node.h @@ -210,8 +210,10 @@ enum Flags : uint32_t { kNoICU = 1 << 3, // Do not modify stdio file descriptor or TTY state. kNoStdioInitialization = 1 << 4, - // Do not register Node.js-specific signal handlers - // and reset other signal handlers to default state. + // Do not register Node.js-specific signal handlers, reset other signal + // handlers to default state, or replace the calling thread's signal mask + // (without this flag, POSIX builds with the inspector set it to block + // SIGUSR1 and nothing else). kNoDefaultSignalHandling = 1 << 5, // Do not perform V8 initialization. kNoInitializeV8 = 1 << 6, From 2b1701f8108e655f2089c865f4ae2db3fec91219 Mon Sep 17 00:00:00 2001 From: greenhead Date: Wed, 16 Sep 2026 19:28:00 +0900 Subject: [PATCH 137/217] fs: add openAsBlobSync Signed-off-by: greenhead Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/65644 Refs: https://github.com/nodejs/node/issues/65462 Refs: https://github.com/nodejs/node/pull/49759 Reviewed-By: James M Snell --- doc/api/fs.md | 15 +++++ lib/fs.js | 24 ++++++++ .../bootstrap/switches/is_main_thread.js | 6 +- lib/internal/vfs/setup.js | 2 + test/fixtures/permission/fs-read.js | 27 ++++++++- test/parallel/test-fs-openAsBlobSync.js | 58 +++++++++++++++++++ test/parallel/test-permission-fs-supported.js | 2 +- test/parallel/test-vfs-fs-openAsBlob.js | 34 +++++++++-- 8 files changed, 158 insertions(+), 10 deletions(-) create mode 100644 test/parallel/test-fs-openAsBlobSync.js diff --git a/doc/api/fs.md b/doc/api/fs.md index 69aaf60c5b6a..a48b4d28497d 100644 --- a/doc/api/fs.md +++ b/doc/api/fs.md @@ -6526,6 +6526,20 @@ with the [`using`][] syntax. The optional `options` argument can be a string specifying an encoding, or an object with an `encoding` property specifying the character encoding to use. +### `fs.openAsBlobSync(path[, options])` + + + +* `path` {string|Buffer|URL} +* `options` {Object} + * `type` {string} An optional mime type for the blob. +* Returns: {Blob} + +For detailed information, see the documentation of the Promise-returning +version of this API: [`fs.openAsBlob()`][]. + ### `fs.opendirSync(path[, options])` + +* `promise` {promise} The promise to mark as handled + +Marks a promise as handled so that unhandled rejections are ignored and are not +reported to the `'unhandledrejection'` event. + ## Class: `util.MIMEType` + +* `bundle` {ArrayBuffer|Buffer|TypedArray|DataView} A DER-encoded PKCS#12 + (`.p12` or `.pfx`) bundle. +* `options` {Object} + * `passphrase` {string|ArrayBuffer|Buffer|TypedArray|DataView} The passphrase + protecting the bundle. Omitting this option is equivalent to passing `''`. +* Returns: {Object} + * `privateKey` {KeyObject|null} The first private key in the bundle, or + `null` if none is present. + * `certificate` {X509Certificate|null} The certificate matching `privateKey`, + or `null` if no matching certificate is present. + * `additionalCertificates` {X509Certificate\[]} All other certificates in + the bundle. If there is no private key, this contains all certificates. + May be empty. + +Parses a PKCS#12 bundle, commonly stored with a `.p12` or `.pfx` extension, +and returns its private key and certificates. + +```mjs +import { parsePKCS12 } from 'node:crypto'; +import { readFileSync } from 'node:fs'; + +const { privateKey, certificate, additionalCertificates } = parsePKCS12( + readFileSync('bundle.p12'), + { passphrase: 'secret' }, +); +``` + ### `crypto.pbkdf2(password, salt, iterations, keylen, digest, callback)` + +* `options` {Object} + * `bins` {number} The number of equal-probability density bins to return. + Must be between 1 and 1000. Cannot be used with `probabilities`. + **Default:** `100`. + * `probabilities` {number\[]} Custom probability boundaries. The array must + contain between 2 and 1001 strictly increasing values, start with `0`, and + end with `1`. Cannot be used with `bins`. + * `dequantize` {string} Controls whether repeated bucket values are spread + deterministically over their equivalent-value ranges. May be `'none'`, + `'hdr'`, or `'all'`. **Default:** `'hdr'`. + * `cache` {boolean} When `true`, retains the expanded histogram snapshot for + reuse by subsequent calls with `cache: true`. The snapshot is invalidated + when the histogram is modified. **Default:** `false`. +* Returns: {Promise} Fulfills with an {Object} containing: + * `probabilities` {Float64Array} The probability boundaries used by the + estimate. + * `quantiles` {Float64Array} The quantiles at the probability boundaries. + * `densities` {Float64Array} The density within each quantile interval. + * `count` {bigint} The number of values in the histogram snapshot. + * `bucketCount` {number} The number of occupied HDR buckets. + * `corrections` {number} The number of non-monotonic floating-point results + that were clamped to the preceding quantile. + * `dequantize` {string} The selected dequantization mode. + +Returns a quantile-respectful density estimate based on the Harrell-Davis +quantile estimator. By default, `bins` generates equal probability boundaries. +The `probabilities` option can instead focus the estimate on regions such as +p90, p99, p99.9, and p99.99. The density for interval `i` contains probability +mass `probabilities[i + 1] - probabilities[i]`. The histogram is snapshotted +when the method is called. Snapshot expansion and the estimate are calculated +in the libuv thread pool. Highly concentrated beta weights use a second-order +asymptotic approximation to avoid numerical convergence loss at large sample +counts. + +Setting `cache` to `true` avoids repeating snapshot capture and expansion when +several estimates are requested from an unchanged histogram. The retained +snapshot uses memory proportional to the number of occupied HDR buckets and is +released when the histogram is next modified. + +QRDE temporarily uses approximately one additional HDR count array plus 32 +bytes per occupied bucket. With `cache: true`, the expanded 32-byte-per-bucket +snapshot remains allocated. The following estimates use `lowest: 1` and +`highest: Number.MAX_SAFE_INTEGER` and exclude allocator and JavaScript object +overhead: + +| `figures` | Histogram | Maximum expanded snapshot | Peak cache-miss QRDE | +| --------- | --------: | ------------------------: | -------------------: | +| 1 | 6.3 KiB | 25 KiB | 31 KiB | +| 2 | 47 KiB | 188 KiB | 235 KiB | +| 3 | 352 KiB | 1.4 MiB | 1.7 MiB | +| 4 | 5.0 MiB | 20 MiB | 25 MiB | +| 5 | 37 MiB | 148 MiB | 185 MiB | + +The maximum snapshot column assumes every representable bucket is occupied. +Lower `highest` values reduce histogram and temporary copy sizes. Concurrent +calls that miss the cache each require their own temporary copy and expanded +snapshot. + +HDR histograms aggregate observations into equivalent-value buckets. The +`'hdr'` dequantization mode models repeated values in buckets wider than one +unit as a continuous uniform distribution over the bucket resolution. This +reduces density artifacts introduced by HDR quantization while preserving +repeated unit-resolution values as point masses. The `'all'` mode also +dequantizes repeated unit-resolution values. Use `'none'` to calculate the +grouped Harrell-Davis estimator using bucket midpoints directly. + +An empty histogram returns the requested `probabilities` but produces empty +`quantiles` and `densities` arrays. A non-dequantized interval whose quantile +boundaries are equal has an infinite density. + ### `histogram.reset()` + +* `options` {Object} + * `chunks` {number} The number of histogram chunks retained. Must be an + integer between `1` and `1024`. + * `chunkDuration` {number} The duration of each chunk in milliseconds. Must + be an integer between `1` and `18_446_744_073_709`. Exactly one of + `chunkDuration` and `recordsPerChunk` must be specified. + * `recordsPerChunk` {number} The number of calls to `record()` assigned to + each chunk. Must be an integer between `1` and `Number.MAX_SAFE_INTEGER`. + Exactly one of `chunkDuration` and `recordsPerChunk` must be specified. + * `lowest` {number|bigint} The lowest discernible value. Must be an integer + value greater than `0`. **Default:** `1`. + * `highest` {number|bigint} The highest recordable value. Must be an integer + value that is equal to or greater than two times `lowest`. + **Default:** `Number.MAX_SAFE_INTEGER`. + * `figures` {number} The number of accuracy digits. Must be an integer between + `1` and `5`. **Default:** `3`. +* Returns: {SlidingWindowHistogram} + +Creates a {SlidingWindowHistogram} that retains the latest `chunks` histogram +chunks. Rotation is lazy and does not create a timer. Time-based rotation is +evaluated when `record()` or `snapshot()` is called. Count-based rotation is +evaluated when `record()` is called. + +One histogram chunk is allocated during construction. Additional chunks are +allocated lazily. The maximum native memory used by the window scales with +`chunks` and with the `lowest`, `highest`, and `figures` histogram options. + +The window boundary has chunk-level precision. With `N` chunks of duration +`D`, a recorded value is retained for between `(N - 1) * D` and `N * D` +milliseconds. Once a count-based window is populated, it retains between +`(N - 1) * C + 1` and `N * C` recording attempts, where `C` is +`recordsPerChunk`. Recording attempts which exceed `highest` are included when +determining count-based rotation. + +```js +const { createSlidingWindowHistogram } = require('node:perf_hooks'); + +const window = createSlidingWindowHistogram({ + chunks: 6, + chunkDuration: 10_000, +}); + +window.record(20_000_000); + +// Materialize the current window as an independent Histogram. +const snapshot = window.snapshot(); +console.log(snapshot.percentile(99)); +``` + ## `perf_hooks.importHistogram(data)` + +Records values into a lazily rotated ring of histogram chunks. Instances are +created using [`perf_hooks.createSlidingWindowHistogram()`][] and cannot be +constructed directly. A `SlidingWindowHistogram` does not extend {Histogram}; +call `snapshot()` to materialize the current window as a {Histogram}. + +`SlidingWindowHistogram` instances cannot be cloned or transferred through a +{MessagePort}. + +### `slidingWindowHistogram.record(val)` + + + +* `val` {number|bigint} The amount to record. + +Records `val` in the current chunk. For a count-based window, every call that +reaches the native histogram counts toward rotation, including values which +exceed the configured `highest` value. + +### `slidingWindowHistogram.reset()` + + + +Invalidates all chunks in the current window. Allocated chunks are reset +lazily when reused. + +### `slidingWindowHistogram.snapshot()` + + + +* Returns: {Histogram} + +Materializes the current window as a new, independent {Histogram}. Values +recorded or expired after this method returns do not change the returned +histogram. Materialization allocates one histogram and merges every retained +chunk. + ## Histogram analysis examples The `Histogram` class provides statistical analysis methods useful for @@ -3115,6 +3218,7 @@ dns.promises.resolve('localhost'); [`'exit'`]: process.md#event-exit [`child_process.spawnSync()`]: child_process.md#child_processspawnsynccommand-args-options [`histogram.export()`]: #histogramexport +[`perf_hooks.createSlidingWindowHistogram()`]: #perf_hookscreateslidingwindowhistogramoptions [`perf_hooks.eventLoopUtilization()`]: #perf_hookseventlooputilizationutilization1-utilization2 [`perf_hooks.importHistogram()`]: #perf_hooksimporthistogramdata [`perf_hooks.monitorEventLoopDelay()`]: #perf_hooksmonitoreventloopdelayoptions diff --git a/lib/internal/histogram.js b/lib/internal/histogram.js index f2e592d9f814..1cc787404fdc 100644 --- a/lib/internal/histogram.js +++ b/lib/internal/histogram.js @@ -1,6 +1,7 @@ 'use strict'; const { + BigInt, Float64Array, Map, MapPrototypeEntries, @@ -12,6 +13,7 @@ const { const { Histogram: _Histogram, + SlidingWindowHistogram: _SlidingWindowHistogram, } = internalBinding('performance'); const { @@ -47,7 +49,11 @@ const { const kDestroy = Symbol('kDestroy'); const kHandle = Symbol('kHandle'); const kRecordable = Symbol('kRecordable'); +const kSlidingWindowHandle = Symbol('kSlidingWindowHandle'); const kQrdeDequantizationModes = ['none', 'hdr', 'all']; +const kMaxSlidingWindowHistogramChunks = 1024; +const kMaxChunkDuration = 18_446_744_073_709; +const kMaxInt64 = 9_223_372_036_854_775_807n; const { kClone, @@ -801,6 +807,48 @@ class RecordableHistogram extends Histogram { } } +class SlidingWindowHistogram { + constructor(skipThrowSymbol = undefined) { + if (skipThrowSymbol !== kSkipThrow) { + throw new ERR_ILLEGAL_CONSTRUCTOR(); + } + } + + /** + * @param {number|bigint} val + * @returns {void} + */ + record(val) { + if (this[kSlidingWindowHandle] === undefined) + throw new ERR_INVALID_THIS('SlidingWindowHistogram'); + if (typeof val === 'bigint') { + this[kSlidingWindowHandle].record(val); + return; + } + + validateInteger(val, 'val', 1); + this[kSlidingWindowHandle].record(val); + } + + /** + * @returns {Histogram} + */ + snapshot() { + if (this[kSlidingWindowHandle] === undefined) + throw new ERR_INVALID_THIS('SlidingWindowHistogram'); + return new ClonedHistogram(this[kSlidingWindowHandle].snapshot()); + } + + /** + * @returns {void} + */ + reset() { + if (this[kSlidingWindowHandle] === undefined) + throw new ERR_INVALID_THIS('SlidingWindowHistogram'); + this[kSlidingWindowHandle].reset(); + } +} + function ClonedHistogram(handle) { const histogram = new Histogram(kSkipThrow); markTransferMode(histogram, true, false); @@ -827,6 +875,32 @@ function createRecordableHistogram(handle) { return new ClonedRecordableHistogram(handle); } +function validateHistogramOptions(lowest, highest, figures) { + if (typeof lowest !== 'bigint') { + validateInteger(lowest, 'options.lowest', 1, NumberMAX_SAFE_INTEGER); + } else if (lowest < 1n || lowest > kMaxInt64) { + throw new ERR_OUT_OF_RANGE( + 'options.lowest', `>= 1n && <= ${kMaxInt64}n`, lowest); + } + + if (typeof highest !== 'bigint') { + validateInteger(highest, 'options.highest', 1, NumberMAX_SAFE_INTEGER); + } else if (highest < 1n || highest > kMaxInt64) { + throw new ERR_OUT_OF_RANGE( + 'options.highest', `>= 1n && <= ${kMaxInt64}n`, highest); + } + + const minimumHighest = 2n * + (typeof lowest === 'bigint' ? lowest : BigInt(lowest)); + const highestBigInt = typeof highest === 'bigint' ? + highest : BigInt(highest); + if (highestBigInt < minimumHighest) { + throw new ERR_OUT_OF_RANGE( + 'options.highest', `>= 2 * options.lowest (${minimumHighest}n)`, highest); + } + validateInteger(figures, 'options.figures', 1, 5); +} + /** * @param {{ * lowest? : number, @@ -846,15 +920,7 @@ function createHistogram(options = kEmptyObject) { halfLife = 0, threshold = 0, } = options; - if (typeof lowest !== 'bigint') - validateInteger(lowest, 'options.lowest', 1, NumberMAX_SAFE_INTEGER); - if (typeof highest !== 'bigint') { - validateInteger(highest, 'options.highest', - 2 * lowest, NumberMAX_SAFE_INTEGER); - } else if (highest < 2n * lowest) { - throw new ERR_INVALID_ARG_VALUE.RangeError('options.highest', highest); - } - validateInteger(figures, 'options.figures', 1, 5); + validateHistogramOptions(lowest, highest, figures); validateNumber(halfLife, 'options.halfLife'); if (halfLife < 0) throw new ERR_OUT_OF_RANGE('options.halfLife', '>= 0', halfLife); @@ -865,6 +931,57 @@ function createHistogram(options = kEmptyObject) { new _Histogram(lowest, highest, figures, halfLife, threshold)); } +/** + * @param {{ + * chunks: number, + * chunkDuration? : number, + * recordsPerChunk? : number, + * lowest? : number|bigint, + * highest? : number|bigint, + * figures? : number, + * }} options + * @returns {SlidingWindowHistogram} + */ +function createSlidingWindowHistogram(options) { + validateObject(options, 'options'); + const { + chunks, + chunkDuration, + recordsPerChunk, + lowest = 1, + highest = NumberMAX_SAFE_INTEGER, + figures = 3, + } = options; + + validateInteger( + chunks, 'options.chunks', 1, kMaxSlidingWindowHistogramChunks); + validateHistogramOptions(lowest, highest, figures); + + const timeBased = chunkDuration !== undefined; + if (timeBased === (recordsPerChunk !== undefined)) { + throw new ERR_INVALID_ARG_VALUE( + 'options', options, + 'must specify exactly one of "chunkDuration" or "recordsPerChunk"'); + } + + let rotateAt; + if (timeBased) { + validateInteger( + chunkDuration, 'options.chunkDuration', 1, kMaxChunkDuration); + rotateAt = BigInt(chunkDuration) * 1_000_000n; + } else { + validateInteger( + recordsPerChunk, 'options.recordsPerChunk', 1, NumberMAX_SAFE_INTEGER); + rotateAt = BigInt(recordsPerChunk); + } + + const histogram = new SlidingWindowHistogram(kSkipThrow); + markTransferMode(histogram, false, false); + histogram[kSlidingWindowHandle] = new _SlidingWindowHistogram( + lowest, highest, figures, chunks, timeBased, rotateAt); + return histogram; +} + /** * Reconstructs a histogram from a CBOR-encoded Uint8Array previously * produced by `histogram.export()`. @@ -880,6 +997,7 @@ function importHistogram(data) { module.exports = { Histogram, RecordableHistogram, + SlidingWindowHistogram, ClonedHistogram, ClonedRecordableHistogram, isHistogram, @@ -887,5 +1005,6 @@ module.exports = { kHandle, kSkipThrow, createHistogram, + createSlidingWindowHistogram, importHistogram, }; diff --git a/lib/perf_hooks.js b/lib/perf_hooks.js index cc158e5c7625..5de247442b52 100644 --- a/lib/perf_hooks.js +++ b/lib/perf_hooks.js @@ -25,6 +25,7 @@ const { const { createHistogram, + createSlidingWindowHistogram, importHistogram, } = require('internal/histogram'); @@ -44,6 +45,7 @@ module.exports = { eventLoopUtilization, timerify, createHistogram, + createSlidingWindowHistogram, importHistogram, performance, }; diff --git a/src/histogram.cc b/src/histogram.cc index 4b99d9f97f9d..63f297936773 100644 --- a/src/histogram.cc +++ b/src/histogram.cc @@ -23,6 +23,7 @@ using v8::BigInt; using v8::CFunction; using v8::Context; using v8::Exception; +using v8::FastApiCallbackOptions; using v8::Float64Array; using v8::FunctionCallbackInfo; using v8::FunctionTemplate; @@ -1741,6 +1742,8 @@ CFunction HistogramBase::fast_record_( CFunction::Make(&HistogramBase::FastRecord)); CFunction HistogramBase::fast_record_delta_( CFunction::Make(&HistogramBase::FastRecordDelta)); +CFunction SlidingWindowHistogram::fast_record_( + CFunction::Make(&SlidingWindowHistogram::FastRecord)); CFunction IntervalHistogram::fast_start_( CFunction::Make(&IntervalHistogram::FastStart)); CFunction IntervalHistogram::fast_stop_( @@ -2102,6 +2105,254 @@ void HistogramBase::HistogramTransferData::MemoryInfo( tracker->TrackField("histogram", histogram_); } +SlidingWindowHistogram::SlidingWindowHistogram( + Environment* env, + Local wrap, + const Histogram::Options& options, + size_t chunk_count, + bool time_based, + uint64_t rotate_at, + std::shared_ptr spare) + : BaseObject(env, wrap), + options_(options), + chunks_(chunk_count), + generations_(chunk_count, kNoGeneration), + spare_(std::move(spare)), + time_based_(time_based), + rotate_at_(rotate_at), + origin_(uv_hrtime()) { + MakeWeak(); + external_memory_ = spare_->GetMemorySize(); + env->external_memory_accounter()->Increase(env->isolate(), external_memory_); +} + +SlidingWindowHistogram::~SlidingWindowHistogram() { + env()->external_memory_accounter()->Decrease(env()->isolate(), + external_memory_); +} + +void SlidingWindowHistogram::MemoryInfo(MemoryTracker* tracker) const { + tracker->TrackField("chunks", chunks_); + tracker->TrackField("generations", generations_); + tracker->TrackField("spare", spare_); +} + +uint64_t SlidingWindowHistogram::CurrentTimeGeneration() const { + const uint64_t now = uv_hrtime(); + CHECK_GE(now, origin_); + return (now - origin_) / rotate_at_; +} + +Histogram* SlidingWindowHistogram::GetChunk(uint64_t generation) { + const size_t index = generation % chunks_.size(); + if (generations_[index] == generation) { + CHECK(chunks_[index]); + return chunks_[index].get(); + } + + if (chunks_[index]) { + chunks_[index]->Reset(); + } else if (spare_) { + chunks_[index] = std::move(spare_); + } else { + chunks_[index] = Histogram::Create(options_); + if (!chunks_[index]) return nullptr; + const size_t size = chunks_[index]->GetMemorySize(); + external_memory_ += size; + env()->external_memory_accounter()->Increase(env()->isolate(), size); + } + + generations_[index] = generation; + return chunks_[index].get(); +} + +bool SlidingWindowHistogram::RecordValue(int64_t value) { + uint64_t generation; + if (time_based_) { + generation = CurrentTimeGeneration(); + } else if (records_in_current_chunk_ == rotate_at_) { + CHECK_LT(current_generation_, kNoGeneration - 1); + generation = current_generation_ + 1; + } else { + generation = current_generation_; + } + + Histogram* chunk = GetChunk(generation); + if (chunk == nullptr) return false; + + chunk->Record(value); + if (!time_based_) { + if (generation != current_generation_) { + current_generation_ = generation; + records_in_current_chunk_ = 0; + } + records_in_current_chunk_++; + has_count_records_ = true; + } + return true; +} + +std::shared_ptr SlidingWindowHistogram::CreateSnapshot() const { + std::shared_ptr snapshot = Histogram::Create(options_); + if (!snapshot) return {}; + + uint64_t current_generation; + if (time_based_) { + current_generation = CurrentTimeGeneration(); + } else { + if (!has_count_records_) return snapshot; + current_generation = current_generation_; + } + + for (size_t i = 0; i < chunks_.size(); i++) { + const uint64_t generation = generations_[i]; + if (generation == kNoGeneration || generation > current_generation || + current_generation - generation >= chunks_.size()) { + continue; + } + CHECK(chunks_[i]); + CHECK_EQ(snapshot->Add(*chunks_[i]), 0); + } + return snapshot; +} + +void SlidingWindowHistogram::ResetWindow() { + std::fill(generations_.begin(), generations_.end(), kNoGeneration); + origin_ = uv_hrtime(); + current_generation_ = 0; + records_in_current_chunk_ = 0; + has_count_records_ = false; +} + +void SlidingWindowHistogram::New(const FunctionCallbackInfo& args) { + CHECK(args.IsConstructCall()); + CHECK_IMPLIES(!args[0]->IsNumber(), args[0]->IsBigInt()); + CHECK_IMPLIES(!args[1]->IsNumber(), args[1]->IsBigInt()); + CHECK(args[2]->IsUint32()); + CHECK(args[3]->IsUint32()); + CHECK(args[4]->IsBoolean()); + CHECK(args[5]->IsBigInt()); + + Environment* env = Environment::GetCurrent(args); + bool lossless = true; + int64_t lowest = 1; + int64_t highest = std::numeric_limits::max(); + + if (args[0]->IsNumber()) { + lowest = args[0].As()->Value(); + } else { + lowest = args[0].As()->Int64Value(&lossless); + if (!lossless) + return THROW_ERR_OUT_OF_RANGE(env, "options.lowest is out of range"); + } + + if (args[1]->IsNumber()) { + highest = args[1].As()->Value(); + } else { + highest = args[1].As()->Int64Value(&lossless); + if (!lossless) + return THROW_ERR_OUT_OF_RANGE(env, "options.highest is out of range"); + } + + const int figures = args[2].As()->Value(); + const uint32_t chunk_count = args[3].As()->Value(); + if (chunk_count == 0) + return THROW_ERR_OUT_OF_RANGE(env, "options.chunks is out of range"); + + lossless = true; + const uint64_t rotate_at = args[5].As()->Uint64Value(&lossless); + if (!lossless || rotate_at == 0) { + return THROW_ERR_OUT_OF_RANGE(env, "rotation interval is out of range"); + } + + Histogram::Options options{lowest, highest, figures}; + std::shared_ptr spare = Histogram::Create(options); + if (!spare) + return THROW_ERR_INVALID_ARG_VALUE(env, "Invalid histogram options"); + + new SlidingWindowHistogram(env, + args.This(), + options, + chunk_count, + args[4]->IsTrue(), + rotate_at, + std::move(spare)); +} + +void SlidingWindowHistogram::Record(const FunctionCallbackInfo& args) { + Environment* env = Environment::GetCurrent(args); + CHECK_IMPLIES(!args[0]->IsNumber(), args[0]->IsBigInt()); + bool lossless = true; + const int64_t value = + args[0]->IsBigInt() ? args[0].As()->Int64Value(&lossless) + : static_cast(args[0].As()->Value()); + if (!lossless || value < 1) + return THROW_ERR_OUT_OF_RANGE(env, "value is out of range"); + + SlidingWindowHistogram* histogram; + ASSIGN_OR_RETURN_UNWRAP(&histogram, args.This()); + if (!histogram->RecordValue(value)) THROW_ERR_MEMORY_ALLOCATION_FAILED(env); +} + +void SlidingWindowHistogram::FastRecord(Local receiver, + int64_t value, + // NOLINTNEXTLINE(runtime/references) + FastApiCallbackOptions& options) { + CHECK_GE(value, 1); + TRACK_V8_FAST_API_CALL("histogram.slidingWindow.record"); + SlidingWindowHistogram* histogram; + ASSIGN_OR_RETURN_UNWRAP(&histogram, receiver); + if (!histogram->RecordValue(value)) { + HandleScope scope(options.isolate); + THROW_ERR_MEMORY_ALLOCATION_FAILED(histogram->env()); + } +} + +void SlidingWindowHistogram::Snapshot(const FunctionCallbackInfo& args) { + Environment* env = Environment::GetCurrent(args); + SlidingWindowHistogram* histogram; + ASSIGN_OR_RETURN_UNWRAP(&histogram, args.This()); + + std::shared_ptr snapshot = histogram->CreateSnapshot(); + if (!snapshot) return THROW_ERR_MEMORY_ALLOCATION_FAILED(env); + + BaseObjectPtr result = + HistogramBase::Create(env, std::move(snapshot)); + if (result) args.GetReturnValue().Set(result->object()); +} + +void SlidingWindowHistogram::Reset(const FunctionCallbackInfo& args) { + SlidingWindowHistogram* histogram; + ASSIGN_OR_RETURN_UNWRAP(&histogram, args.This()); + histogram->ResetWindow(); +} + +void SlidingWindowHistogram::Initialize(IsolateData* isolate_data, + Local target) { + Isolate* isolate = isolate_data->isolate(); + Local tmpl = NewFunctionTemplate(isolate, New); + tmpl->SetClassName(FIXED_ONE_BYTE_STRING(isolate, "SlidingWindowHistogram")); + auto instance = tmpl->InstanceTemplate(); + instance->SetInternalFieldCount(BaseObject::kInternalFieldCount); + SetFastMethod(isolate, instance, "record", Record, &fast_record_); + SetProtoMethod(isolate, tmpl, "snapshot", Snapshot); + SetProtoMethod(isolate, tmpl, "reset", Reset); + SetConstructorFunction(isolate, + target, + "SlidingWindowHistogram", + tmpl, + SetConstructorFunctionFlag::NONE); +} + +void SlidingWindowHistogram::RegisterExternalReferences( + ExternalReferenceRegistry* registry) { + registry->Register(New); + registry->Register(Record); + registry->Register(fast_record_); + registry->Register(Snapshot); + registry->Register(Reset); +} + Local IntervalHistogram::GetConstructorTemplate( Environment* env) { Local tmpl = env->intervalhistogram_constructor_template(); diff --git a/src/histogram.h b/src/histogram.h index 623915e47e45..bc72fa36e104 100644 --- a/src/histogram.h +++ b/src/histogram.h @@ -360,6 +360,60 @@ class HistogramBase final : public BaseObject, public HistogramImpl { static v8::CFunction fast_record_delta_; }; +// BaseObject disallows cloning and transfer, so ring state is confined to the +// owning Environment's thread. +class SlidingWindowHistogram final : public BaseObject { + public: + static void Initialize(IsolateData* isolate_data, + v8::Local target); + static void RegisterExternalReferences(ExternalReferenceRegistry* registry); + + void MemoryInfo(MemoryTracker* tracker) const override; + SET_MEMORY_INFO_NAME(SlidingWindowHistogram) + SET_SELF_SIZE(SlidingWindowHistogram) + + private: + static constexpr uint64_t kNoGeneration = + std::numeric_limits::max(); + + static void New(const v8::FunctionCallbackInfo& args); + static void Record(const v8::FunctionCallbackInfo& args); + static void FastRecord(v8::Local receiver, + int64_t value, + v8::FastApiCallbackOptions& options); + static void Snapshot(const v8::FunctionCallbackInfo& args); + static void Reset(const v8::FunctionCallbackInfo& args); + + SlidingWindowHistogram(Environment* env, + v8::Local wrap, + const Histogram::Options& options, + size_t chunk_count, + bool time_based, + uint64_t rotate_at, + std::shared_ptr spare); + ~SlidingWindowHistogram() override; + + Histogram* GetChunk(uint64_t generation); + bool RecordValue(int64_t value); + std::shared_ptr CreateSnapshot() const; + void ResetWindow(); + uint64_t CurrentTimeGeneration() const; + + Histogram::Options options_; + std::vector> chunks_; + std::vector generations_; + std::shared_ptr spare_; + bool time_based_; + uint64_t rotate_at_; + uint64_t origin_; + uint64_t current_generation_ = 0; + uint64_t records_in_current_chunk_ = 0; + size_t external_memory_ = 0; + bool has_count_records_ = false; + + static v8::CFunction fast_record_; +}; + // CRTP mixin for HandleWrap-based histograms with start/stop support. // Provides: StartFlags enum, Start/Stop slow-path handlers, enabled_ flag, // and InitTemplate (shared GetConstructorTemplate body). diff --git a/src/node_perf.cc b/src/node_perf.cc index 75a62b89a534..b4c74e9a09a7 100644 --- a/src/node_perf.cc +++ b/src/node_perf.cc @@ -341,6 +341,7 @@ static void CreatePerIsolateProperties(IsolateData* isolate_data, Isolate* isolate = isolate_data->isolate(); HistogramBase::Initialize(isolate_data, target); + SlidingWindowHistogram::Initialize(isolate_data, target); SetMethod(isolate, target, "setupObservers", SetupPerformanceObservers); SetMethod(isolate, @@ -432,6 +433,7 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) { registry->Register(SlowPerformanceNow); registry->Register(fast_performance_now); HistogramBase::RegisterExternalReferences(registry); + SlidingWindowHistogram::RegisterExternalReferences(registry); IntervalHistogram::RegisterExternalReferences(registry); IterationHistogram::RegisterExternalReferences(registry); } diff --git a/test/parallel/test-perf-hooks-sliding-window-histogram-fast-calls.js b/test/parallel/test-perf-hooks-sliding-window-histogram-fast-calls.js new file mode 100644 index 000000000000..1097920f5f73 --- /dev/null +++ b/test/parallel/test-perf-hooks-sliding-window-histogram-fast-calls.js @@ -0,0 +1,31 @@ +// Flags: --expose-internals --no-warnings --allow-natives-syntax +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { internalBinding } = require('internal/test/binding'); +const { + createSlidingWindowHistogram, +} = require('perf_hooks'); + +const histogram = createSlidingWindowHistogram({ + chunks: 2, + recordsPerChunk: 1, +}); + +function record() { + histogram.record(1); +} + +eval('%PrepareFunctionForOptimization(histogram.record)'); +record(); +eval('%OptimizeFunctionOnNextCall(histogram.record)'); +record(); + +assert.strictEqual(histogram.snapshot().count, 2); + +if (common.isDebug) { + const { getV8FastApiCallCount } = internalBinding('debug'); + assert.strictEqual( + getV8FastApiCallCount('histogram.slidingWindow.record'), 1); +} diff --git a/test/parallel/test-perf-hooks-sliding-window-histogram.js b/test/parallel/test-perf-hooks-sliding-window-histogram.js new file mode 100644 index 000000000000..9e28677e5d62 --- /dev/null +++ b/test/parallel/test-perf-hooks-sliding-window-histogram.js @@ -0,0 +1,177 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { setTimeout: delay } = require('timers/promises'); +const { MessageChannel } = require('worker_threads'); +const { + createSlidingWindowHistogram, +} = require('perf_hooks'); + +{ + const histogram = createSlidingWindowHistogram({ + chunks: 3, + recordsPerChunk: 2, + highest: 100, + }); + + assert.strictEqual(histogram.constructor.name, 'SlidingWindowHistogram'); + assert.strictEqual(histogram.recordDelta, undefined); + assert.strictEqual(histogram.snapshot().count, 0); + + for (let value = 1; value <= 6; value++) histogram.record(value); + + const full = histogram.snapshot(); + assert.strictEqual(full.count, 6); + assert.strictEqual(full.min, 1); + assert.strictEqual(full.max, 6); + assert.strictEqual(full.record, undefined); + + histogram.record(7); + let current = histogram.snapshot(); + assert.strictEqual(current.count, 5); + assert.strictEqual(current.min, 3); + assert.strictEqual(current.max, 7); + + histogram.record(8); + histogram.record(9); + current = histogram.snapshot(); + assert.strictEqual(current.count, 5); + assert.strictEqual(current.min, 5); + assert.strictEqual(current.max, 9); + + // Materialized snapshots do not change with the sliding window. + assert.strictEqual(full.count, 6); + assert.strictEqual(full.min, 1); + assert.strictEqual(full.max, 6); + + histogram.reset(); + assert.strictEqual(histogram.snapshot().count, 0); + histogram.record(10n); + assert.strictEqual(histogram.snapshot().maxBigInt, 10n); + + assert.throws(() => new histogram.constructor(), { + code: 'ERR_ILLEGAL_CONSTRUCTOR', + }); + assert.throws(() => histogram.record.call({}, 1), { + code: 'ERR_INVALID_THIS', + }); + assert.throws(() => histogram.snapshot.call({}), { + code: 'ERR_INVALID_THIS', + }); + assert.throws(() => histogram.reset.call({}), { + code: 'ERR_INVALID_THIS', + }); + assert.throws(() => structuredClone(histogram), { + name: 'DataCloneError', + }); + + const { port1, port2 } = new MessageChannel(); + assert.throws(() => port1.postMessage(histogram), { + name: 'DataCloneError', + }); + assert.throws(() => port1.postMessage(histogram, [histogram]), { + name: 'DataCloneError', + }); + port1.close(); + port2.close(); +} + +{ + const histogram = createSlidingWindowHistogram({ + chunks: 2, + recordsPerChunk: 1, + highest: 10, + }); + + // Out-of-range recording attempts count toward count-based rotation. + histogram.record(11); + histogram.record(1); + let current = histogram.snapshot(); + assert.strictEqual(current.count, 1); + assert.strictEqual(current.exceeds, 1); + + histogram.record(2); + current = histogram.snapshot(); + assert.strictEqual(current.count, 2); + assert.strictEqual(current.exceeds, 0); +} + +{ + for (const options of [ + undefined, + null, + {}, + { chunks: 2 }, + { chunks: 2, chunkDuration: 1, recordsPerChunk: 1 }, + ]) { + assert.throws(() => createSlidingWindowHistogram(options), { + code: options?.chunks === undefined ? + 'ERR_INVALID_ARG_TYPE' : 'ERR_INVALID_ARG_VALUE', + }); + } + + for (const chunks of [0, 1025, 1.5, '2']) { + assert.throws(() => createSlidingWindowHistogram({ + chunks, + recordsPerChunk: 1, + }), { + code: typeof chunks === 'number' ? + 'ERR_OUT_OF_RANGE' : 'ERR_INVALID_ARG_TYPE', + }); + } + + for (const chunkDuration of [0, 1.5, 18_446_744_073_710]) { + assert.throws(() => createSlidingWindowHistogram({ + chunks: 2, + chunkDuration, + }), { code: 'ERR_OUT_OF_RANGE' }); + } + + for (const recordsPerChunk of [0, 1.5, Number.MAX_SAFE_INTEGER + 1]) { + assert.throws(() => createSlidingWindowHistogram({ + chunks: 2, + recordsPerChunk, + }), { code: 'ERR_OUT_OF_RANGE' }); + } + + assert.throws(() => createSlidingWindowHistogram({ + chunks: 2, + recordsPerChunk: 1, + lowest: 10, + highest: 10, + }), { code: 'ERR_OUT_OF_RANGE' }); + + for (const bounds of [ + { lowest: 1n }, + { lowest: 1n, highest: 100 }, + { lowest: 1, highest: 100n }, + ]) { + const histogram = createSlidingWindowHistogram({ + chunks: 1, + recordsPerChunk: 1, + ...bounds, + }); + histogram.record(1); + assert.strictEqual(histogram.snapshot().count, 1); + } +} + +(async () => { + const histogram = createSlidingWindowHistogram({ + chunks: 1, + chunkDuration: 100, + highest: 100, + }); + + histogram.record(1); + assert.strictEqual(histogram.snapshot().count, 1); + + await delay(common.platformTimeout(200)); + assert.strictEqual(histogram.snapshot().count, 0); + + histogram.record(2); + const current = histogram.snapshot(); + assert.strictEqual(current.count, 1); + assert.strictEqual(current.min, 2); +})().then(common.mustCall()); diff --git a/typings/internalBinding/performance.d.ts b/typings/internalBinding/performance.d.ts index dc4d1e20c6b3..cf3ef0a664f0 100644 --- a/typings/internalBinding/performance.d.ts +++ b/typings/internalBinding/performance.d.ts @@ -76,6 +76,20 @@ declare namespace InternalPerformanceBinding { subtract(other: Histogram): number; } + class SlidingWindowHistogram { + constructor( + lowest: number | bigint, + highest: number | bigint, + figures: number, + chunks: number, + timeBased: boolean, + rotateAt: bigint, + ); + record(value: number | bigint): void; + snapshot(): Histogram; + reset(): void; + } + interface Constants { NODE_PERFORMANCE_GC_MAJOR: number; NODE_PERFORMANCE_GC_MINOR: number; @@ -116,6 +130,8 @@ type PerformanceObserverCallback = export interface PerformanceBinding { Histogram: typeof InternalPerformanceBinding.Histogram; + SlidingWindowHistogram: + typeof InternalPerformanceBinding.SlidingWindowHistogram; constants: InternalPerformanceBinding.Constants; observerCounts: Uint32Array; milestones: Float64Array; From f7d18ec36060a87a026e3e71d9af224890f9c0a2 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 5 Sep 2026 20:27:04 +0000 Subject: [PATCH 149/217] test: expand histogram test coverage Signed-off-by: James M Snell Assisted-by: Opencode PR-URL: https://github.com/nodejs/node/pull/65825 Reviewed-By: Matteo Collina --- .../test-perf-hooks-histogram-qrde-worker.js | 21 ++++++++ .../test-perf-hooks-histogram-qrde.js | 27 ++++++++++ ...est-perf-hooks-sliding-window-histogram.js | 18 +++++++ .../test-perf-hooks-histogram-heapdump.js | 54 +++++++++++++++++++ 4 files changed, 120 insertions(+) create mode 100644 test/parallel/test-perf-hooks-histogram-qrde-worker.js create mode 100644 test/sequential/test-perf-hooks-histogram-heapdump.js diff --git a/test/parallel/test-perf-hooks-histogram-qrde-worker.js b/test/parallel/test-perf-hooks-histogram-qrde-worker.js new file mode 100644 index 000000000000..ba8366516652 --- /dev/null +++ b/test/parallel/test-perf-hooks-histogram-qrde-worker.js @@ -0,0 +1,21 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { once } = require('events'); +const { Worker } = require('worker_threads'); + +const worker = new Worker(` + const { parentPort } = require('worker_threads'); + const { createHistogram } = require('perf_hooks'); + + const histogram = createHistogram({ highest: 200000, figures: 5 }); + for (let i = 1; i <= 100000; i++) histogram.record(i); + histogram.qrde({ bins: 1000, dequantize: 'all' }); + parentPort.postMessage('scheduled'); +`, { eval: true }); + +(async () => { + assert.deepStrictEqual(await once(worker, 'message'), ['scheduled']); + assert.strictEqual(await worker.terminate(), 1); +})().then(common.mustCall()); diff --git a/test/parallel/test-perf-hooks-histogram-qrde.js b/test/parallel/test-perf-hooks-histogram-qrde.js index a417942d916a..40eb1e71f1eb 100644 --- a/test/parallel/test-perf-hooks-histogram-qrde.js +++ b/test/parallel/test-perf-hooks-histogram-qrde.js @@ -9,6 +9,16 @@ function assertClose(actual, expected, tolerance = 1e-12) { `${actual} != ${expected}`); } +function recordRepeated(histogram, options, value, count) { + const block = createHistogram(options); + block.record(value); + while (count > 0) { + if (count % 2 === 1) histogram.add(block); + count = Math.floor(count / 2); + if (count > 0) block.add(block); + } +} + (async () => { const empty = createHistogram(); const emptyResult = await empty.qrde(); @@ -23,6 +33,9 @@ function assertClose(actual, expected, tolerance = 1e-12) { assert.strictEqual(emptyResult.corrections, 0); assert.strictEqual(emptyResult.dequantize, 'hdr'); + assert.throws(() => empty.qrde.call({}), { + code: 'ERR_INVALID_THIS', + }); assert.throws(() => empty.qrde(null), { code: 'ERR_INVALID_ARG_TYPE', }); @@ -219,4 +232,18 @@ function assertClose(actual, expected, tolerance = 1e-12) { await largeCount.qrde({ bins: 2, dequantize: 'none' }); assert.strictEqual(largeCountResult.count, (1n << 53n) + 1n); assertClose(largeCountResult.quantiles[1], 2); + + // Exercise correction across the exact-to-asymptotic beta CDF threshold. + const correctionOptions = { highest: 131071, figures: 5 }; + const correction = createHistogram(correctionOptions); + recordRepeated(correction, correctionOptions, 1, 26239); + recordRepeated(correction, correctionOptions, 131071, 973761); + const count = 1_000_000; + const threshold = (1 - Math.sqrt(1 - 100_000 / (count + 1))) / 2; + const corrected = await correction.qrde({ + probabilities: [0, threshold - 1e-10, threshold + 1e-10, 1], + dequantize: 'none', + }); + assert.strictEqual(corrected.corrections, 1); + assert.strictEqual(corrected.quantiles[1], corrected.quantiles[2]); })().then(common.mustCall()); diff --git a/test/parallel/test-perf-hooks-sliding-window-histogram.js b/test/parallel/test-perf-hooks-sliding-window-histogram.js index 9e28677e5d62..3ee1ca4ea437 100644 --- a/test/parallel/test-perf-hooks-sliding-window-histogram.js +++ b/test/parallel/test-perf-hooks-sliding-window-histogram.js @@ -49,6 +49,11 @@ const { assert.strictEqual(histogram.snapshot().count, 0); histogram.record(10n); assert.strictEqual(histogram.snapshot().maxBigInt, 10n); + for (const value of [0n, 2n ** 63n]) { + assert.throws(() => histogram.record(value), { + code: 'ERR_OUT_OF_RANGE', + }); + } assert.throws(() => new histogram.constructor(), { code: 'ERR_ILLEGAL_CONSTRUCTOR', @@ -142,6 +147,19 @@ const { highest: 10, }), { code: 'ERR_OUT_OF_RANGE' }); + for (const [name, value] of [ + ['lowest', 0n], + ['lowest', 2n ** 63n], + ['highest', 0n], + ['highest', 2n ** 63n], + ]) { + assert.throws(() => createSlidingWindowHistogram({ + chunks: 2, + recordsPerChunk: 1, + [name]: value, + }), { code: 'ERR_OUT_OF_RANGE' }); + } + for (const bounds of [ { lowest: 1n }, { lowest: 1n, highest: 100 }, diff --git a/test/sequential/test-perf-hooks-histogram-heapdump.js b/test/sequential/test-perf-hooks-histogram-heapdump.js new file mode 100644 index 000000000000..cf310eb973b1 --- /dev/null +++ b/test/sequential/test-perf-hooks-histogram-heapdump.js @@ -0,0 +1,54 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { + createJSHeapSnapshot, + validateByRetainingPathFromNodes, +} = require('../common/heap'); +const { + createHistogram, + createSlidingWindowHistogram, +} = require('perf_hooks'); + +(async () => { + const uncached = createHistogram(); + const cached = createHistogram(); + cached.record(1); + cached.record(1000); + await cached.qrde({ cache: true }); + + const sliding = createSlidingWindowHistogram({ + chunks: 2, + recordsPerChunk: 1, + }); + + const nodes = createJSHeapSnapshot(); + const snapshots = validateByRetainingPathFromNodes( + nodes, + 'Node / Histogram', + [{ node_name: 'Node / qrde_snapshot', edge_name: 'qrde_snapshot' }], + ); + assert.strictEqual(snapshots.length, 1); + assert.ok(snapshots[0].self_size > 0); + + const windows = validateByRetainingPathFromNodes( + nodes, + 'Node / SlidingWindowHistogram', + [], + ); + for (const [edgeName, nodeName] of [ + ['chunks', 'Node / chunks'], + ['generations', 'Node / generations'], + ['spare', 'Node / Histogram'], + ]) { + validateByRetainingPathFromNodes(windows, 'Node / SlidingWindowHistogram', [ + { node_name: nodeName, edge_name: edgeName }, + ]); + } + + // Keep all three wrappers live through snapshot generation. + assert.strictEqual(uncached.count, 0); + assert.strictEqual(cached.count, 2); + assert.strictEqual(sliding.snapshot().count, 0); +})().then(common.mustCall()); From 336f33ccc1b5caa2dc6877319b1ddb9640c3efaf Mon Sep 17 00:00:00 2001 From: James M Snell Date: Tue, 8 Sep 2026 02:16:50 +0000 Subject: [PATCH 150/217] util: implement debounce I found myself using debounce quite a bit recently while testing some recent other additions (quic and dtls testing, perf_hooks improvements, etc). I was using an npm dependency right up until I realized just how generally useful it is to actually have it Just There. So, since it was a holiday and I just felt like it... util.debounce(...) Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/65899 Reviewed-By: Matteo Collina --- doc/api/util.md | 98 +++++++ lib/internal/util/debounce.js | 237 +++++++++++++++++ lib/util.js | 9 + test/parallel/test-util-debounce.js | 381 ++++++++++++++++++++++++++++ 4 files changed, 725 insertions(+) create mode 100644 lib/internal/util/debounce.js create mode 100644 test/parallel/test-util-debounce.js diff --git a/doc/api/util.md b/doc/api/util.md index 77e4ceaa2d80..f06521894336 100644 --- a/doc/api/util.md +++ b/doc/api/util.md @@ -387,6 +387,104 @@ The `--throw-deprecation` command-line flag and `process.throwDeprecation` property take precedence over `--trace-deprecation` and `process.traceDeprecation`. +## `util.debounce(fn, wait[, options])` + + + +* `fn` {Function} The function to debounce. +* `wait` {integer} The number of milliseconds to delay `fn`. +* `options` {Object} + * `leading` {boolean} When `true`, invokes `fn` immediately when a new + debounce window begins. **Default:** `false`. + * `rejectOnCancel` {boolean} When `true`, a call superseded by a later call + rejects with an `AbortError`. **Default:** `false`. + * `signal` {AbortSignal} An `AbortSignal` that cancels pending calls and + prevents future calls when aborted. +* Returns: {Function} The debounced function. + +Creates a function that delays calling `fn` until `wait` milliseconds have +elapsed since the most recent invocation. The debounced function returns a +{Promise} for the value returned by `fn`. If `fn` throws or returns a rejected +promise, the returned promise is rejected with the same reason. + +When the debounced function is called more than once before the delay expires, +`fn` receives the arguments from the most recent call. By default, the promises +from all calls resolve or reject with the result of that invocation. If +`options.rejectOnCancel` is `true`, the promises from superseded calls reject +with an `AbortError` instead. + +When `options.leading` is `true`, the first call in a debounce window invokes +`fn` immediately. Calls made during that window are delayed until `wait` +milliseconds have elapsed since the most recent call. A trailing invocation +only occurs if the debounced function was called again during the window. +The window begins before `fn` is invoked, so recursive calls and calls made +while an asynchronous `fn` is pending are part of the same window if they occur +before the delay expires. This also applies to calls made after a synchronous +`fn` returns but before the delay expires. + +If `options.signal` is aborted, pending and future calls reject with an +`AbortError`, with the signal's reason set as the error's `cause`, and `fn` is +not invoked by those calls. If the signal is already aborted, `debounce()` +throws an `AbortError`. + +The returned function has the following properties: + +* `cancel([reason])` cancels the current debounce window. Its pending promises + reject with an `AbortError`. If provided, `reason` is set as the error's + `cause`. +* `flush()` cancels the delay and invokes `fn` immediately. It has no effect if + no invocation is pending. +* `pending` {Promise|null} is the promise returned by the most recent call in + the current debounce window, or `null` if no invocation is pending. +* `pendingCount` {integer} is the number of calls awaiting the invocation in + the current debounce window. +* `ref()` makes the pending and future timeout keep the Node.js event loop + active. Returns the debounced function. +* `unref()` allows the event loop to exit while a timeout is pending. This also + applies to future timeouts. Returns the debounced function. + +When invoked, `fn` has the debounced function as its `this` value. After a +trailing invocation, a new debounce window can begin even if a promise returned +by `fn` is still pending. The debounced function preserves the `name` and +`length` of `fn`. + +```mjs +import { setTimeout as wait } from 'node:timers/promises'; +import { debounce } from 'node:util'; + +const fn = debounce(async (value) => { + await wait(100); + return value; +}, 50); + +const first = fn(1); +const second = fn(2); + +console.log(await first); // 2 +console.log(await second); // 2 +``` + +A debounced function can be used to trigger an action after a period of +inactivity. Each call resets the timeout: + +```cjs +const { debounce } = require('node:util'); + +const onInactivity = debounce(() => { + console.log('No activity for 5 seconds'); +}, 5_000).unref(); + +process.stdin.on('data', (data) => { + console.log(`Received ${data.length} bytes`); + onInactivity(); +}); + +// Start the initial inactivity timeout. +onInactivity(); +``` + ## `util.diff(actual, expected)` + +* `fn` {Function} The function to throttle. +* `limit` {integer} The maximum number of times to invoke `fn` during an + interval. Must be greater than `0`. +* `interval` {integer} The length of each interval in milliseconds. +* `options` {Object} + * `concurrency` {number} The maximum number of invocations of `fn` whose + return values may be unsettled at once. Must be a positive integer or + `Infinity`. **Default:** `Infinity`. + * `maxPending` {number} The maximum number of calls that may be queued when + `overflow` is `'queue'`. Must be a non-negative integer or `Infinity`. + **Default:** `Infinity`. + * `overflow` {string} Determines how calls exceeding the limit are handled. + **Default:** `'queue'`. + * `'queue'`: Queue calls in the order received. + * `'drop'`: Reject calls immediately without queueing them. + * `signal` {AbortSignal} An `AbortSignal` that cancels pending calls and + prevents future calls when aborted. + * `strict` {boolean} When `true`, ensures that `limit` is not exceeded during + any rolling interval. **Default:** `false`. +* Returns: {Function} The throttled function. + +Creates a function that limits how often `fn` is invoked. By default, calls that +exceed the limit are queued in the order received rather than discarded. The +throttled function returns a {Promise} for the value returned by `fn`. If `fn` +throws or returns a rejected promise, the returned promise is rejected with the +same reason. + +An invocation starts only when both rate and concurrency capacity are +available. Rate capacity is consumed when `fn` starts, not when a call enters +the queue. Concurrency capacity is released when the value returned by `fn` +settles. Non-promise values settle during the next microtask. + +When `options.overflow` is `'drop'`, calls made without available rate or +concurrency capacity are rejected immediately. When `options.overflow` is +`'queue'` and `options.maxPending` calls are already queued, additional calls +are also rejected immediately. `maxPending` has no effect when `overflow` is +`'drop'`. + +In both cases, rejected calls return a promise rejected with an +`ERR_THROTTLED` error. The rejected promise is marked as handled, so ignoring it +does not emit an `'unhandledRejection'` event. Awaiting or explicitly handling +the promise still observes the rejection. Rejected calls do not consume rate +or concurrency capacity, enter the queue, or schedule a timeout. + +By default, the interval begins when the first call in a new window invokes +`fn`. Up to `limit` calls can invoke `fn` during that window. Queued calls are +processed in groups of up to `limit` as each subsequent window begins. This +windowed behavior can result in calls occurring close together at a window +boundary. + +When `options.strict` is `true`, invocation times are tracked individually. +This ensures that no more than `limit` calls begin during any rolling interval, +at the cost of additional bookkeeping. + +If `options.signal` is aborted, pending and future calls reject with an +`AbortError`, with the signal's reason set as the error's `cause`, and `fn` is +not invoked by those calls. If the signal is already aborted, `throttle()` +throws an `AbortError`. + +The returned function has the following properties: + +* `cancel([reason])` cancels all queued calls and resets the current throttle + window. The queued promises reject with an `AbortError`. If provided, + `reason` is set as the error's `cause`. Does not cancel invocations that have + already started. +* `hasImmediateCapacity()` returns `true` if a call made at that moment could + invoke `fn` without being queued or rejected. The check does not reserve + capacity, and the throttled function always checks again when called. It + returns `false` while calls are queued to preserve their order. Callers can + avoid creating a timeout by only calling the throttled function when this + method returns `true`. +* `pending` {Promise|null} is the promise returned by the most recently queued + call, or `null` if no invocation is queued. +* `pendingCount` {integer} is the number of calls awaiting invocation. +* `activeCount` {integer} is the number of invocations whose return values have + not settled. +* `ref()` makes the pending and future timeout keep the Node.js event loop + active. Returns the throttled function. +* `unref()` allows the event loop to exit while a timeout is pending. This also + applies to future timeouts. Returns the throttled function. + +Calls that have already invoked `fn` are not affected by `cancel()` or by an +aborted signal. When invoked, `fn` has the throttled function as its `this` +value. The throttled function preserves the `name` and `length` of `fn`. + +```mjs +import { throttle } from 'node:util'; + +const request = throttle(async (id) => { + const response = await fetch(`https://example.com/items/${id}`); + return response.json(); +}, 2, 1_000); + +// At most two requests begin during each one-second interval. All other calls +// remain queued and retain their original arguments. +const results = await Promise.all([ + request(1), + request(2), + request(3), + request(4), +]); +``` + ## `util.diff(actual, expected)` -> Stability: 1 - Experimental +> Stability: 1.1 - Active Development @@ -74,20 +74,73 @@ added: v26.9.0 * `options` {Object} * `cert` {string|Buffer} Server certificate in PEM format. **Required.** * `key` {string|Buffer} Server private key in PEM format. **Required.** + * `secureContext` {DTLSSecureContext} A context from + [`dtls.createSecureContext()`][] to use instead of building one from the + credential options below. Must have been created with `isServer: true`. + Cannot be combined with any option the context already carries. + * `sni` {Object|Function} Server Name Indication. A map of host names to the + identity to serve them with, or a function returning one. Cannot be + combined with `secureContext`; set it on the context instead. See + [Server Name Indication][]. + * `passphrase` {string} Passphrase to decrypt `key`, if it is encrypted. + Ignored when `key` is not encrypted. Unlike `key` and `cert`, this must be + a string, matching [`tls.createSecureContext()`][]. * `port` {number} Port to bind to. **Required.** * `host` {string} Address to bind to. **Default:** `'0.0.0.0'`. * `ca` {string|Buffer|string\[]|Buffer\[]} CA certificates in PEM format. * `ciphers` {string} OpenSSL cipher list string. - * `alpn` {string\[]|Buffer} ALPN protocol names. + * `alpn` {string\[]|Buffer} ALPN protocol names. Each name must be between + 1 and 255 bytes. A `Buffer` must already be in ALPN wire format: one + length byte followed by that many bytes, repeated. * `srtp` {string} Colon-separated SRTP protection profile names (e.g., `'SRTP_AES128_CM_SHA1_80:SRTP_AEAD_AES_128_GCM'`). - * `requestCert` {boolean} Request client certificate. **Default:** `false`. - * `mtu` {number} Maximum transmission unit for DTLS records. - **Default:** `1200`. + * `requestCert` {boolean} Request a certificate from the client. + **Default:** `false`. + * `rejectUnauthorized` {boolean} Only has an effect together with + `requestCert`. When `true`, a client that presents no certificate, or one + that does not chain to a trusted CA, is rejected during the handshake and + receives a TLS alert. When `false`, the certificate is still requested and + verified but the handshake completes regardless, leaving the decision to + the application via [`session.authorized`][]. **Default:** `true`. + * `mtu` {number} Maximum size in bytes of a DTLS datagram. **Default:** + `1200`. + * `handshakeTimeout` {number} Milliseconds a handshake may take before it is + abandoned. `0` disables it. **Default:** `60000`. See + [Handshake timeout][]. + * `ipv6Only` {boolean} When `true`, an IPv6 endpoint serves IPv6 only. When + `false`, binding `'::'` also accepts IPv4 peers, which arrive with mapped + addresses such as `'::ffff:203.0.113.1'` -- anything keyed on the peer + address, including `maxSessionsPerHost`, sees them in that form. Has no + effect on an IPv4 endpoint. **Default:** `false`. + * `reusePort` {boolean} When `true`, sets `SO_REUSEPORT`, so several + processes may bind the same port and the kernel spreads arriving + datagrams between them. Every one of them must set it. **Default:** + `false`. + * `udpReceiveBufferSize` {number} Size in bytes for the socket's receive + buffer (`SO_RCVBUF`). Raising it gives the endpoint room for bursts that + the default would drop. The kernel clamps this to its own maximum. + **Default:** the system default. + * `udpSendBufferSize` {number} Size in bytes for the socket's send buffer + (`SO_SNDBUF`). Clamped as above. **Default:** the system default. + * `udpTTL` {number} IP time-to-live for outgoing datagrams, from `1` to + `255`. **Default:** the system default. + * `maxSessions` {number} The maximum number of concurrent sessions the + endpoint will hold. Set to `0` for no limit. **Default:** `10000`. + * `maxSessionsPerHost` {number} The maximum number of concurrent sessions + from any single source IP address, ignoring port. Set to `0` for no limit. + **Default:** `1000`. + * `sessionIdContext` {string} Opaque identifier scoping resumable sessions + to this server, at most 32 bytes. **Default:** a value derived from + `process.argv`, as in `tls.createServer()`. * Returns: {DTLSEndpoint} Creates a DTLS server bound to the specified address and port. The server -uses automatic HMAC-based cookie exchange for DoS protection. +uses automatic HMAC-based cookie exchange for DoS protection. See +[Denial of service][]. + +Binding failures are thrown with the code the operating system gave, as in +`net` and `dgram`: an address already in use throws an error whose `code` is +`'EADDRINUSE'`, with `errno` and `syscall` set. ```mjs import { listen } from 'node:dtls'; @@ -117,26 +170,50 @@ console.log('DTLS server listening on', endpoint.address); added: v26.9.0 --> -* `host` {string} Remote host to connect to. +* `host` {string} Remote host to connect to, as an IPv4 or IPv6 literal. + Host names are not resolved. * `port` {number} Remote port to connect to. * `options` {Object} * `ca` {string|Buffer|string\[]|Buffer\[]} CA certificates in PEM format. * `cert` {string|Buffer} Client certificate in PEM format. * `key` {string|Buffer} Client private key in PEM format. + * `secureContext` {DTLSSecureContext} A context from + [`dtls.createSecureContext()`][] to use instead of building one from the + credential options below. Must **not** have been created with + `isServer: true`. Cannot be combined with any option the context already + carries. + * `psk` {Object|Function} A pre-shared key as `{ identity, key }`, or a + function returning one. See [Pre-shared keys][]. + * `session` {Buffer} A session from [`session.session`][] on an earlier + connection, to resume rather than handshake in full. See + [Session resumption][]. + * `passphrase` {string} Passphrase to decrypt `key`, if it is encrypted. + Ignored when `key` is not encrypted. Unlike `key` and `cert`, this must be + a string, matching [`tls.createSecureContext()`][]. * `rejectUnauthorized` {boolean} When `true`, the server's certificate must both chain to a trusted CA and match the expected identity (`servername`, or `host` when `servername` is not set); otherwise the handshake is - aborted and `session.opened` rejects. When `false`, the certificate is not - verified. **Default:** `true`. + aborted and `session.opened` rejects. When `false`, the certificate is + still verified and the handshake completes regardless, leaving the + decision to the application via [`session.authorized`][] and + [`session.authorizationError`][]. **Default:** `true`. * `servername` {string} Server name used for the SNI (Server Name Indication) extension and as the identity checked during certificate verification. **Default:** the `host` argument. Set to `''` to disable SNI. SNI is never sent for IP address literals. - * `bindHost` {string} Local bind address. **Default:** `'0.0.0.0'`. + * `bindHost` {string} Local bind address. **Default:** `'::'` when `host` is an + IPv6 literal, otherwise `'0.0.0.0'`. The local socket must be in the same + address family as the peer. * `bindPort` {number} Local bind port. **Default:** `0` (ephemeral). - * `alpn` {string\[]|Buffer} ALPN protocol names. + * `alpn` {string\[]|Buffer} ALPN protocol names. Each name must be between + 1 and 255 bytes. A `Buffer` must already be in ALPN wire format: one + length byte followed by that many bytes, repeated. * `srtp` {string} SRTP protection profile names. - * `mtu` {number} Maximum transmission unit. **Default:** `1200`. + * `mtu` {number} Maximum size in bytes of a DTLS datagram. **Default:** + `1200`. + * `handshakeTimeout` {number} Milliseconds a handshake may take before it is + abandoned and `session.opened` rejects. `0` disables it. **Default:** + `60000`. See [Handshake timeout][]. * Returns: {DTLSSession} Connects to a DTLS server. Returns a `DTLSSession` whose `opened` property @@ -146,7 +223,7 @@ is a `Promise` that resolves when the handshake completes. import { connect } from 'node:dtls'; import { readFileSync } from 'node:fs'; -const session = connect('localhost', 4433, { +const session = connect('127.0.0.1', 4433, { ca: [readFileSync('ca-cert.pem')], }); @@ -158,6 +235,419 @@ session.onmessage = (data) => { }; ``` +## `dtls.createSecureContext([options])` + + + +* `options` {Object} + * `alpn` {string\[]} ALPN protocols. + * `ca` {string|Buffer|Array} CA certificates in PEM format. When omitted, + the bundled default certificate authorities are used. + * `cert` {string|Buffer} Certificate in PEM format. + * `ciphers` {string} OpenSSL cipher suite list. + * `ecdhCurve` {string} Named curve or curve list for ECDH. + * `isServer` {boolean} Build a context for a server. **Default:** `false`. + * `key` {string|Buffer} Private key in PEM format. + * `passphrase` {string} Passphrase for `key`, if it is encrypted. + * `rejectUnauthorized` {boolean} Verification behaviour, as for + [`dtls.listen()`][] and [`dtls.connect()`][]. + * `requestCert` {boolean} Request a certificate from the peer. Servers only. + * `sessionIdContext` {string} Session id context. Servers only. + * `sni` {Object|Function} Server Name Indication. Servers only. See + [Server Name Indication][]. + * `psk` {Object|Function} Pre-shared keys. See [Pre-shared keys][]. + * `pskIdentityHint` {string} Identity hint to advertise, naming which key a + client should pick. Requires `psk`. Servers only. + * `srtp` {string} SRTP profile list. + * `ticketKeys` {Buffer} Session ticket keys, for resuming sessions across + endpoints and restarts. Servers only. See [Session resumption][]. +* Returns: {DTLSSecureContext} + +Options marked "Servers only" require `isServer: true`. Passing one to a +client context throws `ERR_INVALID_ARG_VALUE`, rather than being ignored or +applied where it can have no effect. + +Creates a reusable secure context. Pass it to [`dtls.listen()`][] or +[`dtls.connect()`][] as `secureContext` in place of the credential options. + +A context holds a parsed certificate and key and, when `ca` is given, its own +certificate store; roughly 28 KiB in total. Building one per connection is +therefore expensive in memory rather than in time -- two thousand of them cost +about 54 MiB, against 2 MiB when a single context is shared. Clients opening +many connections should build the context once. + +The peer identity checked during verification is **not** part of the context. +It is bound to each connection from `servername` (or the host), so one context +can be used against different peers and still reject the wrong certificate. + +`isServer` is fixed when the context is created, because it selects the +underlying OpenSSL method. Passing a server context to [`dtls.connect()`][], +or a client context to [`dtls.listen()`][], throws. + +```mjs +import { connect, createSecureContext, listen } from 'node:dtls'; +import { readFileSync } from 'node:fs'; + +const serverContext = createSecureContext({ + cert: readFileSync('server-cert.pem'), + key: readFileSync('server-key.pem'), + isServer: true, +}); + +// One context, several endpoints. +const a = listen(onsession, { secureContext: serverContext, port: 5684 }); +const b = listen(onsession, { secureContext: serverContext, port: 5685 }); + +const clientContext = createSecureContext({ + ca: readFileSync('ca-cert.pem'), +}); + +// One context, many connections, each verified against its own name. +const s1 = connect('192.0.2.1', 5684, { + secureContext: clientContext, + servername: 'a.example.com', +}); +const s2 = connect('192.0.2.2', 5684, { + secureContext: clientContext, + servername: 'b.example.com', +}); +``` + +## Server Name Indication + +An endpoint can serve more than one identity by giving `listen()` an `sni` +map, or a function. Each key of a map is a host name and each value is either +a +[`DTLSSecureContext`][] created with `isServer: true`, or a plain object of +the same options [`dtls.createSecureContext()`][] takes: + +```mjs +import { createSecureContext, listen } from 'node:dtls'; +import { readFileSync } from 'node:fs'; + +const endpoint = listen(onsession, { + cert: readFileSync('default-cert.pem'), + key: readFileSync('default-key.pem'), + port: 5684, + sni: { + 'api.example.com': { + cert: readFileSync('api-cert.pem'), + key: readFileSync('api-key.pem'), + }, + 'www.example.com': createSecureContext({ + cert: readFileSync('www-cert.pem'), + key: readFileSync('www-key.pem'), + isServer: true, + }), + '*': { + cert: readFileSync('default-cert.pem'), + key: readFileSync('default-key.pem'), + }, + }, +}); +``` + +The `'*'` key is the fallback, used when the client's name matches nothing and +when the client sends no name at all. **Without it, an unmatched name is +refused with an `unrecognized_name` alert** rather than falling back to the +endpoint's own `cert` and `key`; providing an `sni` map is taken to mean that +only the names in it are served. [`tls.createServer()`][] differs here: its +`SNICallback` falls back to the default identity silently. + +Verification follows the selected identity, so an entry carrying its own `ca` +accepts only client certificates issued under it. `requestCert` and +`rejectUnauthorized` are not per-identity: they belong to the endpoint and +apply to every name it serves. + +A function may be given instead of a map, for identities that are chosen +rather than enumerated: + +```mjs +listen(onsession, { + port: 5684, + cert, + key, + sni: (servername) => contexts.get(servername), +}); +``` + +It is called with the name the client asked for, or `undefined` if the client +sent no SNI extension, and returns what a map entry holds: a +[`dtls.createSecureContext()`][] result or the options to build one. Returning +nothing declines the name, which is refused exactly as an unmatched map with no +`'*'` entry is, rather than falling back to the endpoint's own certificate. + +The function runs during the handshake and must return synchronously, so it +cannot consult a database. Returning a prepared context is worth doing: +building one from options parses the certificate again on every handshake. + +An exception thrown by the function fails that handshake and is reported to the +session's error handler, like any other handshake failure. It does not reach +the process as an uncaught exception. + +The certificate and the cipher list both follow the selected context. +Pre-shared keys do not. OpenSSL installs the PSK callbacks on the connection +when it is created, before any name is known, and selecting an identity does +not replace them, so the keys a server accepts are always the endpoint's own. +A `psk` given on an SNI identity is never consulted, and an identity cannot be +served over PSK alone. + +`sni` belongs to the secure context rather than to the endpoint, so it can be +given to [`dtls.createSecureContext()`][] and cannot be combined with a +`secureContext` that already exists. Applying it to a prepared context would +reconfigure that context for every endpoint sharing it, and the identities a +server serves are part of what its context is. + +A connection refused for an unrecognized name still reaches the `listen()` +callback: the session exists once the client's address is validated, which +happens before the name is examined. It then fails like any other handshake +failure. + +## Denial of service + +Cookie exchange proves a peer can receive at its claimed address, but it does +not limit how many sessions that peer may then establish, and each session +holds a TLS state machine, two buffers and a timer. `maxSessions` bounds the +total; `maxSessionsPerHost` is what prevents one peer from taking all of it. +A peer refused by either cap is answered with silence rather than an alert, +because replying to an address that has not completed cookie exchange would +create an amplification vector; a legitimate client retransmits and is +admitted once there is room. Refusals are counted by +[`endpointStats.serverRefusedCount`][]. + +Deployments serving many clients behind a single NAT may need to raise +`maxSessionsPerHost`. + +## Handshake timeout + +A handshake that never finishes is abandoned after `handshakeTimeout` +milliseconds, and its session error is `DTLS handshake timeout`. + +OpenSSL already gives up on its own, but only after twelve retransmits on a +doubling backoff capped at 60 seconds -- around eight minutes in total. Until +then the session holds its place against `maxSessions` (see +[Denial of service][]), +so handshakes that are started and abandoned can occupy an endpoint for the +cost of starting them. That needs no spoofing: the peer completes the cookie +exchange and then simply stops. + +The two limits coexist and whichever comes first ends the handshake. The +retransmit schedule itself is untouched, deliberately -- compressing it to +force earlier failure would cause spurious retransmissions on exactly the +lossy links DTLS is meant for. + +The timeout covers resumed and PSK handshakes as well, and stops applying once +the handshake completes; it is not an idle timeout. + +A handshake can stall without either peer being at fault or aware. +DTLS discards records it cannot authenticate rather than answering them +(RFC 6347 section 4.1.2.1), so a mismatched pre-shared key or a cipher list +with nothing in common produces silence rather than an alert. This timeout is +what ends those. + +## Pre-shared keys + +DTLS can authenticate with a key both peers already hold instead of a +certificate (RFC 4279). This is how it is usually deployed to constrained +devices, which frequently have no certificate at all. + +A server gives the identities it accepts; a client gives the one it is. No +certificate is needed on either side: + +```mjs +import { connect, listen } from 'node:dtls'; + +const endpoint = listen(onsession, { + port: 5684, + psk: { 'device-42': deviceKey }, +}); + +const client = connect('192.0.2.1', 5684, { + psk: { identity: 'device-42', key: deviceKey }, +}); +``` + +Either side may pass a function instead, for keys that are looked up or +derived rather than known up front. A server's is called with the identity the +client offered and returns the key, or nothing to refuse it. A client's is +called with the server's identity hint, if it sent one, and returns +`{ identity, key }`: + +```mjs +listen(onsession, { + port: 5684, + psk: (identity) => deriveKey(masterSecret, identity), +}); +``` + +The callback runs during the handshake and must return synchronously, so it +cannot consult a database. Where both are given, the map is checked first and +the callback is only reached when the map has no answer -- a configuration +using only the map never runs JavaScript inside the handshake. + +An exception thrown by the callback fails that handshake and is reported to +the session's error handler. It does not reach the process as an uncaught +exception. + +### Cipher suites + +The default cipher list excludes PSK, so giving `psk` without `ciphers` +enables the PSK suites. Supplying `ciphers` disables that and uses exactly +what was asked for. + +A server keeps the certificate suites as well, since it may serve both kinds +of client on one port. A client does not: a client that configured a +pre-shared key and no CA wants the key, and leaving the certificate suites +enabled would let a server choose one, failing the handshake while verifying a +certificate the caller never meant to rely on. + +Forward-secret PSK key exchanges are preferred over plain PSK of the same +strength. Plain PSK derives its keys from the shared secret alone, so anyone +who later learns that key can decrypt traffic they recorded earlier. `RSA-PSK` +is excluded: it needs a certificate and adds no forward secrecy. + +CoAP requires `TLS_PSK_WITH_AES_128_CCM_8` (RFC 7252), whose 64-bit +authentication tag OpenSSL rejects at security level 1 and above. Node.js +default is above it, so that suite has to be asked for explicitly and with the +security level lowered: + +```mjs +listen(onsession, { port: 5684, psk, ciphers: 'PSK-AES128-CCM8@SECLEVEL=0' }); +``` + +### Failure modes + +A wrong key does not produce an error. The identity only names the key, so the +handshake proceeds and the two sides derive different secrets; the first +record that fails authentication is then discarded rather than answered, since +DTLS discards invalid records instead of replying to them (RFC 6347 section +4.1.2.1). Neither peer is told anything and both retransmit. + +A cipher list with nothing in common behaves the same way, which is what makes +the `CCM8` case above present as a stall rather than a rejection. Both are +ended by [`handshakeTimeout`][], after 60 seconds by default. + +An identity the server does not recognise is refused outright, and the client +sees the handshake fail. + +## Session resumption + +A resumed handshake skips the server's certificate, which matters more here +than it does over TCP: the `Certificate` flight is fragmented across several +datagrams, and losing any one of them costs a retransmission timeout. Measured +on loopback, a full handshake has the server send 1850 bytes in 4 packets +against 280 bytes in 3 for a resumed one. + +A client reads [`session.session`][] once the session is open and passes it to +a later [`dtls.connect()`][]: + +```mjs +import { connect } from 'node:dtls'; + +const first = connect('192.0.2.1', 5684, { ca, servername: 'device.example' }); +await first.opened; +const ticket = first.session; // Buffer. +await first.close(); + +const second = connect('192.0.2.1', 5684, { + ca, + servername: 'device.example', + session: ticket, +}); +await second.opened; +console.log(second.reused); // True. +``` + +A session that the server will not accept -- expired, or issued by a different +endpoint -- is not an error. The handshake simply proceeds in full, and +[`session.reused`][] is `false`. + +The cookie exchange still happens for a resumed handshake, so resumption is not +a way around the address validation described under [Denial of service][]. + +### Binding to the authenticated host + +A session may only be resumed against the identity it was authenticated for -- +the `servername`, or the host when there is none. Reusing it for anything else +throws. + +This is not a convenience check. A resumed handshake does not re-send or +re-verify the peer's certificate; it inherits the authenticated identity of the +original session. Replaying a session against a different host would therefore +skip verification while appearing to succeed. For the same reason a `session` +that did not come from [`session.session`][] is rejected outright: nothing +records which identity it belongs to, so it cannot be checked. + +### Resuming under `rejectUnauthorized` + +A session carries the verification result it was established with, so a session +established with `rejectUnauthorized: false` cannot be resumed by a connection +that asked for a verified peer. The handshake fails: + +```mjs +import { connect } from 'node:dtls'; + +// Connected without verifying anything. +const first = connect('192.0.2.1', 5684, { rejectUnauthorized: false }); +await first.opened; +console.log(first.authorized); // False. +const ticket = first.session; +await first.close(); + +const second = connect('192.0.2.1', 5684, { + rejectUnauthorized: true, + session: ticket, +}); +await second.opened; // Rejects: verification failed. +``` + +The host is the same in both, so binding the session to its authenticated +identity does not cover this on its own; what differs is whether the caller +asked for the peer to be verified. Because a resumed handshake runs no +verification of its own, the recorded result is re-checked once it completes, +and a session whose peer never verified is refused wherever verification is +required. [`session.authorized`][] and [`session.authorizationError`][] report +the recorded result on a resumed session either way. + +### Ticket keys + +The key that encrypts session tickets is generated at random for each context, +so by default a ticket is only good for the endpoint that issued it and only +until the process restarts. Give every endpoint the same `ticketKeys` to let +tickets be resumed across a restart or a cluster: + +```mjs +import { listen } from 'node:dtls'; +import { randomBytes } from 'node:crypto'; + +const ticketKeys = randomBytes(80); // Share this between processes. +const endpoint = listen(onsession, { cert, key, port: 5684, ticketKeys }); +``` + +The length is OpenSSL's: a key name followed by an HMAC key and an AES key. It +differs from the 48 bytes [`tls.createServer()`][] uses, which is a layout +`node:tls` defines for itself. Supplying the wrong length throws and reports +the length expected. + +Ticket keys are long-lived secrets. Anyone holding them can decrypt tickets and +recover the sessions they protect, so treat them as key material and rotate +them. + +## Class: `DTLSSecureContext` + + + +An opaque, reusable bundle of credentials and TLS settings, created by +[`dtls.createSecureContext()`][]. It cannot be constructed directly. + +### `secureContext.isServer` + +* Returns: {boolean} `true` if the context was created for a server. + ## Class: `DTLSEndpoint` + +* Type: {bigint} The number of datagrams discarded before a handshake was + attempted because they could not be a ClientHello. Read only. + +Datagrams arriving at a listening endpoint that do not match an existing +session are screened for the shape of a DTLS ClientHello record before any +state is allocated for them. A steadily rising value indicates junk or scan +traffic rather than failing clients, which are counted as sessions that never +complete. + +### `endpointStats.serverRefusedCount` + + + +* Type: {bigint} The number of otherwise valid handshake attempts refused + because the endpoint was at `maxSessions` or the peer was at + `maxSessionsPerHost`. Read only. + ### `endpointStats.isConnected` + +* Returns: {X509Certificate|undefined} The peer's certificate, or `undefined` + if the peer sent none. + +An [`X509Certificate`][] for the peer's leaf certificate. The issuer chain is +reachable through its `issuerCertificate` property, and the parsed fields -- +`subject`, `issuer`, `validFrom`, `validTo`, `fingerprint256`, `serialNumber` +and the rest -- are properties of that object. + +Where [`tls.TLSSocket.getPeerCertificate()`][] returns a plain dictionary with +`valid_from`, `valid_to` and a chain walked through `issuerCertificate`, this +returns the same `X509Certificate` class that +[`tls.TLSSocket.getPeerX509Certificate()`][] does. Call `toLegacyObject()` on +it to get the dictionary form. + +The same object is returned on every access once the peer's certificate is +available. + +### `session.session` + + + +* Returns: {Buffer|undefined} An opaque session for resuming this connection + later, or `undefined` on a server session or before the handshake completes. + +Pass it as the `session` option to a later [`dtls.connect()`][]. It is bound to +the host this connection authenticated against and is refused elsewhere; see +[Session resumption][]. + +Server sessions return `undefined`: a server has no identity to bind the value +to, and it is the client that carries a session between connections. + +### `session.reused` + + + +* Returns: {boolean} `true` if this connection resumed an earlier session + rather than performing a full handshake. + +Like [`session.authorized`][], this reads `false` once the session is closed. + +### `session.authorized` + + + +* Returns: {boolean} `true` if the peer presented a certificate chain that + verified against the configured certificate authorities, and, for a client, + matched the requested identity. `false` before the handshake completes. + +### `session.authorizationError` + + + +* Returns: {string|undefined} The short X509 verification error code, for + example `'CERT_HAS_EXPIRED'` or `'HOSTNAME_MISMATCH'`, or `undefined` if the + peer's chain verified. + +A peer that presented no certificate at all reports +`'UNABLE_TO_GET_ISSUER_CERT'`, so this can be used to distinguish "no +certificate" from "a certificate that failed to verify". + +The chain is verified even when `rejectUnauthorized` is `false`; the result is +simply not enforced. That makes these two properties the way to apply a custom +authorization policy: + +```mjs +import { connect } from 'node:dtls'; + +const session = connect('192.0.2.1', 4433, { + ca: [caCert], + servername: 'example.com', + rejectUnauthorized: false, +}); + +await session.opened; + +if (!session.authorized && session.authorizationError !== 'CERT_HAS_EXPIRED') { + await session.close(); +} +``` ### `session.alpnProtocol` -* Returns: {string|undefined} The negotiated ALPN protocol. +* Returns: {string|undefined} The negotiated ALPN protocol, or `undefined` if + ALPN was not used. + +If a server has `alpn` configured and a client offers only protocols the +server does not support, the server sends a fatal `no_application_protocol` +alert and the handshake fails, as required by [RFC 7301][] section 3.2. A +server with no `alpn` configured declines the extension instead, and the +handshake completes with no protocol negotiated. ### `session.srtpProfile` @@ -388,7 +1043,8 @@ live and updated as data flows through the session. ### `session.exportKeyingMaterial(length, label[, context])` -* `length` {number} Number of bytes to export. +* `length` {number} Number of bytes to export. Must be an integer between + `1` and `65536`. * `label` {string} The label for the exported keying material. * `context` {Buffer} Optional context value. * Returns: {Buffer} @@ -397,6 +1053,45 @@ Exports keying material from the DTLS session, as defined in [RFC 5705][]. This is commonly used with DTLS-SRTP to derive encryption keys for media streams. +Throws `ERR_OUT_OF_RANGE` if `length` is outside the accepted range. The upper +bound is not imposed by [RFC 5705][]; it exists so that a caller cannot request +an arbitrarily large allocation, and is far above what any defined exporter +needs (DTLS-SRTP uses 60 bytes). + +### Callback properties + +#### `session.onmessage` + +* {Function} + * `data` {Buffer} + +Set to receive application data from the peer. + +#### `session.onerror` + +* {Function} + * `error` {Error} + +Set to receive error notifications. + +#### `session.onhandshake` + +* {Function} + * `protocol` {string} + +Set to receive handshake completion notifications. + +#### `session.onkeylog` + +* {Function} + * `line` {string} + +Set to receive TLS key log lines (for debugging with Wireshark). + +### `session[Symbol.asyncDispose]()` + +Equivalent to calling `session.close()`. + ## Class: `DTLSSession.Stats` + +* `source` {string} A directory or an archive file to mount and run. + +Requires [`--experimental-vfs`][]. May be given at most once. + +Mounts `source` exactly as [`--vfs-mount`][] does, and additionally runs the +entry point and all subsequent `require()`/`import` resolution against that +mount rather than the real file system. The entry point is taken from the mount +the same way `node ` takes one: the mount's own `package.json` +`"main"`, or `index.js`. Any positional command-line argument is the program's +own (available from `process.argv[2]` onward), never an entry-point override. + +`process.argv[1]` reports `source` rather than the reserved mount point, since +the mount point is an opaque implementation detail. + +Mounting the same source twice mounts it twice, at two separate mount points. +The entry point then comes from the mount `--vfs-load` itself contributed, not +from an earlier `--vfs-mount` of the same source. + +In worker threads `--vfs-load` mounts but does not load: a worker inherits the +same mounts, in the same order, and runs its own entry point. + +`--vfs-load` is not permitted in [`NODE_OPTIONS`][]: which entry point runs is +the command line's decision, and the environment must not be able to redirect +it. + +```console +$ node --experimental-vfs --vfs-load=app.zip +$ node --experimental-vfs --vfs-mount=lib.zip --vfs-load=app.zip +``` + +### `--vfs-mount=source` + + + +* `source` {string} A directory or an archive file to mount. + +Requires [`--experimental-vfs`][]. May be repeated to mount several sources. + +Mounts `source` as a virtual file system ([`node:vfs`][]). Each mount is placed +at a reserved mount point assigned by Node.js, so mounts never shadow real +paths and no target can be chosen. Mounting alone does not change the entry +point; use [`--vfs-load`][] for the source to run from. + +`--vfs-mount` and [`--vfs-load`][] mount in the order they are written, so + +```console +$ node --experimental-vfs --vfs-mount=a --vfs-load=b --vfs-mount=c +``` + +mounts `a`, `b` and `c` in that order and runs `b`. Mounts contributed by +[`NODE_OPTIONS`][] are mounted before the command line's. + +The provider backing a source is chosen from the source itself rather than from +its file name: + +* A directory is mounted with a [`RealFSProvider`][] rooted there. +* A file whose bytes are a ZIP archive is mounted with a [`ZipProvider`][], so + an archive can carry any name. + +Providers registered with `vfs.registerProvider()` (typically from a module +preloaded with [`--require`][] or [`--import`][]) are consulted first, in +reverse registration order, and may claim directories as well as files. If no +provider claims the source, Node.js exits with an error. + ### `--watch` + +* `entry` {Object} + * `name` {string} A short identifier, used in diagnostics. + * `canHandle` {Function} Called with the resolved path and its + [`fs.Stats`][]. Returns `true` if this provider should back the source. + * `create` {Function} Called with the resolved path and its [`fs.Stats`][]. + Returns the {VirtualProvider} backing the source. + +Registers a provider that [`--vfs-mount`][] can select for a source it +recognizes, so a file format Node.js has no built-in provider for can still be +mounted. + +A source is claimed by the first provider whose `canHandle()` returns `true`. +Registered providers are consulted before the built-in ones, newest +registration first, and are offered directories as well as files, so a +registered provider can back, wrap, or vet any source. If none claims the +source, the built-in providers handle it: a directory with +[`RealFSProvider`][], and a file whose bytes are a ZIP archive with +[`ZipProvider`][]. + +Providers must be registered before the mounts are created. Register from a +module preloaded with [`--require`][] or [`--import`][]: + +```cjs +// provider.js, preloaded with --require +const fs = require('node:fs'); +const vfs = require('node:vfs'); + +const MAGIC = Buffer.from('CUSTOMFMT'); + +vfs.registerProvider({ + name: 'customfmt', + canHandle(path, stats) { + if (!stats.isFile()) return false; + const head = Buffer.alloc(MAGIC.length); + const fd = fs.openSync(path, 'r'); + try { + fs.readSync(fd, head, 0, MAGIC.length, 0); + } finally { + fs.closeSync(fd); + } + return head.equals(MAGIC); + }, + create(path) { + return new MyCustomProvider(path); + }, +}); +``` + +```console +$ node --experimental-vfs --require ./provider.js \ + --vfs-load archive.customfmt +``` + ## Class: `VirtualFileSystem` * `source` {string} A directory or an archive file to mount and run. @@ -3812,7 +3812,7 @@ $ node --experimental-vfs --vfs-mount=lib.zip --vfs-load=app.zip ### `--vfs-mount=source` * `source` {string} A directory or an archive file to mount. diff --git a/doc/api/crypto.md b/doc/api/crypto.md index 76c1db64eee4..a549755b4ddd 100644 --- a/doc/api/crypto.md +++ b/doc/api/crypto.md @@ -5476,7 +5476,7 @@ console.log(Buffer.from(derivedKey).toString('hex')); // '24156e2...5391653' ### `crypto.parsePKCS12(bundle[, options])` * `bundle` {ArrayBuffer|Buffer|TypedArray|DataView} A DER-encoded PKCS#12 diff --git a/doc/api/dtls.md b/doc/api/dtls.md index 50f23e8444d8..44e090dc5bf3 100644 --- a/doc/api/dtls.md +++ b/doc/api/dtls.md @@ -238,7 +238,7 @@ session.onmessage = (data) => { ## `dtls.createSecureContext([options])` * `options` {Object} @@ -638,7 +638,7 @@ them. ## Class: `DTLSSecureContext` An opaque, reusable bundle of credentials and TLS settings, created by @@ -788,7 +788,7 @@ added: v26.9.0 ### `endpointStats.serverRejectedCount` * Type: {bigint} The number of datagrams discarded before a handshake was @@ -803,7 +803,7 @@ complete. ### `endpointStats.serverRefusedCount` * Type: {bigint} The number of otherwise valid handshake attempts refused @@ -924,7 +924,7 @@ parsing either. ### `session.peerX509Certificate` * Returns: {X509Certificate|undefined} The peer's certificate, or `undefined` @@ -947,7 +947,7 @@ available. ### `session.session` * Returns: {Buffer|undefined} An opaque session for resuming this connection @@ -963,7 +963,7 @@ to, and it is the client that carries a session between connections. ### `session.reused` * Returns: {boolean} `true` if this connection resumed an earlier session @@ -974,7 +974,7 @@ Like [`session.authorized`][], this reads `false` once the session is closed. ### `session.authorized` * Returns: {boolean} `true` if the peer presented a certificate chain that @@ -984,7 +984,7 @@ added: REPLACEME ### `session.authorizationError` * Returns: {string|undefined} The short X509 verification error code, for diff --git a/doc/api/ffi.md b/doc/api/ffi.md index e3ecdf93626e..2632bbf4ceed 100644 --- a/doc/api/ffi.md +++ b/doc/api/ffi.md @@ -211,7 +211,7 @@ const path = `libsqlite3.${suffix}`; * `path` {string|Buffer|URL} diff --git a/doc/api/perf_hooks.md b/doc/api/perf_hooks.md index 0edd91b9bba6..024eb138dd42 100644 --- a/doc/api/perf_hooks.md +++ b/doc/api/perf_hooks.md @@ -1721,7 +1721,7 @@ Returns a {RecordableHistogram}. ## `perf_hooks.createSlidingWindowHistogram(options)` * `options` {Object} @@ -2493,7 +2493,7 @@ efficient pass over the histogram data. More efficient than calling ### `histogram.qrde([options])` * `options` {Object} @@ -2743,7 +2743,7 @@ are clamped to zero. ## Class: `SlidingWindowHistogram` Records values into a lazily rotated ring of histogram chunks. Instances are @@ -2757,7 +2757,7 @@ call `snapshot()` to materialize the current window as a {Histogram}. ### `slidingWindowHistogram.record(val)` * `val` {number|bigint} The amount to record. @@ -2769,7 +2769,7 @@ exceed the configured `highest` value. ### `slidingWindowHistogram.reset()` Invalidates all chunks in the current window. Allocated chunks are reset @@ -2778,7 +2778,7 @@ lazily when reused. ### `slidingWindowHistogram.snapshot()` * Returns: {Histogram} diff --git a/doc/api/quic.md b/doc/api/quic.md index 6f347dfec976..486b348370b5 100644 --- a/doc/api/quic.md +++ b/doc/api/quic.md @@ -1871,7 +1871,7 @@ added: v23.8.0 ### `stream.opened` * Type: {Promise} diff --git a/doc/api/sqlite.md b/doc/api/sqlite.md index fd0fd0c9c195..f9d58ab3f54e 100644 --- a/doc/api/sqlite.md +++ b/doc/api/sqlite.md @@ -1100,7 +1100,7 @@ bound. Binding any other value throws an `ERR_INVALID_ARG_TYPE` error. * `fn` {Function} The function to debounce. @@ -488,7 +488,7 @@ onInactivity(); ## `util.throttle(fn, limit, interval[, options])` * `fn` {Function} The function to throttle. @@ -1901,7 +1901,7 @@ equality. ## `util.markPromiseAsHandled(promise)` * `promise` {promise} The promise to mark as handled diff --git a/doc/api/vfs.md b/doc/api/vfs.md index bc4a94f80e8b..18f6ff87b4bc 100644 --- a/doc/api/vfs.md +++ b/doc/api/vfs.md @@ -96,7 +96,7 @@ const realVfs = vfs.create(new vfs.RealFSProvider('/tmp/vfs-root')); ## `vfs.registerProvider(entry)` * `entry` {Object} diff --git a/doc/api/webcrypto.md b/doc/api/webcrypto.md index cee4eaf5262b..0611f4bb608c 100644 --- a/doc/api/webcrypto.md +++ b/doc/api/webcrypto.md @@ -2,7 +2,7 @@