diff --git a/changelog.d/11331-tokio-lane-l3-db-wrappers.md b/changelog.d/11331-tokio-lane-l3-db-wrappers.md new file mode 100644 index 0000000000..06aef137d7 --- /dev/null +++ b/changelog.d/11331-tokio-lane-l3-db-wrappers.md @@ -0,0 +1,5 @@ +**compile: drop the dead pg / mysql2 `async-runtime` rule (tokio lane L3).** The auto-optimize driver used to select perry-stdlib's tokio `async-runtime` for `pg`, `mysql2` and `mysql2/promise` by module name. That rule could never run. The pg and mysql2 wrappers, with their `Handle::current().block_on` decline paths, were deleted in #10677 and #10680, so `well_known_bindings.toml` has no row for either module, and the flip loop skips any module without a row at `lookup_well_known`. The rule now covers `mongodb` only, and the new test `well_known::tests::pg_and_mysql2_have_no_well_known_row` fails if either module gets a row again. Prose that still described the old setup is updated in `stdlib_features.rs`, `freshness.rs`, `perry-stdlib/Cargo.toml`, the `perry-db-turnloop` slot docs and `scripts/tokio_inventory.json`. + +With lane L2 (#11277), an auto-optimized pg-only or mysql2-only program now links no tokio. Measured on Linux x86_64 with pg 8.23.0 and mysql2 3.24.4: 0 tokio members in the stdlib archive and 0 `tokio-1.` strings in the binary. Main before L2 had 16 and 51–52. `PERRY_NO_AUTO_OPTIMIZE` builds still link tokio through `full`. + +These failures also happen on main and are not fixed here: pg scram-sha-256 authentication fails (md5 and trust work); mysql2 segfaults during `createConnection` on Linux; and pg on a `worker_threads` agent intermittently throws `reading 'setBuffer'`. diff --git a/crates/perry-db-turnloop/src/lib.rs b/crates/perry-db-turnloop/src/lib.rs index 7d21812946..0bc609079d 100644 --- a/crates/perry-db-turnloop/src/lib.rs +++ b/crates/perry-db-turnloop/src/lib.rs @@ -138,9 +138,12 @@ pub mod subsystem { //! the database slots, and `perry-runtime`'s `turnloop_net::sink` carries //! the full map. The ceiling is `MAX_SUBSYSTEMS` (16). - /// `perry-ext-pg`. + /// Formerly `perry-ext-pg` (removed, #10677: npm `pg` now compiles from + /// source over `net`). Kept reserved so the database band never renumbers + /// and a stale archive cannot collide. pub const PG: u8 = 9; - /// `perry-ext-mysql2`. + /// Formerly `perry-ext-mysql2` (removed, #10680: npm `mysql2` now compiles + /// from source over `net`). Kept reserved, as `PG`. pub const MYSQL: u8 = 10; /// Formerly `perry-ext-ioredis` (removed: npm `redis`/`ioredis`/`iovalkey` /// now compile from source over `net`). Kept reserved so the database diff --git a/crates/perry-stdlib/Cargo.toml b/crates/perry-stdlib/Cargo.toml index 887fdd46bf..ff71bd5439 100644 --- a/crates/perry-stdlib/Cargo.toml +++ b/crates/perry-stdlib/Cargo.toml @@ -196,13 +196,14 @@ external-tls-server = [ # turnloop P8 group H: the bundled `pg` / `mysql2` / `ioredis` / `mongodb` # copies are GONE. They were the pre-#466 in-stdlib implementations, kept as # the fallback the well-known flip declined to. Every `extern "C"` symbol they -# defined is also defined — and strictly extended — by perry-ext-pg / -# perry-ext-mysql2 / perry-ext-ioredis (since removed) / perry-ext-mongodb, which is the only -# path the flip has routed those imports down since v0.5.565-568. Keeping them -# cost perry-stdlib a `sqlx` + `redis` + `mongodb` manifest edge, and 76 of the -# 502 packages in this crate's own dependency closure (502 -> 426), for code no -# default compile reached. Cargo.lock itself does NOT shrink: perry-ext-pg / -# -mysql2 / -ioredis / -mongodb keep the same drivers. `database-sqlite` is +# defined was also defined — and strictly extended — by perry-ext-pg / +# perry-ext-mysql2 / perry-ext-ioredis / perry-ext-mongodb, the only path the +# flip had routed those imports down since v0.5.565-568. Keeping them cost +# perry-stdlib a `sqlx` + `redis` + `mongodb` manifest edge, and 76 of the 502 +# packages in this crate's own dependency closure (502 -> 426), for code no +# default compile reached. The pg / mysql2 / ioredis wrappers have since been +# removed too (the npm packages compile from source over `net`); only +# perry-ext-mongodb remains. `database-sqlite` is # NOT in that set: rusqlite is not a tokio dependency and `dispatch_sqlite_stmt` # is deliberately retained through the flip (Refs #643). database = ["database-sqlite"] diff --git a/crates/perry/src/commands/compile/optimized_libs/driver.rs b/crates/perry/src/commands/compile/optimized_libs/driver.rs index d30ee3b414..50cbbb0259 100644 --- a/crates/perry/src/commands/compile/optimized_libs/driver.rs +++ b/crates/perry/src/commands/compile/optimized_libs/driver.rs @@ -301,7 +301,7 @@ pub(crate) fn build_optimized_libs( ); } // Multiple import spellings can resolve to one archive - // (`mysql2` + `mysql2/promise`, `http` + `https`). Keep the + // (`http` + `https` + `http2`). Keep the // Cargo package set and link line unique; alias multiplicity // is not a different build graph. if !tokio_using_bindings @@ -380,16 +380,23 @@ pub(crate) fn build_optimized_libs( }) { features.insert("async-bridge"); } - // turnloop P8 group H: the bundled pg / mysql2 / mongodb - // modules were deleted, so `module_to_features` names no feature - // for them and the check above cannot see them. The wrappers still - // settle every promise through perry-stdlib's `perry_ffi_*` shim, - // which only compiles under `async-runtime` — key it on the module - // name, the same way `undici` / `nodemailer` / `fastify` do below. - if matches!( - module_normalized, - "pg" | "mysql2" | "mysql2/promise" | "mongodb" - ) { + // turnloop P8 group H: the bundled mongodb module was deleted, so + // `module_to_features` names no feature for it and the check above + // cannot see it. perry-ext-mongodb still settles every promise + // through perry-stdlib's `perry_ffi_*` shim and runs each operation + // on tokio (`Handle::current().block_on` inside + // `perry_ffi_spawn_blocking`), so key `async-runtime` on the module + // name. + // + // pg / mysql2 used to be listed here too. Their wrappers + // (perry-ext-pg #10677, perry-ext-mysql2 #10680) are gone and + // `well_known_bindings.toml` has no row for either, so this loop — + // which `continue`s at `lookup_well_known` for any module without a + // row — could never reach them: those imports compile the real npm + // package, whose sockets are `net` / `tls` and select whatever + // those select. `well_known::tests::pg_and_mysql2_have_no_well_known_row` + // pins that. + if module_normalized == "mongodb" { features.insert("async-runtime"); } // `undici` (#466): perry-ext-undici is thin glue over the diff --git a/crates/perry/src/commands/compile/optimized_libs/freshness.rs b/crates/perry/src/commands/compile/optimized_libs/freshness.rs index d322256530..116ff7ac10 100644 --- a/crates/perry/src/commands/compile/optimized_libs/freshness.rs +++ b/crates/perry/src/commands/compile/optimized_libs/freshness.rs @@ -145,7 +145,7 @@ pub(crate) fn auto_optimized_cache_key( // dependency graphs (#9470; the same class produced #9094's Linux link). // // Sort and deduplicate here as a defensive measure: aliases such as - // `mysql2` + `mysql2/promise` name the same wrapper and must + // `http` + `https` name the same wrapper and must // describe the same graph regardless of discovery order or alias // multiplicity. let mut tokio_bindings: Vec = tokio_using_bindings @@ -818,9 +818,9 @@ pub(crate) fn binding_needs_shared_tokio(module: &str) -> bool { /// compilation), but asking for `async-runtime` on their behalf would put /// tokio back into every net / ws program for nothing. /// -/// `pg` / `mysql2` are not here because they are not in the co-build set -/// either; the driver selects `async-runtime` for their decline paths by -/// module name. +/// `pg` / `mysql2` are not here and need nothing: their wrappers were removed +/// (#10677 / #10680), so the npm packages compile from source over `net` / +/// `tls` and their decline paths are gone with them. pub(crate) fn binding_bundles_tokio(module: &str) -> bool { matches!( module, diff --git a/crates/perry/src/commands/compile/well_known.rs b/crates/perry/src/commands/compile/well_known.rs index e70d2dfed9..dcb922151e 100644 --- a/crates/perry/src/commands/compile/well_known.rs +++ b/crates/perry/src/commands/compile/well_known.rs @@ -376,6 +376,22 @@ mod tests { assert_eq!(binding.lib, "perry_ext_undici"); } + #[test] + fn pg_and_mysql2_have_no_well_known_row() { + // optimized_libs/driver.rs no longer selects perry-stdlib's + // tokio `async-runtime` for `pg` / `mysql2` by module name: the flip + // loop only reaches modules with a row here, and their wrappers were + // removed (#10677 / #10680), so the npm packages compile from source. + // If a wrapper ever comes back, whether it needs tokio is + // `binding_needs_shared_tokio`'s call, not a module-name rule. + for module in ["pg", "mysql2", "mysql2/promise", "node:pg"] { + assert!( + lookup_well_known(module).is_none(), + "{module} must not route to a native wrapper" + ); + } + } + #[test] fn node_prefix_stripped_on_lookup() { let bare = lookup_well_known("bcrypt"); diff --git a/crates/perry/src/commands/stdlib_features.rs b/crates/perry/src/commands/stdlib_features.rs index 05f247f8d3..4c08ca5bd7 100644 --- a/crates/perry/src/commands/stdlib_features.rs +++ b/crates/perry/src/commands/stdlib_features.rs @@ -64,14 +64,13 @@ pub fn module_to_features(module: &str) -> &'static [&'static str] { // ── Databases ───────────────────────────────────────────────── // pg / mysql2 / mongodb need no perry-stdlib feature: the - // bundled copies were deleted in turnloop P8 group H, so these imports - // are served entirely by perry-ext-pg / perry-ext-mysql2 / - // perry-ext-mongodb via the well-known flip — the - // same shape `fastify` and `node:http` already have above. Their - // `async-runtime` requirement (the `perry_ffi_*` shim each wrapper - // settles its promises through) is re-asserted in + // bundled copies were deleted in turnloop P8 group H. `mongodb` is + // served by perry-ext-mongodb via the well-known flip, and its + // `async-runtime` requirement is re-asserted in // optimized_libs/driver.rs rather than named here, because everything - // named here gets STRIPPED by the flip loop. + // named here gets STRIPPED by the flip loop. `pg` / `mysql2` have no + // wrapper at all any more (#10677 / #10680): the real npm packages + // compile from source over `net` / `tls`, so they need nothing here. "mysql2" | "mysql2/promise" => &[], "pg" => &[], "better-sqlite3" => &["database-sqlite"], @@ -291,9 +290,9 @@ mod tests { fn bundled_database_copies_map_to_no_stdlib_features() { // turnloop P8 group H deleted perry-stdlib's bundled pg / mysql2 / // mongodb modules. Naming a feature here would ask cargo - // for a gate that no longer exists; the wrappers own these imports - // outright, and their `async-runtime` need is re-asserted by the - // flip loop in optimized_libs/driver.rs. + // for a gate that no longer exists. mongodb's wrapper owns its import + // and its `async-runtime` need is re-asserted by the flip loop in + // optimized_libs/driver.rs; pg / mysql2 compile the npm package. for module in ["pg", "mysql2", "mysql2/promise", "mongodb", "node:mongodb"] { assert_eq!( module_to_features(module), diff --git a/scripts/tokio_inventory.json b/scripts/tokio_inventory.json index f3ecd79e98..64d706fe81 100644 --- a/scripts/tokio_inventory.json +++ b/scripts/tokio_inventory.json @@ -56,8 +56,8 @@ "optional": true, "target": null, "surface": "the tokio HALF of the async bridge only (turnloop P8 lane L split it out): `common::tokio_bridge` \u2014 the current-thread `RUNTIME`, its wait-driver tick, `spawn` / `spawn_for_promise*` \u2014 plus the `perry_ffi_spawn_async` / `perry_ffi_spawn_blocking_with_reactor` C ABI and `perry_ffi_spawn_blocking`'s tokio-pool arm. The promise bridge itself (`common::async_bridge`'s settle queue and pump, and the promise / pool / blocking `perry_ffi_*` shims) is tokio-free under the `async-bridge` feature, which is what the auto-optimize driver now force-enables. `web-fetch`, the `node:tls` server (`tls-runtime`, on turnloop handles since lane L part 2) and the external net / ws adapters compile against that bridge only.", - "reached_when": "the `async-runtime` feature, which is selected only by (1) a Cargo feature whose code hands tokio a future \u2014 `bundled-net` (and so the `net` / `tls` umbrellas) and `bundled-ws` (tokio sockets); or (2) the auto-optimize driver, for every wrapper that still bundles tokio (`binding_bundles_tokio`: mongodb) and for pg / mysql2 / mongodb by module name. `full` still implies it, so every PERRY_NO_AUTO_OPTIMIZE / prebuilt-archive build links tokio. Lane L part 2 took `web-fetch`, `tls-runtime`, `external-tls-server`, `external-net-tls`, `external-net-pump`, `external-ws-pump` and (after lane D, #11265) `external-http-server-pump` / `external-http-client-pump` off it (the `node:tls` server's sockets are turnloop handles, and perry-ext-net / perry-ext-ws carry no tokio), so an auto-optimized program whose network imports are only `fetch`, `net`, `tls` (both route to perry-ext-net, with the TLS server here), `ws` and `http` / `https` / `http2` links no tokio \u2014 on top of part 1's crypto, bcrypt, argon2, zlib, readline, worker_threads, timers and UI programs, and lane K's `container`.", - "blocker": "each selector in `reached_when` has to go; then `tokio_bridge.rs`, `tls_stream.rs` and the three `cfg(feature = \"async-runtime\")` shims in `perry_ffi_async.rs` delete whole and `async-runtime` collapses into `async-bridge` \u2014 nothing in `async_bridge.rs` has to move. In order: (1) B removes the db wrappers' decline paths (pg / mysql2 / mongodb), which call `Handle::current()` inside `perry_ffi_spawn_blocking`. (2) P1 for the two bundled socket modules left \u2014 `net/mod.rs` (bundled `net.Socket`, `tls.connect`, `upgradeToTLS`) and `ws.rs`, both reached only with the well-known flip disabled or through `full`; `tls/turnloop_server.rs` is the worked example for moving a perry-stdlib socket surface onto `turnloop_net` with `perry-tls-session` above it. K took `container` off tokio (`perry_container_compose::rt`). PerryTS/turnloop#42 is NOT a blocker: `Occupancy::Long` shipped in turnloop 0.1.0-alpha.5, and the tokio-free `perry_ffi_spawn_blocking` runs on it through `turnloop_pool::submit_long`.", + "reached_when": "the `async-runtime` feature, which is selected only by (1) a Cargo feature whose code hands tokio a future \u2014 `bundled-net` (and so the `net` / `tls` umbrellas) and `bundled-ws` (tokio sockets); or (2) the auto-optimize driver, for every wrapper that still bundles tokio (`binding_bundles_tokio`: mongodb) and for mongodb by module name. pg / mysql2 have had no wrapper since #10677 / #10680, and lane L3 deleted their unreachable module-name rule; their npm packages compile from source over `net` / `tls`, so an auto-optimized pg- or mysql2-only program links no tokio either. `full` still implies it, so every PERRY_NO_AUTO_OPTIMIZE / prebuilt-archive build links tokio. Lane L part 2 took `web-fetch`, `tls-runtime`, `external-tls-server`, `external-net-tls`, `external-net-pump`, `external-ws-pump` and (after lane D, #11265) `external-http-server-pump` / `external-http-client-pump` off it (the `node:tls` server's sockets are turnloop handles, and perry-ext-net / perry-ext-ws carry no tokio), so an auto-optimized program whose network imports are only `fetch`, `net`, `tls` (both route to perry-ext-net, with the TLS server here), `ws` and `http` / `https` / `http2` links no tokio \u2014 on top of part 1's crypto, bcrypt, argon2, zlib, readline, worker_threads, timers and UI programs, and lane K's `container`.", + "blocker": "each selector in `reached_when` has to go; then `tokio_bridge.rs`, `tls_stream.rs` and the three `cfg(feature = \"async-runtime\")` shims in `perry_ffi_async.rs` delete whole and `async-runtime` collapses into `async-bridge` \u2014 nothing in `async_bridge.rs` has to move. In order: (1) B: the only db decline path left is perry-ext-mongodb's (`Handle::current().block_on` inside `perry_ffi_spawn_blocking`), which goes by deleting that wrapper; pg's and mysql2's went with their wrappers (#10677 / #10680). (2) P1 for the two bundled socket modules left \u2014 `net/mod.rs` (bundled `net.Socket`, `tls.connect`, `upgradeToTLS`) and `ws.rs`, both reached only with the well-known flip disabled or through `full`; `tls/turnloop_server.rs` is the worked example for moving a perry-stdlib socket surface onto `turnloop_net` with `perry-tls-session` above it. K took `container` off tokio (`perry_container_compose::rt`). PerryTS/turnloop#42 is NOT a blocker: `Occupancy::Long` shipped in turnloop 0.1.0-alpha.5, and the tokio-free `perry_ffi_spawn_blocking` runs on it through `turnloop_pool::submit_long`.", "issue": "unfiled \u2014 P8; PerryTS/turnloop#42 closed (Occupancy::Long, turnloop 0.1.0-alpha.5+); lane L split the bridge from the runtime (#11115) and moved fetch / the TLS server / the net+ws adapters off tokio (part 2)", "plan": "L" },