From aab9c58de23a30dd0ec416af7a99b5d7979a0ce3 Mon Sep 17 00:00:00 2001 From: Qiiks Date: Thu, 17 Sep 2026 14:06:39 +0530 Subject: [PATCH 1/3] floor: probe driver API and compute capability via a short-lived worker subprocess --- README.md | 31 ++++ crates/synapse-engine-cuda/src/cuda.rs | 61 ++++++- crates/synapse-engine-cuda/src/lib.rs | 13 ++ crates/synapse-module/src/lib.rs | 236 ++++++++++++++++++++++--- crates/synapse-worker-cuda/src/main.rs | 24 +++ 5 files changed, 343 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index a6c81afb..c0d6f527 100644 --- a/README.md +++ b/README.md @@ -69,3 +69,34 @@ Example user-tier `~/.config/cortexkit/synapse.jsonc` (project configs must omit Tests can point at a file with `SYNAPSE_CONFIG_PATH`. Only one synapse module per machine (singleton lease); a second instance refuses to start. + +### Owned-CUDA hardware floor + +`ck-synapse-worker-cuda` implements `--probe-floor` (hidden, like the +`--test-abort*` surfaces). It prints one JSON object and exits 0: + +```json +{"driver_api": 13030, "compute_capability": {"major": 8, "minor": 9}} +``` + +The module probes the configured `worker_bin`, the engine's worker-binary +environment override, or the sibling `ck-synapse-worker-cuda`, in that order. +It caches one result per process unless both environment readings parse +successfully. The child wait is bounded to 10 seconds; stdout is capped at +4096 bytes and each pipe completion wait is bounded to another 100 ms. +A missing binary, non-zero exit, timeout, or invalid output produces +`HardwareUnavailable`. Refusal and model evidence carry diagnostic context, +including the last 4096 bytes of stderr when available, under `observed`. +Failed probes do not fabricate numeric hardware readings. + +The environment overrides the probe only as a complete, parseable pair. +Otherwise both readings come from the probe; partial overrides are not merged: + +- `SYNAPSE_CUDA_DRIVER_API` (alias `CUDA_DRIVER_API`) — the raw CUDA **driver + API** integer from `cuDriverGetVersion()`, not the marketing driver version. + For example, a measured driver API value is `13030`. `610.88` is not a valid + API integer; without a parseable alias, it causes fallback to the probe. +- `SYNAPSE_CUDA_COMPUTE_CAPABILITY` (alias `CUDA_COMPUTE_CAPABILITY`) — device + 0's compute capability as `major.minor`, for example `8.9`. +- `SYNAPSE_CUDA_PACKAGING_DRIVER` — optional; the driver string a packaging + build was tested against, carried into the refusal for diagnostics. diff --git a/crates/synapse-engine-cuda/src/cuda.rs b/crates/synapse-engine-cuda/src/cuda.rs index 7ba10341..b7bc4cfd 100644 --- a/crates/synapse-engine-cuda/src/cuda.rs +++ b/crates/synapse-engine-cuda/src/cuda.rs @@ -116,6 +116,51 @@ mod enabled { Ok(()) } + /// Read the driver API version and device 0's compute capability. + /// + /// This runs before an owned-CUDA load is admitted, so it deliberately + /// touches nothing else: no context is retained, no model is loaded, and + /// no weights are mapped. The reading is what the floor predicate is + /// applied to, which is why it reports the raw numbers rather than a + /// verdict. + pub fn probe_hardware_floor() -> Result { + cuda_driver_check(unsafe { cuInit(0) }, "cuInit")?; + let mut driver_api = 0; + cuda_driver_check( + unsafe { cuDriverGetVersion(&mut driver_api) }, + "cuDriverGetVersion", + )?; + let mut device = 0; + cuda_driver_check(unsafe { cuDeviceGet(&mut device, 0) }, "cuDeviceGet")?; + let mut major = 0; + cuda_driver_check( + unsafe { + cuDeviceGetAttribute( + &mut major, + CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, + device, + ) + }, + "cuDeviceGetAttribute(COMPUTE_CAPABILITY_MAJOR)", + )?; + let mut minor = 0; + cuda_driver_check( + unsafe { + cuDeviceGetAttribute( + &mut minor, + CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, + device, + ) + }, + "cuDeviceGetAttribute(COMPUTE_CAPABILITY_MINOR)", + )?; + Ok(crate::HardwareFloorProbe { + driver_api: driver_api as u32, + compute_major: major as u32, + compute_minor: minor as u32, + }) + } + pub struct MiniLmContext { binding: DeviceBinding, raw: NonNull, @@ -465,8 +510,16 @@ mod enabled { } } + /// `CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR` from `cuda.h`. + const CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR: i32 = 75; + /// `CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR` from `cuda.h`. + const CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR: i32 = 76; + unsafe extern "C" { fn cuInit(flags: u32) -> i32; + fn cuDriverGetVersion(version: *mut i32) -> i32; + fn cuDeviceGet(device: *mut i32, ordinal: i32) -> i32; + fn cuDeviceGetAttribute(value: *mut i32, attrib: i32, device: i32) -> i32; fn cuCtxGetDevice(device: *mut i32) -> i32; fn cuCtxSetCurrent(context: *mut c_void) -> i32; fn cuDevicePrimaryCtxRetain(context: *mut *mut c_void, device: i32) -> i32; @@ -551,6 +604,10 @@ mod enabled { bail!("owned CUDA requires a non-macOS build with cargo feature `cuda`") } + pub fn probe_hardware_floor() -> Result { + bail!("owned CUDA requires a non-macOS build with cargo feature `cuda`") + } + pub struct MiniLmContext; impl MiniLmContext { pub fn new(_graphs: bool) -> Result { @@ -629,4 +686,6 @@ mod enabled { } } -pub use enabled::{ensure_available, MiniLmContext, ModernBertContext, Qwen3Context}; +pub use enabled::{ + ensure_available, probe_hardware_floor, MiniLmContext, ModernBertContext, Qwen3Context, +}; diff --git a/crates/synapse-engine-cuda/src/lib.rs b/crates/synapse-engine-cuda/src/lib.rs index 6de832d3..3a97141c 100644 --- a/crates/synapse-engine-cuda/src/lib.rs +++ b/crates/synapse-engine-cuda/src/lib.rs @@ -18,6 +18,8 @@ use synapse_core::{ mod cuda; mod model; +pub use cuda::probe_hardware_floor; + pub const ENGINE_VERSION: &str = "owned-cuda-v1"; /// The source revision from which the CUDA kernels were ported. pub const KERNEL_REVISION: &str = "4d0ded67c30286fe2be37cc7413359ad745dd751"; @@ -183,6 +185,17 @@ pub fn build_identity(family: ModelFamily, dtype: StorageDType) -> CudaBuildIden } } +/// A hardware-floor reading taken before any owned-CUDA worker is spawned. +/// +/// Carried separately from [`device_meets_floor`] so the caller can log or +/// refuse on the observed values rather than on a bare boolean. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +pub struct HardwareFloorProbe { + pub driver_api: u32, + pub compute_major: u32, + pub compute_minor: u32, +} + /// Hardware-floor predicate used by capability probes before worker creation. #[must_use] pub fn device_meets_floor(driver_api: u32, compute_major: u32, compute_minor: u32) -> bool { diff --git a/crates/synapse-module/src/lib.rs b/crates/synapse-module/src/lib.rs index 8043393b..43024e15 100644 --- a/crates/synapse-module/src/lib.rs +++ b/crates/synapse-module/src/lib.rs @@ -5388,7 +5388,7 @@ fn load_catalog_model_blocking( spec.model_id ))); } - ensure_owned_cuda_floor()?; + ensure_owned_cuda_floor(spec.worker_bin.as_deref())?; } let model_path = locator_path(&spec.model_locator, &model_cache)?; let tokenizer_path = locator_path(&spec.tokenizer_locator, &model_cache)?; @@ -5991,7 +5991,7 @@ fn locator_path( } } -fn owned_cuda_floor_decision() -> CudaFloorDecision { +fn owned_cuda_floor_decision(worker: Option<&Path>) -> CudaFloorDecision { let driver_api = ["SYNAPSE_CUDA_DRIVER_API", "CUDA_DRIVER_API"] .into_iter() .find_map(|name| { @@ -6008,14 +6008,139 @@ fn owned_cuda_floor_decision() -> CudaFloorDecision { }); let packaging_driver = env::var("SYNAPSE_CUDA_PACKAGING_DRIVER").ok(); let (Some(driver_api), Some((major, minor))) = (driver_api, compute) else { - return CudaFloorDecision::Unsupported { - reason: synapse_core::CudaUnsupportedReason::HardwareUnavailable, - observed: None, + // The environment is the override; when it is silent, ask the worker. + // The module deliberately does not link the CUDA driver, so the probe + // has to run in the worker process and report its numbers back. + return match owned_cuda_probe_floor(worker) { + Ok(reading) => evaluate_cuda_floor( + reading.driver_api, + reading.compute_major, + reading.compute_minor, + packaging_driver, + ), + Err(_) => CudaFloorDecision::Unsupported { + reason: synapse_core::CudaUnsupportedReason::HardwareUnavailable, + observed: None, + }, }; }; evaluate_cuda_floor(driver_api, major, minor, packaging_driver) } +/// A hardware reading reported by `ck-synapse-worker-cuda --probe-floor`. +#[derive(Clone, Copy, Debug)] +struct OwnedCudaFloorReading { + driver_api: u32, + compute_major: u32, + compute_minor: u32, +} + +static OWNED_CUDA_PROBE: OnceLock> = OnceLock::new(); + +/// Cache one short-lived worker probe; a complete environment override skips it. +fn owned_cuda_probe_floor(worker: Option<&Path>) -> &'static Result { + OWNED_CUDA_PROBE.get_or_init(|| { + let worker = worker + .map(Path::to_path_buf) + .or_else(|| env::var_os(worker_binary_env_var(CUDA_WORKER_ENGINE)).map(PathBuf::from)) + .or_else(|| resolve_worker_binary_sibling(CUDA_WORKER_ENGINE)) + .ok_or_else(|| "CUDA floor probe worker binary not found".to_string())?; + let mut command = std::process::Command::new(worker); + command.arg("--probe-floor"); + run_owned_cuda_probe(&mut command, Duration::from_secs(10)) + }) +} + +fn run_owned_cuda_probe( + command: &mut std::process::Command, + timeout: Duration, +) -> Result { + command + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + let mut child = command + .spawn() + .map_err(|error| format!("spawn CUDA floor probe: {error}"))?; + let stdout = child.stdout.take().expect("piped stdout"); + let mut stderr = child.stderr.take().expect("piped stderr"); + let (stdout_tx, stdout_rx) = std::sync::mpsc::sync_channel(1); + let (stderr_tx, stderr_rx) = std::sync::mpsc::sync_channel(1); + std::thread::spawn(move || { + let mut bytes = Vec::new(); + let result = stdout.take(4097).read_to_end(&mut bytes).map(|_| bytes); + let _ = stdout_tx.send(result); + }); + std::thread::spawn(move || { + let mut tail = Vec::new(); + let mut chunk = [0_u8; 4096]; + while let Ok(count) = stderr.read(&mut chunk) { + if count == 0 { + break; + } + let discard = (tail.len() + count).saturating_sub(4096); + tail.drain(..discard); + tail.extend_from_slice(&chunk[..count]); + } + let _ = stderr_tx.send(String::from_utf8_lossy(&tail).into_owned()); + }); + let deadline = std::time::Instant::now() + timeout; + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break Ok(status), + Ok(None) if std::time::Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(20)); + } + other => { + let _ = child.kill(); + let _ = child.wait(); + break Err(match other { + Err(error) => format!("wait for CUDA floor probe: {error}"), + _ => "CUDA floor probe timed out".to_string(), + }); + } + } + }; + // Bound pipe completion too: a descendant may still hold an inherited pipe. + let stderr = stderr_rx + .recv_timeout(Duration::from_millis(100)) + .unwrap_or_default(); + let fail = |reason: String| format!("{reason}; stderr: {stderr}"); + let status = status.map_err(&fail)?; + if !status.success() { + return Err(fail(format!("CUDA floor probe exited {status}"))); + } + let stdout = stdout_rx + .recv_timeout(Duration::from_millis(100)) + .map_err(|error| fail(format!("CUDA floor probe stdout: {error}")))? + .map_err(|error| fail(format!("read CUDA floor probe stdout: {error}")))?; + if stdout.len() > 4096 { + return Err(fail( + "CUDA floor probe stdout exceeds 4096 bytes".to_string(), + )); + } + let parsed: Value = serde_json::from_slice(&stdout) + .map_err(|error| fail(format!("invalid CUDA floor probe JSON: {error}")))?; + let reading = || { + Some(OwnedCudaFloorReading { + driver_api: parsed.get("driver_api")?.as_u64()?.try_into().ok()?, + compute_major: parsed + .get("compute_capability")? + .get("major")? + .as_u64()? + .try_into() + .ok()?, + compute_minor: parsed + .get("compute_capability")? + .get("minor")? + .as_u64()? + .try_into() + .ok()?, + }) + }; + reading().ok_or_else(|| fail("invalid CUDA floor probe hardware fields".to_string())) +} + fn parse_compute_capability(value: &str) -> Option<(u32, u32)> { let mut parts = value.trim().split('.'); let major = parts.next()?.parse().ok()?; @@ -6023,21 +6148,41 @@ fn parse_compute_capability(value: &str) -> Option<(u32, u32)> { parts.next().is_none().then_some((major, minor)) } -fn ensure_owned_cuda_floor() -> Result<(), WireOperationError> { - let decision = owned_cuda_floor_decision(); +fn owned_cuda_floor_observed(decision: &CudaFloorDecision) -> Value { + floor_observed_with_probe_error( + decision, + OWNED_CUDA_PROBE + .get() + .and_then(|result| result.as_ref().err()) + .map(String::as_str), + ) +} + +fn floor_observed_with_probe_error(decision: &CudaFloorDecision, error: Option<&str>) -> Value { + match decision { + CudaFloorDecision::Supported { observed } + | CudaFloorDecision::Unsupported { + observed: Some(observed), + .. + } => serde_json::to_value(observed).unwrap_or(Value::Null), + CudaFloorDecision::Unsupported { observed: None, .. } => error + .map(|stderr| json!({ "probe_stderr": stderr })) + .unwrap_or(Value::Null), + } +} + +fn ensure_owned_cuda_floor(worker: Option<&Path>) -> Result<(), WireOperationError> { + let decision = owned_cuda_floor_decision(worker); if decision.is_supported() { return Ok(()); } - let observed = match &decision { - CudaFloorDecision::Unsupported { observed, .. } => observed, - CudaFloorDecision::Supported { .. } => unreachable!(), - }; + let observed = owned_cuda_floor_observed(&decision); Err(WireOperationError::from_stable( StableError::owned_cuda_unsupported(), format!( "owned-cuda floor refused before worker creation: decision={}, observed={}", decision.refusal_code().unwrap_or("owned_cuda_unsupported"), - serde_json::to_string(observed).unwrap_or_else(|_| "null".to_string()), + observed, ), )) } @@ -6046,15 +6191,8 @@ fn owned_cuda_evidence(state: &ModuleState, model: &EmbeddingModel) -> Option serde_json::to_value(observed).ok(), - CudaFloorDecision::Unsupported { observed: None, .. } => None, - }; + let decision = owned_cuda_floor_decision(None); + let observed = owned_cuda_floor_observed(&decision); Some(json!({ "engine": CUDA_WORKER_ENGINE, "backend": model.engine_identity.build_flags.get("backend"), @@ -15003,6 +15141,62 @@ fn now_ms() -> u64 { #[cfg(test)] mod tests { use super::*; + #[test] + fn cuda_floor_probe_retains_child_failure_and_rejects_bad_json() { + let mut failed = std::process::Command::new(if cfg!(windows) { "cmd.exe" } else { "sh" }); + if cfg!(windows) { + failed.args(["/D", "/C", "echo driver unavailable 1>&2 & exit /b 7"]); + } else { + failed.args(["-c", "echo 'driver unavailable' >&2; exit 7"]); + } + let error = run_owned_cuda_probe(&mut failed, Duration::from_secs(2)).unwrap_err(); + assert!(error.contains("driver unavailable"), "{error}"); + assert!(error.contains("exited"), "{error}"); + let mut malformed = + std::process::Command::new(if cfg!(windows) { "cmd.exe" } else { "sh" }); + if cfg!(windows) { + malformed.args(["/D", "/C", "echo invalid-json"]); + } else { + malformed.args(["-c", "echo invalid-json"]); + } + let error = run_owned_cuda_probe(&mut malformed, Duration::from_secs(2)).unwrap_err(); + assert!(error.contains("invalid CUDA floor probe JSON"), "{error}"); + } + + #[test] + fn cuda_floor_probe_matches_real_worker_binary_output() { + let worker = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../../target/release/ck-synapse-worker-cuda.exe"); + if !worker.is_file() { + return; // release worker not staged on this host + } + let reading = run_owned_cuda_probe( + &mut std::process::Command::new(&worker), + Duration::from_secs(10), + ) + .expect("real worker probe"); + assert!(reading.driver_api >= synapse_core::OWNED_CUDA_MINIMUM_DRIVER_API); + assert!( + reading.compute_major as f32 + reading.compute_minor as f32 / 10.0 + >= synapse_core::OWNED_CUDA_MINIMUM_DEVICE_CC + ); + } + + #[test] + fn cuda_floor_failure_evidence_preserves_stderr_without_fabricating_hardware() { + let unavailable = CudaFloorDecision::Unsupported { + reason: synapse_core::CudaUnsupportedReason::HardwareUnavailable, + observed: None, + }; + let observed = + floor_observed_with_probe_error(&unavailable, Some("CUDA driver unavailable")); + assert_eq!(observed["probe_stderr"], "CUDA driver unavailable"); + assert!(observed.get("driver_api").is_none()); + let below_floor = evaluate_cuda_floor(11000, 8, 9, None); + let observed = floor_observed_with_probe_error(&below_floor, Some("stale error")); + assert_eq!(observed["driver_api"], 11000); + assert!(observed.get("probe_stderr").is_none()); + } #[test] fn probe_report_separates_certification_from_serving_admission() { diff --git a/crates/synapse-worker-cuda/src/main.rs b/crates/synapse-worker-cuda/src/main.rs index ed8aef96..7d136b1b 100644 --- a/crates/synapse-worker-cuda/src/main.rs +++ b/crates/synapse-worker-cuda/src/main.rs @@ -68,6 +68,27 @@ fn version_probe() -> bool { } } +/// Print the observed hardware floor as a single JSON object and exit 0. +/// +/// Only the CUDA-enabled build can answer; a build without the feature prints +/// the error to stderr and exits non-zero so the caller records the refusal +/// rather than mistaking silence for a pass. +fn probe_floor() -> Result<()> { + #[cfg(feature = "cuda")] + { + let probe = synapse_engine_cuda::probe_hardware_floor()?; + println!( + "{{\"driver_api\":{},\"compute_capability\":{{\"major\":{},\"minor\":{}}}}}", + probe.driver_api, probe.compute_major, probe.compute_minor + ); + Ok(()) + } + #[cfg(not(feature = "cuda"))] + { + anyhow::bail!("--probe-floor requires a build with cargo feature `cuda`") + } +} + /// Build the identity announced in the worker HELLO handshake. pub fn engine_identity() -> synapse_core::EngineIdentity { owned_cuda_engine_identity("worker", "f16", KERNEL_REVISION) @@ -77,6 +98,9 @@ fn main() -> Result<()> { if version_probe() { return Ok(()); } + if std::env::args().skip(1).any(|arg| arg == "--probe-floor") { + return probe_floor(); + } let args = Args::parse(); let hello = WorkerHello { v: WORKER_PROTOCOL_VERSION, From ce87e1beef98eda1c8427fe594d258932c650a9e Mon Sep 17 00:00:00 2001 From: Qiiks Date: Thu, 17 Sep 2026 15:59:54 +0530 Subject: [PATCH 2/3] probe: single deadline, real-worker regression, drop Serialize derive --- Cargo.lock | 33 +++++-------------------- crates/synapse-engine-cuda/src/lib.rs | 2 +- crates/synapse-module/src/lib.rs | 35 +++++++++++++++++++-------- 3 files changed, 32 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 307605fa..15b1f5ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1486,19 +1486,7 @@ name = "cortexkit-log" version = "0.2.0" dependencies = [ "chrono", - "cortexkit-store-types 0.2.2", - "regex", - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "cortexkit-log" -version = "0.2.0" -source = "git+https://github.com/cortexkit/commons.git?rev=99736150c6d0c769c1c7479894fba7f07c322a5f#99736150c6d0c769c1c7479894fba7f07c322a5f" -dependencies = [ - "chrono", - "cortexkit-store-types 0.2.1", + "cortexkit-store-types", "regex", "tracing", "tracing-subscriber", @@ -1515,18 +1503,10 @@ name = "cortexkit-store" version = "0.2.0" dependencies = [ "cortexkit-lease", - "cortexkit-store-types 0.2.2", + "cortexkit-store-types", "rusqlite", ] -[[package]] -name = "cortexkit-store-types" -version = "0.2.1" -source = "git+https://github.com/cortexkit/commons.git?rev=99736150c6d0c769c1c7479894fba7f07c322a5f#99736150c6d0c769c1c7479894fba7f07c322a5f" -dependencies = [ - "serde", -] - [[package]] name = "cortexkit-store-types" version = "0.2.2" @@ -6678,7 +6658,7 @@ dependencies = [ [[package]] name = "subc-control" -version = "0.11.3" +version = "0.11.2" dependencies = [ "serde", "serde_json", @@ -6687,10 +6667,9 @@ dependencies = [ [[package]] name = "subc-core" -version = "0.17.45" +version = "0.17.39" dependencies = [ "base64 0.22.1", - "cortexkit-log 0.2.0 (git+https://github.com/cortexkit/commons.git?rev=99736150c6d0c769c1c7479894fba7f07c322a5f)", "cortexkit-paths", "ed25519-dalek", "fs4", @@ -6853,9 +6832,9 @@ version = "0.1.0-alpha.2" dependencies = [ "anyhow", "cortexkit-lease", - "cortexkit-log 0.2.0", + "cortexkit-log", "cortexkit-store", - "cortexkit-store-types 0.2.2", + "cortexkit-store-types", "half", "hex", "httpdate", diff --git a/crates/synapse-engine-cuda/src/lib.rs b/crates/synapse-engine-cuda/src/lib.rs index 3a97141c..9af36f48 100644 --- a/crates/synapse-engine-cuda/src/lib.rs +++ b/crates/synapse-engine-cuda/src/lib.rs @@ -189,7 +189,7 @@ pub fn build_identity(family: ModelFamily, dtype: StorageDType) -> CudaBuildIden /// /// Carried separately from [`device_meets_floor`] so the caller can log or /// refuse on the observed values rather than on a bare boolean. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct HardwareFloorProbe { pub driver_api: u32, pub compute_major: u32, diff --git a/crates/synapse-module/src/lib.rs b/crates/synapse-module/src/lib.rs index 43024e15..bee556e1 100644 --- a/crates/synapse-module/src/lib.rs +++ b/crates/synapse-module/src/lib.rs @@ -6055,6 +6055,7 @@ fn run_owned_cuda_probe( command: &mut std::process::Command, timeout: Duration, ) -> Result { + let deadline = std::time::Instant::now() + timeout; command .stdin(std::process::Stdio::null()) .stdout(std::process::Stdio::piped()) @@ -6084,12 +6085,14 @@ fn run_owned_cuda_probe( } let _ = stderr_tx.send(String::from_utf8_lossy(&tail).into_owned()); }); - let deadline = std::time::Instant::now() + timeout; let status = loop { match child.try_wait() { Ok(Some(status)) => break Ok(status), Ok(None) if std::time::Instant::now() < deadline => { - std::thread::sleep(Duration::from_millis(20)); + std::thread::sleep( + Duration::from_millis(20) + .min(deadline.saturating_duration_since(std::time::Instant::now())), + ); } other => { let _ = child.kill(); @@ -6103,7 +6106,7 @@ fn run_owned_cuda_probe( }; // Bound pipe completion too: a descendant may still hold an inherited pipe. let stderr = stderr_rx - .recv_timeout(Duration::from_millis(100)) + .recv_timeout(deadline.saturating_duration_since(std::time::Instant::now())) .unwrap_or_default(); let fail = |reason: String| format!("{reason}; stderr: {stderr}"); let status = status.map_err(&fail)?; @@ -6111,7 +6114,7 @@ fn run_owned_cuda_probe( return Err(fail(format!("CUDA floor probe exited {status}"))); } let stdout = stdout_rx - .recv_timeout(Duration::from_millis(100)) + .recv_timeout(deadline.saturating_duration_since(std::time::Instant::now())) .map_err(|error| fail(format!("CUDA floor probe stdout: {error}")))? .map_err(|error| fail(format!("read CUDA floor probe stdout: {error}")))?; if stdout.len() > 4096 { @@ -15164,14 +15167,26 @@ mod tests { } #[test] + #[ignore = "requires a staged CUDA worker and supported GPU; run explicitly"] fn cuda_floor_probe_matches_real_worker_binary_output() { - let worker = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../../../target/release/ck-synapse-worker-cuda.exe"); - if !worker.is_file() { - return; // release worker not staged on this host - } + let worker = env::var_os("SYNAPSE_TEST_CUDA_WORKER") + .map(PathBuf::from) + .unwrap_or_else(|| { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../target/release") + .join(if cfg!(windows) { + "ck-synapse-worker-cuda.exe" + } else { + "ck-synapse-worker-cuda" + }) + }); + assert!( + worker.is_file(), + "stage CUDA worker at {}", + worker.display() + ); let reading = run_owned_cuda_probe( - &mut std::process::Command::new(&worker), + std::process::Command::new(&worker).arg("--probe-floor"), Duration::from_secs(10), ) .expect("real worker probe"); From 3e47fa9ecfadd11b168c3e285b3d766a767a2b3c Mon Sep 17 00:00:00 2001 From: Qiiks Date: Thu, 17 Sep 2026 18:17:33 +0530 Subject: [PATCH 3/3] chore: preserve upstream dependency lockfile --- Cargo.lock | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 15b1f5ea..307605fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1486,7 +1486,19 @@ name = "cortexkit-log" version = "0.2.0" dependencies = [ "chrono", - "cortexkit-store-types", + "cortexkit-store-types 0.2.2", + "regex", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "cortexkit-log" +version = "0.2.0" +source = "git+https://github.com/cortexkit/commons.git?rev=99736150c6d0c769c1c7479894fba7f07c322a5f#99736150c6d0c769c1c7479894fba7f07c322a5f" +dependencies = [ + "chrono", + "cortexkit-store-types 0.2.1", "regex", "tracing", "tracing-subscriber", @@ -1503,10 +1515,18 @@ name = "cortexkit-store" version = "0.2.0" dependencies = [ "cortexkit-lease", - "cortexkit-store-types", + "cortexkit-store-types 0.2.2", "rusqlite", ] +[[package]] +name = "cortexkit-store-types" +version = "0.2.1" +source = "git+https://github.com/cortexkit/commons.git?rev=99736150c6d0c769c1c7479894fba7f07c322a5f#99736150c6d0c769c1c7479894fba7f07c322a5f" +dependencies = [ + "serde", +] + [[package]] name = "cortexkit-store-types" version = "0.2.2" @@ -6658,7 +6678,7 @@ dependencies = [ [[package]] name = "subc-control" -version = "0.11.2" +version = "0.11.3" dependencies = [ "serde", "serde_json", @@ -6667,9 +6687,10 @@ dependencies = [ [[package]] name = "subc-core" -version = "0.17.39" +version = "0.17.45" dependencies = [ "base64 0.22.1", + "cortexkit-log 0.2.0 (git+https://github.com/cortexkit/commons.git?rev=99736150c6d0c769c1c7479894fba7f07c322a5f)", "cortexkit-paths", "ed25519-dalek", "fs4", @@ -6832,9 +6853,9 @@ version = "0.1.0-alpha.2" dependencies = [ "anyhow", "cortexkit-lease", - "cortexkit-log", + "cortexkit-log 0.2.0", "cortexkit-store", - "cortexkit-store-types", + "cortexkit-store-types 0.2.2", "half", "hex", "httpdate",