diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1ca2926..bf6d1f1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -812,49 +812,19 @@ jobs: "@ | Set-Content build-gates/windows-x86_64-msvc-owned-cuda.txt if ($status -ne 0) { exit $status } - # Hollow-green guard for the compiled backend, with no PE-inspection - # dependency. `--features cuda` is the only thing that makes the build - # link cudart/cublas, and on Windows those are LOAD-TIME imports: an - # exe that imports them cannot start when they are absent from PATH - # (STATUS_DLL_NOT_FOUND, 0xC0000135) even though its --version path - # never calls into them. So the same probe answers both halves: - # off-PATH must FAIL with 0xC0000135 -> the CUDA backend is baked in - # on-PATH must PASS -> the resolved DLL set is right - # A worker that silently compiled CPU-only prints --version both times, - # which this step refuses. It also records, in CI, the sidecar fact the - # release/installer PR must solve: the worker is not self-contained. - # Caveat: the discriminator depends on cudart/cublas staying load-time - # imports; if a future build moves them to delay-load, the off-PATH run - # exits 0 and this step reports "cuda feature is not compiled" — a false - # failure that points at the real cause rather than hiding it. - - name: Assert the packaged worker really carries the CUDA backend + # Derive sidecars from the same verified runtime components used to build. + # No independently maintained DLL manifest; nvcc and driver files are not + # redistribution inputs. The packager carries component licenses/hashes. + - name: Package and verify Windows CUDA sidecars working-directory: synapse shell: pwsh run: | - $ErrorActionPreference = 'Continue' - $exe = (Resolve-Path 'target/release/ck-synapse-worker-cuda.exe').Path - $isolated = Join-Path $env:RUNNER_TEMP 'cuda-isolated' - New-Item -ItemType Directory -Force $isolated | Out-Null - Copy-Item $exe $isolated - # PATH narrowed to the OS directories: no CUDA toolkit, no sidecars. - $env:PATH = "C:\Windows\system32;C:\Windows" - & (Join-Path $isolated 'ck-synapse-worker-cuda.exe') --version 2>&1 | Out-Null - $isolatedCode = $LASTEXITCODE - if ($isolatedCode -eq 0) { - throw "owned-CUDA worker ran without CUDA DLLs on PATH (exit 0): the cuda feature is not compiled into this binary — refusing to record a hollow-green gate" - } - if ($isolatedCode -ne -1073741515) { - throw "owned-CUDA worker failed off-PATH with exit $isolatedCode, expected -1073741515 (0xC0000135 STATUS_DLL_NOT_FOUND)" - } - # CUDA 13's redist layout keeps the runtime DLLs under bin\x64 (the - # nvcc drivers sit in bin); both directories are needed, as verified - # locally. With them restored the worker must start. - $env:PATH = "$env:CUDA_PATH\bin\x64;$env:CUDA_PATH\bin;C:\Windows\system32;C:\Windows" - & $exe --version - if ($LASTEXITCODE -ne 0) { - throw "owned-CUDA worker --version failed with CUDA on PATH (exit $LASTEXITCODE)" - } - "cuda_import_probe=ok off_path=$isolatedCode on_path=0" >> $env:GITHUB_STEP_SUMMARY + ./scripts/package-owned-cuda.ps1 ` + -Worker target/release/ck-synapse-worker-cuda.exe ` + -RuntimeComponents @("$env:RUNNER_TEMP/x-cuda_cudart", "$env:RUNNER_TEMP/x-libcublas") ` + -Output build-gates/owned-cuda-windows-x64.zip + ./scripts/test-owned-cuda-package.ps1 -Archive build-gates/owned-cuda-windows-x64.zip + 'cuda_package_probe=passed; GPU execution requires a GPU runner' >> $env:GITHUB_STEP_SUMMARY - name: Retain Windows owned-CUDA gate evidence if: always() diff --git a/README.md b/README.md index a6c81af..d6de79c 100644 --- a/README.md +++ b/README.md @@ -69,3 +69,58 @@ Example user-tier `~/.config/cortexkit/synapse.jsonc` (project configs must omit Tests can point at a file with `SYNAPSE_CONFIG_PATH`. Only one synapse module per machine (singleton lease); a second instance refuses to start. + +### Owned-CUDA hardware floor + +`ck-synapse-worker-cuda` implements `--probe-floor` (hidden, like the +`--test-abort*` surfaces). It prints one JSON object and exits 0: + +```json +{"driver_api": 13030, "compute_capability": {"major": 8, "minor": 9}} +``` + +The module probes the configured `worker_bin`, the engine's worker-binary +environment override, or the sibling `ck-synapse-worker-cuda`, in that order. +It caches one result per process unless both environment readings parse +successfully. The child wait is bounded to 10 seconds; stdout is capped at +4096 bytes and each pipe completion wait is bounded to another 100 ms. +A missing binary, non-zero exit, timeout, or invalid output produces +`HardwareUnavailable`. Refusal and model evidence carry diagnostic context, +including the last 4096 bytes of stderr when available, under `observed`. +Failed probes do not fabricate numeric hardware readings. + +The environment overrides the probe only as a complete, parseable pair. +Otherwise both readings come from the probe; partial overrides are not merged: + +- `SYNAPSE_CUDA_DRIVER_API` (alias `CUDA_DRIVER_API`) — the raw CUDA **driver + API** integer from `cuDriverGetVersion()`, not the marketing driver version. + For example, a measured driver API value is `13030`. `610.88` is not a valid + API integer; without a parseable alias, it causes fallback to the probe. +- `SYNAPSE_CUDA_COMPUTE_CAPABILITY` (alias `CUDA_COMPUTE_CAPABILITY`) — device + 0's compute capability as `major.minor`, for example `8.9`. +- `SYNAPSE_CUDA_PACKAGING_DRIVER` — optional; the driver string a packaging + build was tested against, carried into the refusal for diagnostics. + +### Windows owned-CUDA package + +The manual Windows CUDA gate packages the worker with runtime DLLs derived +from the same pinned `cuda_cudart` and `libcublas` redistribution archives +used for compilation. `scripts/package-owned-cuda.ps1` places the executable +and DLLs at the ZIP root, includes component licenses, and records source +components and SHA-256 hashes in `manifest.json`. The NVIDIA driver is not +bundled and must already be installed. + +Windows worker builds require CUDA 13. The build script rejects other toolkit +major versions before linking, because this package resolves CUDA 13 DLL names. + +`scripts/test-owned-cuda-package.ps1 -Archive -RequireGpu` extracts a +fresh copy, verifies hashes, and checks no-sidecar `--version`, actionable +missing-library refusal, and a real hardware-floor probe with adjacent DLLs +and CUDA removed from PATH. Without `-RequireGpu`, a runner without NVIDIA +hardware may report an explicit driver/device refusal; this is not a GPU +execution pass. Neither mode loads model weights or certifies embeddings. + +The worker delays its cuBLASLt import and checks runtime library loading +before CUDA calls. No global PATH changes or extra DLL search directories +are needed. Release-matrix publication remains separate from this manual +gate artifact. diff --git a/crates/synapse-engine-cuda/src/cuda.rs b/crates/synapse-engine-cuda/src/cuda.rs index 7ba1034..32a000d 100644 --- a/crates/synapse-engine-cuda/src/cuda.rs +++ b/crates/synapse-engine-cuda/src/cuda.rs @@ -60,8 +60,46 @@ mod enabled { context: NonNull, } + /// Keep successfully loaded libraries resident for all subsequent FFI calls. + /// Resolve explicitly before touching a delay import so failure is a Rust + /// error, not an unhandled Windows loader exception. + fn ensure_libraries_loaded() -> Result<()> { + #[cfg(target_env = "msvc")] + { + use std::sync::LazyLock; + static LOADED: LazyLock> = LazyLock::new(|| { + #[link(name = "kernel32")] + unsafe extern "system" { + fn LoadLibraryW(name: *const u16) -> *mut c_void; + } + for name in [ + "cublasLt64_13.dll", + "cublas64_13.dll", + "cudart64_13.dll", + "nvcuda.dll", + ] { + let wide: Vec = name.encode_utf16().chain(Some(0)).collect(); + // Windows searches the executable directory and installed + // system locations without changing PATH or global policy. + if unsafe { LoadLibraryW(wide.as_ptr()) }.is_null() { + return Err(format!( + "cannot load CUDA library {name}: {}", + std::io::Error::last_os_error() + )); + } + } + Ok(()) + }); + if let Err(message) = &*LOADED { + anyhow::bail!("{message}"); + } + } + Ok(()) + } + impl DeviceBinding { fn capture() -> Result { + ensure_libraries_loaded()?; cuda_driver_check(unsafe { cuInit(0) }, "cuInit")?; let mut runtime_device = 0; cuda_runtime_check( @@ -110,12 +148,59 @@ mod enabled { } pub fn ensure_available() -> Result<()> { + ensure_libraries_loaded()?; cuda_driver_check(unsafe { cuInit(0) }, "cuInit")?; let version = unsafe { synapse_cuda_cublaslt_version() }; ensure!(version > 0, "cuBLASLt did not report a version"); Ok(()) } + /// Read the driver API version and device 0's compute capability. + /// + /// This runs before an owned-CUDA load is admitted, so it deliberately + /// touches nothing else: no context is retained, no model is loaded, and + /// no weights are mapped. The reading is what the floor predicate is + /// applied to, which is why it reports the raw numbers rather than a + /// verdict. + pub fn probe_hardware_floor() -> Result { + ensure_libraries_loaded()?; + cuda_driver_check(unsafe { cuInit(0) }, "cuInit")?; + let mut driver_api = 0; + cuda_driver_check( + unsafe { cuDriverGetVersion(&mut driver_api) }, + "cuDriverGetVersion", + )?; + let mut device = 0; + cuda_driver_check(unsafe { cuDeviceGet(&mut device, 0) }, "cuDeviceGet")?; + let mut major = 0; + cuda_driver_check( + unsafe { + cuDeviceGetAttribute( + &mut major, + CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, + device, + ) + }, + "cuDeviceGetAttribute(COMPUTE_CAPABILITY_MAJOR)", + )?; + let mut minor = 0; + cuda_driver_check( + unsafe { + cuDeviceGetAttribute( + &mut minor, + CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, + device, + ) + }, + "cuDeviceGetAttribute(COMPUTE_CAPABILITY_MINOR)", + )?; + Ok(crate::HardwareFloorProbe { + driver_api: driver_api as u32, + compute_major: major as u32, + compute_minor: minor as u32, + }) + } + pub struct MiniLmContext { binding: DeviceBinding, raw: NonNull, @@ -465,8 +550,16 @@ mod enabled { } } + /// `CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR` from `cuda.h`. + const CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR: i32 = 75; + /// `CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR` from `cuda.h`. + const CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR: i32 = 76; + unsafe extern "C" { fn cuInit(flags: u32) -> i32; + fn cuDriverGetVersion(version: *mut i32) -> i32; + fn cuDeviceGet(device: *mut i32, ordinal: i32) -> i32; + fn cuDeviceGetAttribute(value: *mut i32, attrib: i32, device: i32) -> i32; fn cuCtxGetDevice(device: *mut i32) -> i32; fn cuCtxSetCurrent(context: *mut c_void) -> i32; fn cuDevicePrimaryCtxRetain(context: *mut *mut c_void, device: i32) -> i32; @@ -551,6 +644,10 @@ mod enabled { bail!("owned CUDA requires a non-macOS build with cargo feature `cuda`") } + pub fn probe_hardware_floor() -> Result { + bail!("owned CUDA requires a non-macOS build with cargo feature `cuda`") + } + pub struct MiniLmContext; impl MiniLmContext { pub fn new(_graphs: bool) -> Result { @@ -629,4 +726,6 @@ mod enabled { } } -pub use enabled::{ensure_available, MiniLmContext, ModernBertContext, Qwen3Context}; +pub use enabled::{ + ensure_available, probe_hardware_floor, MiniLmContext, ModernBertContext, Qwen3Context, +}; diff --git a/crates/synapse-engine-cuda/src/lib.rs b/crates/synapse-engine-cuda/src/lib.rs index 6de832d..9af36f4 100644 --- a/crates/synapse-engine-cuda/src/lib.rs +++ b/crates/synapse-engine-cuda/src/lib.rs @@ -18,6 +18,8 @@ use synapse_core::{ mod cuda; mod model; +pub use cuda::probe_hardware_floor; + pub const ENGINE_VERSION: &str = "owned-cuda-v1"; /// The source revision from which the CUDA kernels were ported. pub const KERNEL_REVISION: &str = "4d0ded67c30286fe2be37cc7413359ad745dd751"; @@ -183,6 +185,17 @@ pub fn build_identity(family: ModelFamily, dtype: StorageDType) -> CudaBuildIden } } +/// A hardware-floor reading taken before any owned-CUDA worker is spawned. +/// +/// Carried separately from [`device_meets_floor`] so the caller can log or +/// refuse on the observed values rather than on a bare boolean. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct HardwareFloorProbe { + pub driver_api: u32, + pub compute_major: u32, + pub compute_minor: u32, +} + /// Hardware-floor predicate used by capability probes before worker creation. #[must_use] pub fn device_meets_floor(driver_api: u32, compute_major: u32, compute_minor: u32) -> bool { diff --git a/crates/synapse-module/src/lib.rs b/crates/synapse-module/src/lib.rs index 8043393..a6cb135 100644 --- a/crates/synapse-module/src/lib.rs +++ b/crates/synapse-module/src/lib.rs @@ -5388,7 +5388,7 @@ fn load_catalog_model_blocking( spec.model_id ))); } - ensure_owned_cuda_floor()?; + ensure_owned_cuda_floor(spec.worker_bin.as_deref())?; } let model_path = locator_path(&spec.model_locator, &model_cache)?; let tokenizer_path = locator_path(&spec.tokenizer_locator, &model_cache)?; @@ -5991,7 +5991,7 @@ fn locator_path( } } -fn owned_cuda_floor_decision() -> CudaFloorDecision { +fn owned_cuda_floor_decision(worker: Option<&Path>) -> CudaFloorDecision { let driver_api = ["SYNAPSE_CUDA_DRIVER_API", "CUDA_DRIVER_API"] .into_iter() .find_map(|name| { @@ -6008,14 +6008,152 @@ fn owned_cuda_floor_decision() -> CudaFloorDecision { }); let packaging_driver = env::var("SYNAPSE_CUDA_PACKAGING_DRIVER").ok(); let (Some(driver_api), Some((major, minor))) = (driver_api, compute) else { - return CudaFloorDecision::Unsupported { - reason: synapse_core::CudaUnsupportedReason::HardwareUnavailable, - observed: None, + // The environment is the override; when it is silent, ask the worker. + // The module deliberately does not link the CUDA driver, so the probe + // has to run in the worker process and report its numbers back. + return match owned_cuda_probe_floor(worker) { + Ok(reading) => evaluate_cuda_floor( + reading.driver_api, + reading.compute_major, + reading.compute_minor, + packaging_driver, + ), + Err(_) => CudaFloorDecision::Unsupported { + reason: synapse_core::CudaUnsupportedReason::HardwareUnavailable, + observed: None, + }, }; }; evaluate_cuda_floor(driver_api, major, minor, packaging_driver) } +/// A hardware reading reported by `ck-synapse-worker-cuda --probe-floor`. +#[derive(Clone, Copy, Debug)] +struct OwnedCudaFloorReading { + driver_api: u32, + compute_major: u32, + compute_minor: u32, +} + +static OWNED_CUDA_PROBE: std::sync::LazyLock< + Mutex>>, +> = std::sync::LazyLock::new(|| Mutex::new(HashMap::new())); + +/// Cache successes and failures per worker; a complete environment override skips it. +fn owned_cuda_probe_floor(worker: Option<&Path>) -> Result { + let worker = worker + .map(Path::to_path_buf) + .or_else(|| env::var_os(worker_binary_env_var(CUDA_WORKER_ENGINE)).map(PathBuf::from)) + .or_else(|| resolve_worker_binary_sibling(CUDA_WORKER_ENGINE)) + .ok_or_else(|| "CUDA floor probe worker binary not found".to_string())?; + let worker = fs::canonicalize(&worker).unwrap_or(worker); + let mut cache = OWNED_CUDA_PROBE + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + // Failed probes stay cached deliberately until module restart, for this worker only. + cache + .entry(worker) + .or_insert_with_key(|worker| { + let mut command = std::process::Command::new(worker); + command.arg("--probe-floor"); + run_owned_cuda_probe(&mut command, Duration::from_secs(10)) + }) + .clone() +} + +fn run_owned_cuda_probe( + command: &mut std::process::Command, + timeout: Duration, +) -> Result { + let deadline = std::time::Instant::now() + timeout; + command + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + let mut child = command + .spawn() + .map_err(|error| format!("spawn CUDA floor probe: {error}"))?; + let stdout = child.stdout.take().expect("piped stdout"); + let mut stderr = child.stderr.take().expect("piped stderr"); + let (stdout_tx, stdout_rx) = std::sync::mpsc::sync_channel(1); + let (stderr_tx, stderr_rx) = std::sync::mpsc::sync_channel(1); + std::thread::spawn(move || { + let mut bytes = Vec::new(); + let result = stdout.take(4097).read_to_end(&mut bytes).map(|_| bytes); + let _ = stdout_tx.send(result); + }); + std::thread::spawn(move || { + let mut tail = Vec::new(); + let mut chunk = [0_u8; 4096]; + while let Ok(count) = stderr.read(&mut chunk) { + if count == 0 { + break; + } + let discard = (tail.len() + count).saturating_sub(4096); + tail.drain(..discard); + tail.extend_from_slice(&chunk[..count]); + } + let _ = stderr_tx.send(String::from_utf8_lossy(&tail).into_owned()); + }); + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break Ok(status), + Ok(None) if std::time::Instant::now() < deadline => { + std::thread::sleep( + Duration::from_millis(20) + .min(deadline.saturating_duration_since(std::time::Instant::now())), + ); + } + other => { + let _ = child.kill(); + let _ = child.wait(); + break Err(match other { + Err(error) => format!("wait for CUDA floor probe: {error}"), + _ => "CUDA floor probe timed out".to_string(), + }); + } + } + }; + // Bound pipe completion too: a descendant may still hold an inherited pipe. + let stderr = stderr_rx + .recv_timeout(deadline.saturating_duration_since(std::time::Instant::now())) + .unwrap_or_default(); + let fail = |reason: String| format!("{reason}; stderr: {stderr}"); + let status = status.map_err(&fail)?; + if !status.success() { + return Err(fail(format!("CUDA floor probe exited {status}"))); + } + let stdout = stdout_rx + .recv_timeout(deadline.saturating_duration_since(std::time::Instant::now())) + .map_err(|error| fail(format!("CUDA floor probe stdout: {error}")))? + .map_err(|error| fail(format!("read CUDA floor probe stdout: {error}")))?; + if stdout.len() > 4096 { + return Err(fail( + "CUDA floor probe stdout exceeds 4096 bytes".to_string(), + )); + } + let parsed: Value = serde_json::from_slice(&stdout) + .map_err(|error| fail(format!("invalid CUDA floor probe JSON: {error}")))?; + let reading = || { + Some(OwnedCudaFloorReading { + driver_api: parsed.get("driver_api")?.as_u64()?.try_into().ok()?, + compute_major: parsed + .get("compute_capability")? + .get("major")? + .as_u64()? + .try_into() + .ok()?, + compute_minor: parsed + .get("compute_capability")? + .get("minor")? + .as_u64()? + .try_into() + .ok()?, + }) + }; + reading().ok_or_else(|| fail("invalid CUDA floor probe hardware fields".to_string())) +} + fn parse_compute_capability(value: &str) -> Option<(u32, u32)> { let mut parts = value.trim().split('.'); let major = parts.next()?.parse().ok()?; @@ -6023,21 +6161,41 @@ fn parse_compute_capability(value: &str) -> Option<(u32, u32)> { parts.next().is_none().then_some((major, minor)) } -fn ensure_owned_cuda_floor() -> Result<(), WireOperationError> { - let decision = owned_cuda_floor_decision(); +fn owned_cuda_floor_observed(decision: &CudaFloorDecision, worker: Option<&Path>) -> Value { + let error = match decision { + CudaFloorDecision::Unsupported { observed: None, .. } => { + owned_cuda_probe_floor(worker).err() + } + _ => None, + }; + floor_observed_with_probe_error(decision, error.as_deref()) +} + +fn floor_observed_with_probe_error(decision: &CudaFloorDecision, error: Option<&str>) -> Value { + match decision { + CudaFloorDecision::Supported { observed } + | CudaFloorDecision::Unsupported { + observed: Some(observed), + .. + } => serde_json::to_value(observed).unwrap_or(Value::Null), + CudaFloorDecision::Unsupported { observed: None, .. } => error + .map(|stderr| json!({ "probe_stderr": stderr })) + .unwrap_or(Value::Null), + } +} + +fn ensure_owned_cuda_floor(worker: Option<&Path>) -> Result<(), WireOperationError> { + let decision = owned_cuda_floor_decision(worker); if decision.is_supported() { return Ok(()); } - let observed = match &decision { - CudaFloorDecision::Unsupported { observed, .. } => observed, - CudaFloorDecision::Supported { .. } => unreachable!(), - }; + let observed = owned_cuda_floor_observed(&decision, worker); Err(WireOperationError::from_stable( StableError::owned_cuda_unsupported(), format!( "owned-cuda floor refused before worker creation: decision={}, observed={}", decision.refusal_code().unwrap_or("owned_cuda_unsupported"), - serde_json::to_string(observed).unwrap_or_else(|_| "null".to_string()), + observed, ), )) } @@ -6046,15 +6204,8 @@ fn owned_cuda_evidence(state: &ModuleState, model: &EmbeddingModel) -> Option serde_json::to_value(observed).ok(), - CudaFloorDecision::Unsupported { observed: None, .. } => None, - }; + let decision = owned_cuda_floor_decision(None); + let observed = owned_cuda_floor_observed(&decision, None); Some(json!({ "engine": CUDA_WORKER_ENGINE, "backend": model.engine_identity.build_flags.get("backend"), @@ -15003,6 +15154,132 @@ fn now_ms() -> u64 { #[cfg(test)] mod tests { use super::*; + #[test] + fn cuda_floor_probe_retains_child_failure_and_rejects_bad_json() { + let mut failed = std::process::Command::new(if cfg!(windows) { "cmd.exe" } else { "sh" }); + if cfg!(windows) { + failed.args(["/D", "/C", "echo driver unavailable 1>&2 & exit /b 7"]); + } else { + failed.args(["-c", "echo 'driver unavailable' >&2; exit 7"]); + } + let error = run_owned_cuda_probe(&mut failed, Duration::from_secs(2)).unwrap_err(); + assert!(error.contains("driver unavailable"), "{error}"); + assert!(error.contains("exited"), "{error}"); + let mut malformed = + std::process::Command::new(if cfg!(windows) { "cmd.exe" } else { "sh" }); + if cfg!(windows) { + malformed.args(["/D", "/C", "echo invalid-json"]); + } else { + malformed.args(["-c", "echo invalid-json"]); + } + let error = run_owned_cuda_probe(&mut malformed, Duration::from_secs(2)).unwrap_err(); + assert!(error.contains("invalid CUDA floor probe JSON"), "{error}"); + } + + #[test] + fn cuda_floor_probe_cache_is_keyed_per_worker_binary() { + let root = std::env::temp_dir().join(format!( + "synapse-probe-key-{}-{}", + std::process::id(), + TEST_STATE_COUNTER.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(&root).unwrap(); + let good = root.join(if cfg!(windows) { "good.cmd" } else { "good.sh" }); + let bad = root.join(if cfg!(windows) { "bad.cmd" } else { "bad.sh" }); + let header = if cfg!(windows) { + "@echo off\r\n" + } else { + "#!/bin/sh\n" + }; + let json = r#"{"driver_api":13030,"compute_capability":{"major":8,"minor":9}}"#; + let success = if cfg!(windows) { + format!("{header}echo {json}\r\n") + } else { + format!("{header}echo '{json}'\n") + }; + fs::write(&good, &success).unwrap(); + fs::write( + &bad, + format!( + "{header}echo missing-library >&2\n{}\n", + if cfg!(windows) { "exit /b 9" } else { "exit 9" } + ), + ) + .unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + for path in [&good, &bad] { + fs::set_permissions(path, fs::Permissions::from_mode(0o700)).unwrap(); + } + } + let failure = owned_cuda_probe_floor(Some(&bad)).unwrap_err(); + assert!(failure.contains("missing-library"), "{failure}"); + let reading = owned_cuda_probe_floor(Some(&good)).unwrap(); + assert_eq!( + ( + reading.driver_api, + reading.compute_major, + reading.compute_minor + ), + (13030, 8, 9) + ); + // Failure caching is deliberate, but must not contaminate another worker. + fs::write(&bad, success).unwrap(); + assert_eq!(owned_cuda_probe_floor(Some(&bad)).unwrap_err(), failure); + assert_eq!( + owned_cuda_probe_floor(Some(&good)).unwrap().driver_api, + 13030 + ); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + #[ignore = "requires a staged CUDA worker and supported GPU; run explicitly"] + fn cuda_floor_probe_matches_real_worker_binary_output() { + let worker = env::var_os("SYNAPSE_TEST_CUDA_WORKER") + .map(PathBuf::from) + .unwrap_or_else(|| { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../target/release") + .join(if cfg!(windows) { + "ck-synapse-worker-cuda.exe" + } else { + "ck-synapse-worker-cuda" + }) + }); + assert!( + worker.is_file(), + "stage CUDA worker at {}", + worker.display() + ); + let reading = run_owned_cuda_probe( + std::process::Command::new(&worker).arg("--probe-floor"), + Duration::from_secs(10), + ) + .expect("real worker probe"); + assert!(reading.driver_api >= synapse_core::OWNED_CUDA_MINIMUM_DRIVER_API); + assert!( + reading.compute_major as f32 + reading.compute_minor as f32 / 10.0 + >= synapse_core::OWNED_CUDA_MINIMUM_DEVICE_CC + ); + } + + #[test] + fn cuda_floor_failure_evidence_preserves_stderr_without_fabricating_hardware() { + let unavailable = CudaFloorDecision::Unsupported { + reason: synapse_core::CudaUnsupportedReason::HardwareUnavailable, + observed: None, + }; + let observed = + floor_observed_with_probe_error(&unavailable, Some("CUDA driver unavailable")); + assert_eq!(observed["probe_stderr"], "CUDA driver unavailable"); + assert!(observed.get("driver_api").is_none()); + let below_floor = evaluate_cuda_floor(11000, 8, 9, None); + let observed = floor_observed_with_probe_error(&below_floor, Some("stale error")); + assert_eq!(observed["driver_api"], 11000); + assert!(observed.get("probe_stderr").is_none()); + } #[test] fn probe_report_separates_certification_from_serving_admission() { diff --git a/crates/synapse-worker-cuda/build.rs b/crates/synapse-worker-cuda/build.rs new file mode 100644 index 0000000..9749911 --- /dev/null +++ b/crates/synapse-worker-cuda/build.rs @@ -0,0 +1,31 @@ +fn main() { + if std::env::var_os("CARGO_FEATURE_CUDA").is_some() + && std::env::var("CARGO_CFG_TARGET_ENV").as_deref() == Ok("msvc") + { + let cuda_root = std::env::var_os("CUDA_HOME") + .or_else(|| std::env::var_os("CUDA_PATH")) + .map(std::path::PathBuf::from) + .expect("Windows owned-CUDA packaging requires CUDA_HOME or CUDA_PATH"); + let header = cuda_root.join("include/cuda.h"); + println!("cargo:rerun-if-env-changed=CUDA_HOME"); + println!("cargo:rerun-if-env-changed=CUDA_PATH"); + println!("cargo:rerun-if-changed={}", header.display()); + let contents = + std::fs::read_to_string(&header).expect("cannot read CUDA toolkit include/cuda.h"); + let version = contents.lines().find_map(|line| { + let mut fields = line.split_whitespace(); + (fields.next() == Some("#define") && fields.next() == Some("CUDA_VERSION")) + .then(|| fields.next()?.parse::().ok()) + .flatten() + }); + assert!( + version.is_some_and(|version| version / 1000 == 13), + "Windows owned-CUDA packaging requires CUDA 13; CUDA 12 DLL names are incompatible" + ); + // Link arguments on the engine rlib do not propagate to this executable. + println!("cargo:rustc-link-lib=delayimp"); + // The CUDA 13 import libraries link cudart/cublas statically; cuBLASLt + // is the remaining load-time DLL (verified with dumpbin /dependents). + println!("cargo:rustc-link-arg=/DELAYLOAD:cublasLt64_13.dll"); + } +} diff --git a/crates/synapse-worker-cuda/src/main.rs b/crates/synapse-worker-cuda/src/main.rs index ed8aef9..7d136b1 100644 --- a/crates/synapse-worker-cuda/src/main.rs +++ b/crates/synapse-worker-cuda/src/main.rs @@ -68,6 +68,27 @@ fn version_probe() -> bool { } } +/// Print the observed hardware floor as a single JSON object and exit 0. +/// +/// Only the CUDA-enabled build can answer; a build without the feature prints +/// the error to stderr and exits non-zero so the caller records the refusal +/// rather than mistaking silence for a pass. +fn probe_floor() -> Result<()> { + #[cfg(feature = "cuda")] + { + let probe = synapse_engine_cuda::probe_hardware_floor()?; + println!( + "{{\"driver_api\":{},\"compute_capability\":{{\"major\":{},\"minor\":{}}}}}", + probe.driver_api, probe.compute_major, probe.compute_minor + ); + Ok(()) + } + #[cfg(not(feature = "cuda"))] + { + anyhow::bail!("--probe-floor requires a build with cargo feature `cuda`") + } +} + /// Build the identity announced in the worker HELLO handshake. pub fn engine_identity() -> synapse_core::EngineIdentity { owned_cuda_engine_identity("worker", "f16", KERNEL_REVISION) @@ -77,6 +98,9 @@ fn main() -> Result<()> { if version_probe() { return Ok(()); } + if std::env::args().skip(1).any(|arg| arg == "--probe-floor") { + return probe_floor(); + } let args = Args::parse(); let hello = WorkerHello { v: WORKER_PROTOCOL_VERSION, diff --git a/scripts/package-owned-cuda.ps1 b/scripts/package-owned-cuda.ps1 new file mode 100644 index 0000000..f3b5e75 --- /dev/null +++ b/scripts/package-owned-cuda.ps1 @@ -0,0 +1,45 @@ +param( + [Parameter(Mandatory)][string]$Worker, + [Parameter(Mandatory)][string[]]$RuntimeComponents, + [Parameter(Mandatory)][string]$Output +) +$ErrorActionPreference = 'Stop' +$stage = Join-Path ([IO.Path]::GetTempPath()) ([guid]::NewGuid().ToString()) +New-Item -ItemType Directory $stage | Out-Null +try { + Copy-Item $Worker (Join-Path $stage 'ck-synapse-worker-cuda.exe') + $entries = @() + foreach ($component in $RuntimeComponents) { + $root = (Resolve-Path $component).Path + $dlls = @(Get-ChildItem $root -Recurse -File -Filter '*.dll') + if (!$dlls.Count) { throw "No runtime DLLs in $root" } + foreach ($file in $dlls) { + $destination = Join-Path $stage $file.Name + if (Test-Path $destination) { throw "Duplicate runtime filename: $($file.Name)" } + Copy-Item $file.FullName $destination + $entries += [ordered]@{ + file = $file.Name + sha256 = (Get-FileHash $destination -Algorithm SHA256).Hash.ToLowerInvariant() + component = Split-Path $root -Leaf + source = $file.FullName.Substring($root.Length + 1).Replace('\', '/') + } + } + $licenseDir = Join-Path $stage ('licenses/' + (Split-Path $root -Leaf)) + New-Item -ItemType Directory -Force $licenseDir | Out-Null + $licenses = @(Get-ChildItem $root -Recurse -File | Where-Object { $_.Name -match '^(LICENSE|EULA|COPYING)' }) + if (!$licenses.Count) { throw "Missing redistribution license in $root" } + foreach ($license in $licenses) { + $destination = Join-Path $licenseDir $license.FullName.Substring($root.Length + 1) + New-Item -ItemType Directory -Force (Split-Path $destination) | Out-Null + Copy-Item $license.FullName $destination + } + } + [ordered]@{ + schema = 1 + worker_sha256 = (Get-FileHash (Join-Path $stage 'ck-synapse-worker-cuda.exe') -Algorithm SHA256).Hash.ToLowerInvariant() + runtime_files = $entries + driver = 'System-installed NVIDIA driver; not bundled' + } | ConvertTo-Json -Depth 5 | Set-Content (Join-Path $stage 'manifest.json') -Encoding UTF8 + Compress-Archive -Path (Join-Path $stage '*') -DestinationPath $Output -Force + Get-FileHash $Output -Algorithm SHA256 +} finally { Remove-Item $stage -Recurse -Force } diff --git a/scripts/test-owned-cuda-package.ps1 b/scripts/test-owned-cuda-package.ps1 new file mode 100644 index 0000000..c9f37df --- /dev/null +++ b/scripts/test-owned-cuda-package.ps1 @@ -0,0 +1,62 @@ +param( + [Parameter(Mandatory)][string]$Archive, + [switch]$RequireGpu +) +$ErrorActionPreference = 'Stop' +$root = Join-Path ([IO.Path]::GetTempPath()) ([guid]::NewGuid().ToString()) +$savedPath = $env:PATH +New-Item -ItemType Directory $root | Out-Null +function Invoke-Worker([string]$Exe, [string]$Argument) { + $info = New-Object Diagnostics.ProcessStartInfo + $info.FileName = $Exe + $info.Arguments = $Argument + $info.WorkingDirectory = Split-Path $Exe + $info.UseShellExecute = $false + $info.RedirectStandardOutput = $true + $info.RedirectStandardError = $true + $process = [Diagnostics.Process]::Start($info) + $stdout = $process.StandardOutput.ReadToEndAsync() + $stderr = $process.StandardError.ReadToEndAsync() + try { + if (!$process.WaitForExit(15000)) { $process.Kill(); throw 'Worker probe exceeded 15 seconds' } + return @{ Code = $process.ExitCode; Out = $stdout.GetAwaiter().GetResult(); Err = $stderr.GetAwaiter().GetResult() } + } finally { $process.Dispose() } +} +try { + $package = Join-Path $root 'package' + Expand-Archive $Archive $package + $exe = Join-Path $package 'ck-synapse-worker-cuda.exe' + $manifest = Get-Content (Join-Path $package 'manifest.json') -Raw | ConvertFrom-Json + if ((Get-FileHash $exe -Algorithm SHA256).Hash.ToLowerInvariant() -ne $manifest.worker_sha256) { throw 'Worker hash mismatch' } + foreach ($entry in $manifest.runtime_files) { + if ((Get-FileHash (Join-Path $package $entry.file) -Algorithm SHA256).Hash.ToLowerInvariant() -ne $entry.sha256) { throw "Hash mismatch: $($entry.file)" } + } + $empty = Join-Path $root 'no-sidecars' + New-Item -ItemType Directory $empty | Out-Null + Copy-Item $exe $empty + $env:PATH = "$env:SystemRoot\System32;$env:SystemRoot" + $isolatedExe = Join-Path $empty 'ck-synapse-worker-cuda.exe' + $version = Invoke-Worker $isolatedExe '--version' + if ($version.Code -ne 0 -or $version.Out -notmatch '^ck-synapse-worker-cuda ') { throw "No-DLL version failed: $($version.Err)" } + $missing = Invoke-Worker $isolatedExe '--probe-floor' + if ($missing.Code -eq 0 -or $missing.Err -notmatch 'cannot load CUDA library') { throw "Missing-DLL refusal failed: $($missing.Code) $($missing.Err)" } + $present = Invoke-Worker $exe '--probe-floor' + if ($present.Code -eq 0) { + $floor = $present.Out | ConvertFrom-Json + if ($floor.driver_api -le 0 -or $floor.compute_capability.major -le 0) { throw 'Invalid floor JSON' } + if ($RequireGpu -and ($floor.driver_api -lt 12040 -or $floor.compute_capability.major -lt 7 -or ($floor.compute_capability.major -eq 7 -and $floor.compute_capability.minor -lt 5))) { + throw 'GPU below owned-CUDA floor: driver API >= 12040 and compute capability >= 7.5 required' + } + Write-Output "PASS packaged GPU probe: $($present.Out.Trim())" + } elseif ($RequireGpu) { + throw "Packaged GPU probe failed: $($present.Code) $($present.Err)" + } elseif ($present.Err -match 'cannot load CUDA library (cublas|cudart)' -or $present.Err -notmatch '(nvcuda.dll|cuInit|cuDeviceGet)') { + throw "Packaged runtime resolution failed: $($present.Code) $($present.Err)" + } else { + Write-Output "GPU execution not available on this runner: $($present.Err.Trim())" + } + Write-Output 'PASS archive hashes, no-DLL version, missing-DLL refusal, side-by-side runtime resolution' +} finally { + $env:PATH = $savedPath + Remove-Item $root -Recurse -Force +}