From 3ee16f256ee2e36a9c8551c7fe3a0991758aa80a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 20 Sep 2026 05:37:54 -0700 Subject: [PATCH 1/4] fix(vmm): keep the hugepage NUMA split from overflowing on an absurd vcpu or memory --- dstack/vmm/src/app.rs | 7 ++++++- dstack/vmm/src/app/qemu.rs | 27 ++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index df3e99281..7dd47e114 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -209,6 +209,11 @@ impl GpuConfig { /// Round up a value to the nearest multiple of another value. /// If the value is already a multiple, it remains unchanged. +/// +/// `vcpu` and `memory` are deployment request fields with no upper bound, so +/// the next multiple is not always representable. It is left unchanged when it +/// is not: release builds have no overflow checks, and wrapping here would turn +/// an absurd request into a small `-smp`/`-m` that QEMU happily accepts. pub(crate) fn round_up(value: u32, multiple: u32) -> u32 { if multiple <= 1 { return value; @@ -219,7 +224,7 @@ pub(crate) fn round_up(value: u32, multiple: u32) -> u32 { return value; } - value + (multiple - remainder) + value.checked_add(multiple - remainder).unwrap_or(value) } /// Get the NUMA node associated with a PCI device. diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 74e3a7958..48e17b578 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -896,7 +896,7 @@ impl QemuCommandBuilder<'_> { )); bus_number += device_count + 1; } - Ok((smp, memory_gib * 1024)) + Ok((smp, memory_gib.saturating_mul(1024))) } fn configure_gpus(&self, command: &mut Command) -> Result<()> { @@ -1785,4 +1785,29 @@ mod tests { .iter() .any(|arg| arg.contains("vfio-pci,host=0000:02:00.0"))); } + + /// vcpu and memory are deployment request fields with no upper bound, and + /// the hugepage NUMA split does arithmetic on both. Release builds have no + /// overflow checks, so an overflow here is a silent `-smp`/`-m` of the + /// wrong size rather than an error. + #[test] + fn an_absurd_vcpu_or_memory_does_not_overflow_the_numa_split() { + let (config, mut vm, mut prepared) = test_launch_fixture(); + vm.manifest.hugepages = true; + vm.manifest.vcpu = u32::MAX; + vm.manifest.memory = u32::MAX; + prepared.hugepage_numa_nodes = + Some(HashMap::from([("0".to_string(), 0), ("1".to_string(), 0)])); + let builder = QemuCommandBuilder { + vm: &vm, + cfg: &config.cvm, + gpus: &GpuConfig::default(), + prepared: &prepared, + }; + let (smp, mem) = builder + .configure_hugepage_memory(&mut Command::new("qemu-system-x86_64")) + .unwrap(); + assert!(smp >= u32::MAX / 2, "-smp collapsed to {smp}"); + assert!(mem >= u32::MAX / 2, "-m collapsed to {mem}"); + } } From 039fbecf07a072e993b54650bf92c0929d214a9c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 20 Sep 2026 05:40:22 -0700 Subject: [PATCH 2/4] fix(vmm): reject a zero vcpu, memory or disk size at deployment --- dstack/vmm/src/main_service.rs | 36 ++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index ce29fc748..d483b07a0 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -314,6 +314,17 @@ pub fn create_manifest_from_vm_config( cvm_config: &crate::config::CvmConfig, ) -> Result { validate_label(&request.name)?; + // The same three a resize refuses. Without this a VM is created with a + // `-m 0` and a `0G` data disk, and only the launch says so. + if request.vcpu == 0 { + bail!("vcpu must be greater than zero"); + } + if request.memory == 0 { + bail!("memory must be greater than zero"); + } + if request.disk_size == 0 { + bail!("disk_size must be greater than zero"); + } let port_map = port_map_from_proto(&request.ports, &cvm_config.port_mapping, &[])?; let networks = networks_from_vm_config(&request, cvm_config)?; @@ -1749,6 +1760,31 @@ mod tests { assert!(!err.contains("storage_discard"), "{err}"); } + /// A resize refuses a zero vcpu, memory or disk; a deployment has to + /// refuse the same values, or the VM is created with a `-m 0` and a `0G` + /// data disk and only fails later, out of the caller's sight. + #[test] + fn a_deployment_rejects_the_same_zero_resources_a_resize_does() { + for (name, mutate) in [ + ( + "vcpu", + (|r: &mut VmConfiguration| r.vcpu = 0) as fn(&mut VmConfiguration), + ), + ("memory", |r: &mut VmConfiguration| r.memory = 0), + ("disk_size", |r: &mut VmConfiguration| r.disk_size = 0), + ] { + let mut request = test_vm_configuration(); + mutate(&mut request); + let err = format!( + "{:#}", + create_manifest_from_vm_config(request, &test_cvm_config()) + .expect_err("a zero resource must be refused at deployment") + ); + assert!(err.contains(name), "{name}: {err}"); + } + create_manifest_from_vm_config(test_vm_configuration(), &test_cvm_config()).unwrap(); + } + #[test] fn resize_request_rejects_empty_zero_and_empty_image_updates() { let mut request = ResizeVmRequest { From 3c9eb43acffc5e1ca7f5010ad4a673d7163439b0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 20 Sep 2026 05:45:48 -0700 Subject: [PATCH 3/4] test(vmm): pin the two bounds the MAC and NUMA arithmetic rest on --- dstack/vmm/src/app.rs | 8 ++++++++ dstack/vmm/src/app/network.rs | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 7dd47e114..e1b112c3f 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -3268,6 +3268,14 @@ mod tests { Ok(config) } + /// `configure_hugepage_memory` divides by this count. Nothing else stops + /// the divisor being zero, so the "at least node 0" fallback is the whole + /// guard -- pin it rather than reason about it. + #[test] + fn the_hugepage_numa_split_always_has_at_least_one_node() { + assert_eq!(hugepage_numa_nodes(&GpuConfig::default()).unwrap().len(), 1); + } + #[test] fn effective_vcpu_count_clamps_zero_to_one() { assert_eq!(effective_vcpu_count(0, None), 1); diff --git a/dstack/vmm/src/app/network.rs b/dstack/vmm/src/app/network.rs index b946e3ca1..dcfc907b3 100644 --- a/dstack/vmm/src/app/network.rs +++ b/dstack/vmm/src/app/network.rs @@ -527,6 +527,27 @@ mod tests { ); } + /// `mac_prefix` is operator config, so the only thing standing between it + /// and a `[u8; 6]` is the `.min(3)`. Pin every prefix length it admits, + /// including one longer than the cap, against a later refactor that drops + /// the clamp and reads six bytes out of a three-byte prefix. + #[test] + fn a_mac_prefix_never_reaches_past_the_address_it_fills() { + for prefix in [ + &[][..], + &[0x52][..], + &[0x52, 0x54][..], + &[0x52, 0x54, 0x00][..], + &[0x52, 0x54, 0x00, 0x99, 0x99, 0x99, 0x99][..], + ] { + let mac = mac_address_for_vm_index("vm-123", prefix, 0); + assert_eq!(mac.split(':').count(), 6, "{prefix:?} -> {mac}"); + for (index, byte) in prefix.iter().take(3).enumerate().skip(1) { + assert_eq!(mac.split(':').nth(index).unwrap(), format!("{byte:02x}")); + } + } + } + fn nic(mode: NetworkingMode) -> Networking { Networking { nic: NicNetworking { From 1b77e7f60e5787fda88f8030ff3e85e469bbd737 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 24 Sep 2026 02:10:27 -0700 Subject: [PATCH 4/4] refactor(vmm): trim resource bound comments and tests --- dstack/vmm/src/app.rs | 13 +++---------- dstack/vmm/src/app/network.rs | 21 --------------------- dstack/vmm/src/app/qemu.rs | 25 ------------------------- dstack/vmm/src/main_service.rs | 29 +++++------------------------ 4 files changed, 8 insertions(+), 80 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index e1b112c3f..732b3be97 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -209,11 +209,7 @@ impl GpuConfig { /// Round up a value to the nearest multiple of another value. /// If the value is already a multiple, it remains unchanged. -/// -/// `vcpu` and `memory` are deployment request fields with no upper bound, so -/// the next multiple is not always representable. It is left unchanged when it -/// is not: release builds have no overflow checks, and wrapping here would turn -/// an absurd request into a small `-smp`/`-m` that QEMU happily accepts. +/// Left unchanged if the next multiple overflows, rather than wrapping to a tiny value. pub(crate) fn round_up(value: u32, multiple: u32) -> u32 { if multiple <= 1 { return value; @@ -3268,12 +3264,9 @@ mod tests { Ok(config) } - /// `configure_hugepage_memory` divides by this count. Nothing else stops - /// the divisor being zero, so the "at least node 0" fallback is the whole - /// guard -- pin it rather than reason about it. #[test] - fn the_hugepage_numa_split_always_has_at_least_one_node() { - assert_eq!(hugepage_numa_nodes(&GpuConfig::default()).unwrap().len(), 1); + fn round_up_does_not_wrap() { + assert_eq!(round_up(u32::MAX, 2), u32::MAX); } #[test] diff --git a/dstack/vmm/src/app/network.rs b/dstack/vmm/src/app/network.rs index dcfc907b3..b946e3ca1 100644 --- a/dstack/vmm/src/app/network.rs +++ b/dstack/vmm/src/app/network.rs @@ -527,27 +527,6 @@ mod tests { ); } - /// `mac_prefix` is operator config, so the only thing standing between it - /// and a `[u8; 6]` is the `.min(3)`. Pin every prefix length it admits, - /// including one longer than the cap, against a later refactor that drops - /// the clamp and reads six bytes out of a three-byte prefix. - #[test] - fn a_mac_prefix_never_reaches_past_the_address_it_fills() { - for prefix in [ - &[][..], - &[0x52][..], - &[0x52, 0x54][..], - &[0x52, 0x54, 0x00][..], - &[0x52, 0x54, 0x00, 0x99, 0x99, 0x99, 0x99][..], - ] { - let mac = mac_address_for_vm_index("vm-123", prefix, 0); - assert_eq!(mac.split(':').count(), 6, "{prefix:?} -> {mac}"); - for (index, byte) in prefix.iter().take(3).enumerate().skip(1) { - assert_eq!(mac.split(':').nth(index).unwrap(), format!("{byte:02x}")); - } - } - } - fn nic(mode: NetworkingMode) -> Networking { Networking { nic: NicNetworking { diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 48e17b578..d32a2940d 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -1785,29 +1785,4 @@ mod tests { .iter() .any(|arg| arg.contains("vfio-pci,host=0000:02:00.0"))); } - - /// vcpu and memory are deployment request fields with no upper bound, and - /// the hugepage NUMA split does arithmetic on both. Release builds have no - /// overflow checks, so an overflow here is a silent `-smp`/`-m` of the - /// wrong size rather than an error. - #[test] - fn an_absurd_vcpu_or_memory_does_not_overflow_the_numa_split() { - let (config, mut vm, mut prepared) = test_launch_fixture(); - vm.manifest.hugepages = true; - vm.manifest.vcpu = u32::MAX; - vm.manifest.memory = u32::MAX; - prepared.hugepage_numa_nodes = - Some(HashMap::from([("0".to_string(), 0), ("1".to_string(), 0)])); - let builder = QemuCommandBuilder { - vm: &vm, - cfg: &config.cvm, - gpus: &GpuConfig::default(), - prepared: &prepared, - }; - let (smp, mem) = builder - .configure_hugepage_memory(&mut Command::new("qemu-system-x86_64")) - .unwrap(); - assert!(smp >= u32::MAX / 2, "-smp collapsed to {smp}"); - assert!(mem >= u32::MAX / 2, "-m collapsed to {mem}"); - } } diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index d483b07a0..7177dd312 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -314,8 +314,7 @@ pub fn create_manifest_from_vm_config( cvm_config: &crate::config::CvmConfig, ) -> Result { validate_label(&request.name)?; - // The same three a resize refuses. Without this a VM is created with a - // `-m 0` and a `0G` data disk, and only the launch says so. + // Same checks as `validate_resize_request`. if request.vcpu == 0 { bail!("vcpu must be greater than zero"); } @@ -1760,29 +1759,11 @@ mod tests { assert!(!err.contains("storage_discard"), "{err}"); } - /// A resize refuses a zero vcpu, memory or disk; a deployment has to - /// refuse the same values, or the VM is created with a `-m 0` and a `0G` - /// data disk and only fails later, out of the caller's sight. #[test] - fn a_deployment_rejects_the_same_zero_resources_a_resize_does() { - for (name, mutate) in [ - ( - "vcpu", - (|r: &mut VmConfiguration| r.vcpu = 0) as fn(&mut VmConfiguration), - ), - ("memory", |r: &mut VmConfiguration| r.memory = 0), - ("disk_size", |r: &mut VmConfiguration| r.disk_size = 0), - ] { - let mut request = test_vm_configuration(); - mutate(&mut request); - let err = format!( - "{:#}", - create_manifest_from_vm_config(request, &test_cvm_config()) - .expect_err("a zero resource must be refused at deployment") - ); - assert!(err.contains(name), "{name}: {err}"); - } - create_manifest_from_vm_config(test_vm_configuration(), &test_cvm_config()).unwrap(); + fn deployment_rejects_zero_resources() { + let mut request = test_vm_configuration(); + request.memory = 0; + assert!(create_manifest_from_vm_config(request, &test_cvm_config()).is_err()); } #[test]