From cf20a1e0079021acd56265c96a6fe32ecf6075be Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 11 Sep 2026 09:28:41 -0700 Subject: [PATCH 1/4] fix(qemu-acpi): emit ACPI PCI hotplug AML for root-port buses Co-authored-by: Leechael Yim --- dstack/crates/qemu-acpi/src/dsdt/gpe.rs | 32 ++- dstack/crates/qemu-acpi/src/dsdt/mod.rs | 12 +- dstack/crates/qemu-acpi/src/dsdt/notify.rs | 257 ++++++++++++++++++-- dstack/crates/qemu-acpi/src/golden_tests.rs | 21 +- 4 files changed, 289 insertions(+), 33 deletions(-) diff --git a/dstack/crates/qemu-acpi/src/dsdt/gpe.rs b/dstack/crates/qemu-acpi/src/dsdt/gpe.rs index 46342124b..4a21e68c3 100644 --- a/dstack/crates/qemu-acpi/src/dsdt/gpe.rs +++ b/dstack/crates/qemu-acpi/src/dsdt/gpe.rs @@ -24,7 +24,7 @@ //! `_E01` is deliberately empty here: it is the PCI hotplug event, and the //! baseline machine has no hotplug-capable bridge for it to scan. -use acpi_tables::aml::{Method, MethodCall, Name, Path, Scope}; +use acpi_tables::aml::{Acquire, Method, MethodCall, Name, Path, Release, Scope}; use super::ops::emit_all; @@ -49,10 +49,18 @@ pub(crate) fn e02() -> Vec { )]) } -/// `Scope (_GPE) { Method (_E01, 0, NotSerialized) {} }`, the PCI hotplug -/// event. -pub(crate) fn e01() -> Vec { - let handler = Method::new(Path::new("_E01"), 0, false, vec![]); +/// `Scope (_GPE) { Method (_E01, 0, NotSerialized) { ... } }`, the PCI +/// hotplug event. QEMU leaves the method empty when no bus supplies PCNT. +pub(crate) fn e01(has_pcnt: bool) -> Vec { + let acquire = Acquire::new(Path::new("\\_SB_.PCI0.BLCK"), 0xffff); + let scan = MethodCall::new(Path::new("\\_SB_.PCI0.PCNT"), vec![]); + let release = Release::new(Path::new("\\_SB_.PCI0.BLCK")); + let children: Vec<&dyn acpi_tables::Aml> = if has_pcnt { + vec![&acquire, &scan, &release] + } else { + vec![] + }; + let handler = Method::new(Path::new("_E01"), 0, false, children); emit_all(&[&Scope::new(Path::new("_GPE"), vec![&handler])]) } @@ -70,6 +78,18 @@ mod tests { #[test] fn e01_matches_qemu() { - super::super::fixture::assert_region(&super::e01(), 8245, 8258); + super::super::fixture::assert_region(&super::e01(false), 8245, 8258); + } + + #[test] + fn e01_scans_root_port_buses_when_pcnt_exists() { + use sha2::{Digest, Sha256}; + + let e01 = super::e01(true); + assert_eq!(e01.len() - super::e01(false).len(), 51); + assert_eq!( + hex::encode(Sha256::digest(e01)), + "6e22ff760f3c9e6263713cd7590518d1fd5ca2cd2344e8856bb618d2bef6d470" + ); } } diff --git a/dstack/crates/qemu-acpi/src/dsdt/mod.rs b/dstack/crates/qemu-acpi/src/dsdt/mod.rs index 4f33fb9de..26eba15ea 100644 --- a/dstack/crates/qemu-acpi/src/dsdt/mod.rs +++ b/dstack/crates/qemu-acpi/src/dsdt/mod.rs @@ -101,14 +101,22 @@ pub(crate) fn body(config: &MachineConfig) -> Result, Error> { )); out.extend(sstate::build()); // 7732..7774 Scope(\) _S3/_S4/_S5 out.extend(fwcf::build()); // 7774..7834 Scope(\_SB.PCI0) FWCF + let pxb_devfn = has_pxb.then_some(0x80); out.extend(notify::build( regular_slots, root_ports, modern_serial_irq, - has_pxb.then_some(0x80), + pxb_devfn, + pci_hotplug, )); + let pcnt = pci_hotplug + .then(|| notify::pcnt(regular_slots, root_ports, pxb_devfn)) + .flatten(); + if let Some(pcnt) = &pcnt { + out.extend(pcnt); + } if pci_hotplug { - out.extend(gpe::e01()); + out.extend(gpe::e01(pcnt.is_some())); } Ok(out) } diff --git a/dstack/crates/qemu-acpi/src/dsdt/notify.rs b/dstack/crates/qemu-acpi/src/dsdt/notify.rs index 2039a5bfc..65927a4ec 100644 --- a/dstack/crates/qemu-acpi/src/dsdt/notify.rs +++ b/dstack/crates/qemu-acpi/src/dsdt/notify.rs @@ -7,8 +7,9 @@ //! //! QEMU walks every devfn of the bus (`build_append_pci_bus_devices`, //! `hw/acpi/pcihp.c`) and emits `Device (S) { Name (_ADR, ..) }` for the -//! populated ones, then lets each device append its own AML. Only the LPC -//! bridge does (`build_ich9_isa_aml`, `hw/isa/lpc_ich9.c`). +//! populated ones, then lets each device append its own AML. The LPC bridge +//! appends its ISA children; PCIe root ports append their secondary-bus PCI +//! hotplug slots and recursive notification methods. //! //! ```asl //! Scope (\_SB) { @@ -66,8 +67,9 @@ //! ``` use acpi_tables::aml::{ - Device, EISAName, Field, FieldAccessType, FieldEntry, FieldLockRule, FieldUpdateRule, Name, - OpRegion, OpRegionSpace, Path, ResourceTemplate, Scope, IO, + And, Arg, Device, EISAName, Field, FieldAccessType, FieldEntry, FieldLockRule, FieldUpdateRule, + If, Index, Local, Method, MethodCall, Name, Notify, One, OpRegion, OpRegionSpace, Package, + Path, ResourceTemplate, Return, Scope, Store, Zero, IO, }; use acpi_tables::{Aml, AmlSink}; @@ -85,46 +87,91 @@ pub(crate) fn build( root_port_count: u32, modern_serial_irq: bool, pxb_devfn: Option, + pci_hotplug: bool, ) -> Vec { let lpc = lpc_children(modern_serial_irq); + let root_ports = root_ports(slot_count, root_port_count, pxb_devfn); let mut devices = Vec::new(); for slot in 0..slot_count { - devices.push(((slot * 8) as u8, false)); + devices.push(((slot * 8) as u8, false, None)); } if let Some(devfn) = pxb_devfn { - devices.push((devfn, false)); + devices.push((devfn, false, None)); } - let mut slot = slot_count; - for _ in 0..root_port_count { - if pxb_devfn == Some((slot * 8) as u8) { - slot += 1; - } - devices.push(((slot * 8) as u8, true)); - slot += 1; + for &(devfn, bsel) in &root_ports { + devices.push((devfn, true, pci_hotplug.then_some(bsel))); } for &devfn in CHIPSET_DEVFNS { - devices.push((devfn, false)); + devices.push((devfn, false, None)); } - devices.sort_unstable_by_key(|(devfn, _)| *devfn); + devices.sort_unstable_by_key(|(devfn, _, _)| *devfn); let mut bus = Vec::new(); - for (devfn, root_port) in devices { - bus.extend(pci_device(devfn, root_port, &lpc)); + for (devfn, root_port, bsel) in devices { + bus.extend(pci_device(devfn, root_port, bsel, &lpc)); } Scope::raw(Path::new("\\_SB_"), Scope::raw(Path::new("PCI0"), bus)) } -fn pci_device(devfn: u8, root_port: bool, lpc: &[u8]) -> Vec { +/// Build the recursive notification methods QEMU appends after the root-bus +/// device descriptions. The host bridge has no BSEL; each hotplug-capable +/// root-port secondary bus has one, assigned in bus traversal order. +pub(crate) fn pcnt( + slot_count: u32, + root_port_count: u32, + pxb_devfn: Option, +) -> Option> { + let root_ports = root_ports(slot_count, root_port_count, pxb_devfn); + if root_ports.is_empty() { + return None; + } + + let mut children = Vec::new(); + for &(devfn, bsel) in root_ports.iter().rev() { + children.extend(child_pcnt(devfn, bsel)); + } + + let mut calls = Vec::new(); + for &(devfn, _) in root_ports.iter().rev() { + // `Path` cannot encode AML ParentPrefixChar. A zero-argument method + // invocation is just its NameString, which QEMU emits as + // `^S.PCNT` from the root PCNT method. + calls.push(parent_pcnt_call(devfn)); + } + let raw_calls: Vec<_> = calls.iter().map(|call| Raw(call)).collect(); + let call_refs = raw_calls.iter().map(|call| call as &dyn Aml).collect(); + children.extend(emit(&Method::new(Path::new("PCNT"), 0, false, call_refs))); + + Some(Scope::raw(Path::new("\\_SB_.PCI0"), children)) +} + +fn root_ports(slot_count: u32, count: u32, pxb_devfn: Option) -> Vec<(u8, u32)> { + let mut ports = Vec::with_capacity(count as usize); + let mut slot = slot_count; + for index in 0..count { + if pxb_devfn == Some((slot * 8) as u8) { + slot += 1; + } + // QEMU inserts each secondary bus at the head of the root child list, + // then allocates BSEL values by walking that list. Root ports are + // created in ascending slot order, so their BSEL values run backward. + let bsel = count - index - 1; + ports.push(((slot * 8) as u8, bsel)); + slot += 1; + } + ports +} + +fn pci_device(devfn: u8, root_port: bool, bsel: Option, lpc: &[u8]) -> Vec { // QEMU names the device after the devfn but addresses it by the // ACPI 1.0b Table 6-2 PCI form: (device << 16) | function. let name = format!("S{devfn:02X}_"); let address = (u32::from(devfn >> 3) << 16) | u32::from(devfn & 0x07); let adr = Name::new(Path::new("_ADR"), &address); - let child_address = Name::new(Path::new("_ADR"), &0u8); - let child = root_port.then(|| emit(&Device::new(Path::new("S00_"), vec![&child_address]))); + let child = root_port.then(|| root_port_child(bsel)); let extra = Raw(if devfn == LPC_DEVFN { lpc } else { &[] }); let child = Raw(match child.as_deref() { Some(bytes) => bytes, @@ -133,6 +180,90 @@ fn pci_device(devfn: u8, root_port: bool, lpc: &[u8]) -> Vec { emit(&Device::new(Path::new(&name), vec![&adr, &extra, &child])) } +fn root_port_child(bsel: Option) -> Vec { + let child_address = Name::new(Path::new("_ADR"), &Zero {}); + let mut out = emit(&Device::new(Path::new("S00_"), vec![&child_address])); + let Some(bsel) = bsel else { + return out; + }; + + out.extend(emit(&Name::new(Path::new("BSEL"), &bsel))); + out.extend(hotplug_slot()); + out.extend(dvnt()); + out +} + +fn hotplug_slot() -> Vec { + let zero = Zero {}; + let one = One {}; + let asun = Name::new(Path::new("ASUN"), &zero); + + let local0 = Local(0); + let params = Package::new(vec![&zero, &zero]); + let init_params = Store::new(&local0, ¶ms); + let bus_slot = Index::new(&zero, &local0, &zero); + let bsel_name = Path::new("BSEL"); + let store_bus = Store::new(&bus_slot, &bsel_name); + let sun_slot = Index::new(&zero, &local0, &one); + let asun_name = Path::new("ASUN"); + let store_sun = Store::new(&sun_slot, &asun_name); + let (arg0, arg1, arg2, arg3) = (Arg(0), Arg(1), Arg(2), Arg(3)); + let pdsm = MethodCall::new(Path::new("PDSM"), vec![&arg0, &arg1, &arg2, &arg3, &local0]); + let ret = Return::new(&pdsm); + let dsm = Method::new( + Path::new("_DSM"), + 4, + true, + vec![&init_params, &store_bus, &store_sun, &ret], + ); + + let sun = Name::new(Path::new("_SUN"), &zero); + let bsel_name = Path::new("BSEL"); + let sun_name = Path::new("_SUN"); + let eject = MethodCall::new(Path::new("PCEJ"), vec![&bsel_name, &sun_name]); + let ej0 = Method::new(Path::new("_EJ0"), 1, false, vec![&eject]); + + emit(&Scope::new( + Path::new("S00_"), + vec![&asun, &dsm, &sun, &ej0], + )) +} + +fn dvnt() -> Vec { + let arg0 = Arg(0); + let arg1 = Arg(1); + let one = One {}; + let selected = And::new(&Zero {}, &arg0, &one); + let slot = Path::new("S00_"); + let notify = Notify::new(&slot, &arg1); + let branch = If::new(&selected, vec![¬ify]); + emit(&Method::new(Path::new("DVNT"), 2, false, vec![&branch])) +} + +fn child_pcnt(devfn: u8, bsel: u32) -> Vec { + let bnum = Path::new("BNUM"); + let pciu = Path::new("PCIU"); + let pcid = Path::new("PCID"); + let one = One {}; + let eject_request = 3u8; + let select_bus = Store::new(&bnum, &bsel); + let inserted = MethodCall::new(Path::new("DVNT"), vec![&pciu, &one]); + let removed = MethodCall::new(Path::new("DVNT"), vec![&pcid, &eject_request]); + let method = Method::new( + Path::new("PCNT"), + 0, + false, + vec![&select_bus, &inserted, &removed], + ); + Scope::raw(Path::new(&format!("S{devfn:02X}_")), emit(&method)) +} + +fn parent_pcnt_call(devfn: u8) -> Vec { + let mut call = vec![0x5e, 0x2e]; // ParentPrefixChar, DualNamePrefix + call.extend_from_slice(format!("S{devfn:02X}_PCNT").as_bytes()); + call +} + /// The children the ICH9 LPC bridge appends to its own device descriptor. fn lpc_children(modern_serial_irq: bool) -> Vec { // PCI-to-ISA interrupt routing registers in the bridge's config space. @@ -298,6 +429,90 @@ impl Aml for Irq { mod tests { #[test] fn matches_qemu() { - super::super::fixture::assert_region(&super::build(5, 0, true, None), 7834, 8245); + super::super::fixture::assert_region(&super::build(5, 0, true, None, true), 7834, 8245); + } + + #[test] + fn eight_root_ports_include_qemus_hotplug_aml() { + use sha2::{Digest, Sha256}; + + let hotplug_on = super::build(6, 8, true, None, true); + let hotplug_off = super::build(6, 8, true, None, false); + assert_eq!(hotplug_on.len() - hotplug_off.len(), 854); + + let pcnt = super::pcnt(6, 8, None).unwrap(); + assert_eq!(pcnt.len(), 411); + assert_eq!( + hex::encode(Sha256::digest(pcnt)), + "29f0fc0087802ef8886aa8eff10bbd78b471e50f8ce2d543861cd437a2806851" + ); + } + + #[test] + fn root_port_bsel_mapping_follows_qemus_reverse_child_walk() { + assert_eq!( + super::root_ports(6, 8, None), + vec![ + (0x30, 7), + (0x38, 6), + (0x40, 5), + (0x48, 4), + (0x50, 3), + (0x58, 2), + (0x60, 1), + (0x68, 0), + ] + ); + } + + #[test] + fn eight_root_port_devices_match_independent_qemu_encoding() { + use sha2::{Digest, Sha256}; + + let ports = super::root_ports(6, 8, None); + let mut hotplug_off = Vec::new(); + let mut hotplug_on = Vec::new(); + for (devfn, bsel) in ports { + hotplug_off.extend(super::pci_device(devfn, true, None, &[])); + hotplug_on.extend(super::pci_device(devfn, true, Some(bsel), &[])); + } + assert_eq!(hotplug_off.len(), 240); + assert_eq!(hotplug_on.len(), 1094); + assert_eq!( + hex::encode(Sha256::digest(hotplug_off)), + "7a107a6f0cf575cff4f70489c1b61a8c1ef1ae86a89de182ab0f412cea7bf80d" + ); + assert_eq!( + hex::encode(Sha256::digest(hotplug_on)), + "caec37e9162f026c9e36fe4952fb55f6aa7db09fc92e4e785914f84b05a705b3" + ); + } + + #[test] + fn root_port_hotplug_terms_match_independent_qemu_encoding() { + use sha2::{Digest, Sha256}; + + let base = super::root_port_child(None); + for (bsel, expected) in [ + ( + 0, + "2ab2603633fbf9eee2f7b9104d9f2510a55884520f1f02ddc41276fc9dfee90f", + ), + ( + 2, + "4b39e662fdf5334c777209850bb8984abb1dbda74f2f20bd807198e7683662ee", + ), + ] { + let port = super::root_port_child(Some(bsel)); + assert_eq!(hex::encode(Sha256::digest(&port[base.len()..])), expected); + } + } + + #[test] + fn root_port_hotplug_aml_is_conditional() { + let hotplug_off = super::build(6, 8, true, None, false); + let no_ports = super::build(6, 0, true, None, false); + assert_eq!(hotplug_off.len() - no_ports.len(), 8 * 30); + assert!(super::pcnt(6, 0, None).is_none()); } } diff --git a/dstack/crates/qemu-acpi/src/golden_tests.rs b/dstack/crates/qemu-acpi/src/golden_tests.rs index 7a13b4893..908db43a6 100644 --- a/dstack/crates/qemu-acpi/src/golden_tests.rs +++ b/dstack/crates/qemu-acpi/src/golden_tests.rs @@ -41,6 +41,19 @@ mod tests { Ok(()) } + #[test] + fn no_gpu_hotplug_off_matches_qemu_byte_for_byte() -> Result<(), Error> { + let mut c = config(0, 0); + c.hotplug_off = true; + let actual = build(&c)?; + let expected = include_bytes!("../fixtures/qemu-11.1-q35-hotplug-off-base.bin"); + assert_eq!(&actual.tables[..expected.len()], *expected); + assert!(actual.tables[expected.len()..] + .iter() + .all(|byte| *byte == 0)); + Ok(()) + } + #[test] fn numa_loader_and_rsdp_match_qemu_byte_for_byte() -> Result<(), Error> { let mut numa = config(1, 0); @@ -176,28 +189,28 @@ mod tests { 0, 1, 0, - "f47ab428541cb334c6de6e59e7fcf44a5db7b314e9dd4c643978968917eb25b2", + "f22f486b0e33ed0aad80e6cb26726652d671d3c58f521827825a26f069e499a3", ), ( 1, 0, 8, 0, - "ae3fefc72eb747cbff363e4f5ac7f3366f257849dc5f2719a9368303abaef4cc", + "037b07a2b0d6dc1d3b8370bb8d84564b74cd4f9a09163252c4a8b08d9ad0be70", ), ( 1, 0, 1, 1, - "8a48a13bc6041d73f7decce488054a8d25800cc82e11fa9bd1687e010ac9c9b0", + "6f1e598f7b9acbc24c33c573c81112b4e9df7d576d18e82d94b0253c4a69da71", ), ( 1, 0, 1, 4, - "2052ea73c74e1462947e600c95742e48cae0f7a84bc0ec79ab12f7a7818aec7a", + "e23873ce836a73783d043173f9757da1b36151a3a2987c8ab978e33570120eb9", ), ]; for (nics, volumes, gpus, switches, expected) in cases { From aa4c6797b320d5d0c9590765bda01ed0fa0e4143 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 11 Sep 2026 09:31:37 -0700 Subject: [PATCH 2/4] test(verifier): pin all RTMR0 ACPI digests of a captured 8-GPU CVM --- dstack/verifier/src/verification.rs | 50 +++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/dstack/verifier/src/verification.rs b/dstack/verifier/src/verification.rs index 6d9e56a3b..f14f96f3c 100644 --- a/dstack/verifier/src/verification.rs +++ b/dstack/verifier/src/verification.rs @@ -2462,6 +2462,56 @@ mod tests { .expect_err("an extra vCPU must change the ACPI tables"); } + /// Reproduces the VM shape from the captured eight-GPU QEMU 10.2.1 + /// attestation. The expected digests come from the CVM event log, not from + /// this generator. + /// + /// The loader digest only pins table sizes and offsets; the tables digest + /// pins the AML bytes themselves, including the root-port hotplug AML. + /// + /// The CVM ran with `pci_hole64_size = 0`, so QEMU sized the DSDT's 64-bit + /// PCI window from the BARs OVMF assigned to the eight GPUs instead. The + /// guest reported that window as `[mem 0x380000000000-0x3bc006013fff]`; + /// this test passes its length explicitly so that the `_CRS` bytes match. + #[test] + fn tdx_lite_acpi_matches_captured_eight_gpu_qemu_10_2_vm() { + let vm_config: VmConfig = serde_json::from_value(serde_json::json!({ + "cpu_count": 256, + "memory_size": 1_649_267_441_664u64, + "qemu_version": "10.2.1", + "pci_hole64_size": 0x3c0_0601_4000u64, + "hugepages": false, + "num_gpus": 8, + "num_nvswitches": 0, + "num_nics": 1, + "num_verity_volumes": 0, + "hotplug_off": false, + "host_share_mode": "9p", + "ovmf_variant": "pre202505", + "tdx_attestation_variant": "lite" + })) + .expect("captured VM config parses"); + + let actual = dstack_mr::tdx::expected_rtmr0_acpi_hashes( + &vm_config, + dstack_types::OvmfVariant::Pre202505, + ) + .expect("ACPI blobs are generated through the verifier measurement path"); + + assert_eq!( + hex::encode(actual.loader), + "01f02cba34d8f7213872ce341a587c636016a6268d0b9c2d2198058a8a709fe38cde80f67c808067444cb8fa11cdd27c" + ); + assert_eq!( + hex::encode(actual.rsdp), + "a5a7aa6b9b601386fab910a1840bbcef2a87c130e683b9630ebe53311fe1ed7cc80863f29f167350c147ae34082981e9" + ); + assert_eq!( + hex::encode(actual.tables), + "bbfdfdee5883455bce28fee44d807f2ed9999651e860aba5b2cd91d35040d7ef42ff36bb0fe5ace1299501e869519344" + ); + } + #[test] fn tdx_lite_acpi_hash_mismatch_names_the_table() { let expected = TdxRtmr0AcpiHashes { From bf4e4a2dd8b6e010bc53f980b91cf6c955889b43 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 11 Sep 2026 09:31:37 -0700 Subject: [PATCH 3/4] docs(qemu-acpi): document the VMM device order the BSEL mapping relies on --- dstack/crates/qemu-acpi/src/dsdt/notify.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/dstack/crates/qemu-acpi/src/dsdt/notify.rs b/dstack/crates/qemu-acpi/src/dsdt/notify.rs index 65927a4ec..c52646324 100644 --- a/dstack/crates/qemu-acpi/src/dsdt/notify.rs +++ b/dstack/crates/qemu-acpi/src/dsdt/notify.rs @@ -147,6 +147,20 @@ pub(crate) fn pcnt( Some(Scope::raw(Path::new("\\_SB_.PCI0"), children)) } +/// The `(devfn, BSEL)` of each root port on `pcie.0`, in slot order. +/// +/// QEMU numbers BSEL with a depth-first walk of the root bus's child list +/// (`acpi_set_bsel`, `hw/acpi/pcihp.c`), so the mapping depends on the order +/// dstack-vmm creates devices in: +/// +/// - root ports are created in ascending slot order, one endpoint at function +/// 0 each; +/// - PXB expanders are created before any root port, so their buses sit behind +/// the `pcie.0` root ports in the child list. Root ports under a PXB still +/// take BSEL values, but only after these ones, and QEMU emits no AML for +/// them. +/// +/// Reordering those devices in the VMM changes the measured DSDT. fn root_ports(slot_count: u32, count: u32, pxb_devfn: Option) -> Vec<(u8, u32)> { let mut ports = Vec::with_capacity(count as usize); let mut slot = slot_count; From b6535e4287d82df74fe40281cfd24e8f09569540 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 11 Sep 2026 23:30:44 -0700 Subject: [PATCH 4/4] test(qemu-acpi): sample the differential oracle after machine reset The oracle dumped the ACPI tables built inside acpi_setup(), a machine_init_done notifier. QEMU assigns the ACPI hotplug BSEL property to PCI bridge secondary buses in acpi_set_pci_info(), reached from the ICH9 PM reset handler through acpi_pcihp_reset(), and machine reset runs after that notifier. QEMU then rebuilds the tables via acpi_build_update(), so the blob a guest reads -- and measures into RTMR0 -- is the rebuilt one. Sampling before reset produced a DSDT in which no bus had a BSEL, and so none of the root-port hotplug AML this branch adds. The oracle agreed with any generator that omitted those terms: 17 of its 61 cases could not fail, namely every case with bridge hotplug enabled and a root port on pcie.0. That is why this branch's differential job reported the generator as wrong when it was the reference that was wrong. The fix lives in the QEMU compatibility fork, which now performs the one reset side effect that changes the generated AML before dumping, and folds the directory output mode in so both consumers sample at the same point. Bump to that revision, drop the patch it replaces, and repin the image. Also cross the axes that had only ever run GPU-less. Every per-version and large-CPU fixed case ran with no GPUs, so no version tier and no CPU-count boundary was pinned against a topology with root ports. Add the host sharing mode as an explicit axis too: dstack-mr accepts "9p", "vvfat" and "vhd" and models none of them, on the theory that each contributes exactly one PCI slot and the DSDT records only _ADR; QEMU confirms all three are byte-identical. Fixed cases go from 29 to 40. Document the two measured inputs the oracle structurally cannot reach, so a green run is not read as covering them: the 64-bit PCI window when pci_hole64_size is 0, which QEMU derives from BARs the guest firmware has assigned and no dump-and-exit oracle can observe, and multi-PXB topologies, which MachineConfig cannot express. --- .github/workflows/qemu-acpi-differential.yml | 2 +- dstack/crates/qemu-acpi/fixtures/README.md | 12 +++- dstack/crates/qemu-acpi/reference/Dockerfile | 4 +- dstack/crates/qemu-acpi/reference/README.md | 30 +++++++++- .../qemu-acpi/scripts/differential-random.py | 59 +++++++++++++++++-- .../scripts/qemu-dump-all-blobs.patch | 43 -------------- 6 files changed, 93 insertions(+), 57 deletions(-) delete mode 100644 dstack/crates/qemu-acpi/scripts/qemu-dump-all-blobs.patch diff --git a/.github/workflows/qemu-acpi-differential.yml b/.github/workflows/qemu-acpi-differential.yml index de8dea222..523fe82b5 100644 --- a/.github/workflows/qemu-acpi-differential.yml +++ b/.github/workflows/qemu-acpi-differential.yml @@ -32,7 +32,7 @@ jobs: run: working-directory: dstack env: - REFERENCE_IMAGE: kvin/dstack-acpi-tables@sha256:54e692d8c68c6f02dd7c655bf6de6e7f3ad7a43fded6ca00d8e998918108a3b4 + REFERENCE_IMAGE: kvin/dstack-acpi-tables@sha256:98c42e609d84408cfb9a5e95e04f22058bb999ea62c72de43b9b2a18954d14e6 steps: - uses: actions/checkout@v5 diff --git a/dstack/crates/qemu-acpi/fixtures/README.md b/dstack/crates/qemu-acpi/fixtures/README.md index 109eb4bd8..787bc47f1 100644 --- a/dstack/crates/qemu-acpi/fixtures/README.md +++ b/dstack/crates/qemu-acpi/fixtures/README.md @@ -10,7 +10,7 @@ The fixtures in this directory were generated from: - repository: - branch: `dstack-qemu-acpi-11.1-compat` -- revision: `9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195` +- revision: `0f3d3f6ed099e4cf0b79f59e8b6ba0083b7c414f` - dstack image inputs: `dstack-0.5.5/ovmf.fd` and `dstack-0.5.5/bzImage` @@ -19,8 +19,7 @@ Build the reference in a clean build directory: ```bash git clone https://github.com/kvinwang/qemu-tdx.git qemu-tdx cd qemu-tdx -git checkout 9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195 -git apply /path/to/qemu-acpi/scripts/qemu-dump-all-blobs.patch +git checkout 0f3d3f6ed099e4cf0b79f59e8b6ba0083b7c414f mkdir build-acpi && cd build-acpi CFLAGS='-DDUMP_ACPI_TABLES -Wno-builtin-macro-redefined -D__DATE__="" -D__TIME__="" -D__TIMESTAMP__=""' \ LDFLAGS='-Wl,--build-id=none' \ @@ -29,6 +28,13 @@ LDFLAGS='-Wl,--build-id=none' \ ninja qemu-system-x86_64 ``` +That revision samples the blobs after the PCI bridge `BSEL` properties are +assigned, which QEMU only does during machine reset. Capturing from a build +that dumps inside `acpi_setup()` yields tables missing every root-port hotplug +method, so any fixture taken from a VM with PCIe root ports (GPU or NVSwitch +passthrough) would be wrong. Fixtures for VMs with no root ports are +unaffected either way. + Run the complete three-blob differential matrix with: ```bash diff --git a/dstack/crates/qemu-acpi/reference/Dockerfile b/dstack/crates/qemu-acpi/reference/Dockerfile index 5a2b55f35..99ee9aed3 100644 --- a/dstack/crates/qemu-acpi/reference/Dockerfile +++ b/dstack/crates/qemu-acpi/reference/Dockerfile @@ -4,7 +4,7 @@ FROM ubuntu:24.04 AS builder ARG QEMU_REPOSITORY=https://github.com/kvinwang/qemu-tdx.git -ARG QEMU_REVISION=9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195 +ARG QEMU_REVISION=0f3d3f6ed099e4cf0b79f59e8b6ba0083b7c414f RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ca-certificates git ninja-build pkg-config python3 python3-venv \ @@ -31,7 +31,7 @@ RUN CFLAGS='-O2 -DDUMP_ACPI_TABLES -Wno-builtin-macro-redefined -D__DATE__="" -D FROM ubuntu:24.04 -ARG QEMU_REVISION=9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195 +ARG QEMU_REVISION=0f3d3f6ed099e4cf0b79f59e8b6ba0083b7c414f LABEL org.opencontainers.image.source="https://github.com/kvinwang/qemu-tdx" \ org.opencontainers.image.revision="${QEMU_REVISION}" \ org.opencontainers.image.licenses="GPL-2.0-or-later" diff --git a/dstack/crates/qemu-acpi/reference/README.md b/dstack/crates/qemu-acpi/reference/README.md index 5eabc004e..971d0c0c9 100644 --- a/dstack/crates/qemu-acpi/reference/README.md +++ b/dstack/crates/qemu-acpi/reference/README.md @@ -1,10 +1,36 @@ # dstack ACPI reference image This directory builds the production QEMU compatibility fork at revision -`9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195` with its test-only +`0f3d3f6ed099e4cf0b79f59e8b6ba0083b7c414f` with its test-only `DUMP_ACPI_TABLES` mode. The resulting command writes QEMU's 128 KiB `etc/acpi/tables` blob to standard output and exits without starting a VM. +That revision samples the blob after the PCI bridge `BSEL` properties are +assigned. QEMU only assigns them during machine reset and rebuilds the tables +afterwards, so the blob a guest measures is the post-reset one. Earlier +revisions dumped from inside `acpi_setup()`, which yielded a DSDT with no +`BSEL` anywhere and therefore none of the root-port hotplug AML a GPU CVM +measures — an oracle built from them silently agreed with a generator that +omitted those terms. + +## What this oracle cannot test + +Two measured inputs are out of its reach, so a green differential run says +nothing about them: + +- **The 64-bit PCI window (`_CRS`) when `pci_hole64_size` is left at 0.** QEMU + derives that window from `pci_bus_get_w64_range()`, which only sees BARs the + guest firmware has already assigned. The oracle exits before any firmware + runs, so the range is always empty and the window always falls back to the + configured size. Adding a device with a large 64-bit BAR does not help: a + 64 GiB `ivshmem-plain` BAR still leaves the window at the 32 GiB default. + A real GPU CVM grows it to the span OVMF assigned — several TiB for eight + B200s — which no dump-and-exit oracle can reproduce. Set an explicit + `qemu_pci_hole64_size` on GPU hosts; the explicit path is covered here. +- **More than one PXB.** `dstack-vmm` emits one `pxb-pcie` per GPU NUMA node, + but `MachineConfig` carries only `hugepages` and `num_gpus`, so a multi-node + topology cannot be expressed as a case at all. + The image is a differential-test oracle only. Production Rust code does not depend on it. The source revision and GPL license are recorded as OCI labels; the corresponding source is available from the repository and revision named @@ -13,6 +39,6 @@ in the labels. Build locally with: ```sh -docker build -t kvin/dstack-acpi-tables:qemu-11.1 \ +docker build -t kvin/dstack-acpi-tables:qemu-11.1-20260911 \ -f dstack/crates/qemu-acpi/reference/Dockerfile . ``` diff --git a/dstack/crates/qemu-acpi/scripts/differential-random.py b/dstack/crates/qemu-acpi/scripts/differential-random.py index 409cb4c29..87b49d1d2 100755 --- a/dstack/crates/qemu-acpi/scripts/differential-random.py +++ b/dstack/crates/qemu-acpi/scripts/differential-random.py @@ -39,6 +39,10 @@ class Case: smm: bool = False pic: bool = False pci_hole64_size: int = 0 + # How the host-shared directory is attached. dstack-mr accepts all three + # and models none of them, on the theory that each contributes exactly one + # PCI slot and the DSDT records only _ADR. These cases hold it to that. + share_mode: str = "9p" def fixed_cases(): @@ -58,6 +62,27 @@ def fixed_cases(): Case(hugepages=True, gpus=1), Case(hugepages=True, gpus=8, switches=4), Case(version="9.1.0", hugepages=True, gpus=1, hotplug_off=True), + # Root ports on pcie.0 carry ACPI hotplug AML whose BSEL values follow + # QEMU's reverse child-list walk, and the whole block disappears when + # bridge hotplug is off. Pin both branches with root ports present. + Case(gpus=8), + Case(gpus=8, hotplug_off=True), + Case(hugepages=True, gpus=8, switches=4, hotplug_off=True), + # The per-version cases above all run without root ports, so no version + # tier was pinned against the hotplug AML. Cross the two at the tier + # boundaries: pre-9.2 link triggering, the 9.2 cutoff, and the 11.1 + # serial IRQ change. + Case(version="8.0.0", gpus=8), + Case(version="9.2.0", gpus=8), + Case(version="11.1.0", gpus=8), + # Likewise the large-CPU cases are all GPU-less, so the CPU AML has + # never been pinned alongside root ports. 255 is the x2APIC boundary. + Case(cpus=255, gpus=8), + Case(cpus=256, gpus=8, switches=4), + # dstack-mr accepts three host sharing modes and models none of them. + Case(share_mode="vvfat"), + Case(share_mode="vhd"), + Case(share_mode="vhd", gpus=8, switches=2), ] return cases @@ -93,6 +118,7 @@ def random_case(rng): smm=rng.choice([False, True]), pic=rng.choice([False, True]), pci_hole64_size=rng.choice([0, 32 << 30, 1 << 40]), + share_mode=rng.choice(["9p", "vvfat", "vhd"]), ) @@ -127,12 +153,8 @@ def qemu_args(case): "-device", f"virtio-net-pci,netdev=net{index}", ] - args += [ - "-device", - "vhost-vsock-pci,guest-cid=3", - "-virtfs", - "local,path=/bin,mount_tag=host-shared,readonly=on,security_model=none,id=virtfs0", - ] + args += ["-device", "vhost-vsock-pci,guest-cid=3"] + args += host_share_args(case.share_mode) if case.root_verity: args += [ "-drive", @@ -186,6 +208,31 @@ def qemu_args(case): return args +def host_share_args(mode): + """Return the -device arguments dstack-vmm emits for each host sharing mode.""" + if mode == "9p": + return [ + "-virtfs", + "local,path=/bin,mount_tag=host-shared,readonly=on," + "security_model=none,id=virtfs0", + ] + if mode == "vvfat": + return [ + "-blockdev", + "driver=vvfat,node-name=vvfat0,read-only=on,dir=/tmp,label=SHARED", + "-device", + "virtio-blk-pci,drive=vvfat0", + ] + if mode == "vhd": + return [ + "-drive", + "file=/bin/sh,if=none,id=hd2,format=raw,readonly=on", + "-device", + "virtio-blk-pci,drive=hd2", + ] + raise RuntimeError(f"unknown host sharing mode {mode}") + + def command(kind, payload): data = struct.pack(" -SPDX-License-Identifier: Apache-2.0 - -This instrumentation does not alter ACPI construction. When -QEMU_ACPI_DUMP_DIR is set, it writes the three measured blobs and exits. -Without that variable, the existing DUMP_ACPI_TABLES behavior is unchanged. - ---- a/hw/i386/acpi-build.c -+++ b/hw/i386/acpi-build.c -@@ -2775,6 +2775,31 @@ void acpi_setup(void) - - #ifdef DUMP_ACPI_TABLES - { -+ const char *dump_dir = getenv("QEMU_ACPI_DUMP_DIR"); -+ if (dump_dir) { -+ struct { -+ const char *name; -+ GArray *blob; -+ } outputs[] = { -+ { "tables.bin", tables.table_data }, -+ { "loader.bin", tables.linker->cmd_blob }, -+ { "rsdp.bin", tables.rsdp }, -+ }; -+ -+ for (size_t i = 0; i < ARRAY_SIZE(outputs); i++) { -+ g_autofree char *path = g_build_filename(dump_dir, -+ outputs[i].name, -+ NULL); -+ g_autoptr(GError) error = NULL; -+ if (!g_file_set_contents(path, outputs[i].blob->data, -+ outputs[i].blob->len, &error)) { -+ error_report("failed to dump %s: %s", path, -+ error->message); -+ exit(1); -+ } -+ } -+ exit(0); -+ } - uint8_t *ptr = (uint8_t *)tables.table_data->data; - uint64_t size = tables.table_data->len; - int flags = fcntl(1, F_GETFL);