diff --git a/.github/workflows/qemu-acpi-differential.yml b/.github/workflows/qemu-acpi-differential.yml index de8dea222..523fe82b5 100644 --- a/.github/workflows/qemu-acpi-differential.yml +++ b/.github/workflows/qemu-acpi-differential.yml @@ -32,7 +32,7 @@ jobs: run: working-directory: dstack env: - REFERENCE_IMAGE: kvin/dstack-acpi-tables@sha256:54e692d8c68c6f02dd7c655bf6de6e7f3ad7a43fded6ca00d8e998918108a3b4 + REFERENCE_IMAGE: kvin/dstack-acpi-tables@sha256:98c42e609d84408cfb9a5e95e04f22058bb999ea62c72de43b9b2a18954d14e6 steps: - uses: actions/checkout@v5 diff --git a/dstack/crates/qemu-acpi/fixtures/README.md b/dstack/crates/qemu-acpi/fixtures/README.md index 109eb4bd8..787bc47f1 100644 --- a/dstack/crates/qemu-acpi/fixtures/README.md +++ b/dstack/crates/qemu-acpi/fixtures/README.md @@ -10,7 +10,7 @@ The fixtures in this directory were generated from: - repository: - branch: `dstack-qemu-acpi-11.1-compat` -- revision: `9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195` +- revision: `0f3d3f6ed099e4cf0b79f59e8b6ba0083b7c414f` - dstack image inputs: `dstack-0.5.5/ovmf.fd` and `dstack-0.5.5/bzImage` @@ -19,8 +19,7 @@ Build the reference in a clean build directory: ```bash git clone https://github.com/kvinwang/qemu-tdx.git qemu-tdx cd qemu-tdx -git checkout 9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195 -git apply /path/to/qemu-acpi/scripts/qemu-dump-all-blobs.patch +git checkout 0f3d3f6ed099e4cf0b79f59e8b6ba0083b7c414f mkdir build-acpi && cd build-acpi CFLAGS='-DDUMP_ACPI_TABLES -Wno-builtin-macro-redefined -D__DATE__="" -D__TIME__="" -D__TIMESTAMP__=""' \ LDFLAGS='-Wl,--build-id=none' \ @@ -29,6 +28,13 @@ LDFLAGS='-Wl,--build-id=none' \ ninja qemu-system-x86_64 ``` +That revision samples the blobs after the PCI bridge `BSEL` properties are +assigned, which QEMU only does during machine reset. Capturing from a build +that dumps inside `acpi_setup()` yields tables missing every root-port hotplug +method, so any fixture taken from a VM with PCIe root ports (GPU or NVSwitch +passthrough) would be wrong. Fixtures for VMs with no root ports are +unaffected either way. + Run the complete three-blob differential matrix with: ```bash diff --git a/dstack/crates/qemu-acpi/reference/Dockerfile b/dstack/crates/qemu-acpi/reference/Dockerfile index 5a2b55f35..99ee9aed3 100644 --- a/dstack/crates/qemu-acpi/reference/Dockerfile +++ b/dstack/crates/qemu-acpi/reference/Dockerfile @@ -4,7 +4,7 @@ FROM ubuntu:24.04 AS builder ARG QEMU_REPOSITORY=https://github.com/kvinwang/qemu-tdx.git -ARG QEMU_REVISION=9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195 +ARG QEMU_REVISION=0f3d3f6ed099e4cf0b79f59e8b6ba0083b7c414f RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ca-certificates git ninja-build pkg-config python3 python3-venv \ @@ -31,7 +31,7 @@ RUN CFLAGS='-O2 -DDUMP_ACPI_TABLES -Wno-builtin-macro-redefined -D__DATE__="" -D FROM ubuntu:24.04 -ARG QEMU_REVISION=9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195 +ARG QEMU_REVISION=0f3d3f6ed099e4cf0b79f59e8b6ba0083b7c414f LABEL org.opencontainers.image.source="https://github.com/kvinwang/qemu-tdx" \ org.opencontainers.image.revision="${QEMU_REVISION}" \ org.opencontainers.image.licenses="GPL-2.0-or-later" diff --git a/dstack/crates/qemu-acpi/reference/README.md b/dstack/crates/qemu-acpi/reference/README.md index 5eabc004e..971d0c0c9 100644 --- a/dstack/crates/qemu-acpi/reference/README.md +++ b/dstack/crates/qemu-acpi/reference/README.md @@ -1,10 +1,36 @@ # dstack ACPI reference image This directory builds the production QEMU compatibility fork at revision -`9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195` with its test-only +`0f3d3f6ed099e4cf0b79f59e8b6ba0083b7c414f` with its test-only `DUMP_ACPI_TABLES` mode. The resulting command writes QEMU's 128 KiB `etc/acpi/tables` blob to standard output and exits without starting a VM. +That revision samples the blob after the PCI bridge `BSEL` properties are +assigned. QEMU only assigns them during machine reset and rebuilds the tables +afterwards, so the blob a guest measures is the post-reset one. Earlier +revisions dumped from inside `acpi_setup()`, which yielded a DSDT with no +`BSEL` anywhere and therefore none of the root-port hotplug AML a GPU CVM +measures — an oracle built from them silently agreed with a generator that +omitted those terms. + +## What this oracle cannot test + +Two measured inputs are out of its reach, so a green differential run says +nothing about them: + +- **The 64-bit PCI window (`_CRS`) when `pci_hole64_size` is left at 0.** QEMU + derives that window from `pci_bus_get_w64_range()`, which only sees BARs the + guest firmware has already assigned. The oracle exits before any firmware + runs, so the range is always empty and the window always falls back to the + configured size. Adding a device with a large 64-bit BAR does not help: a + 64 GiB `ivshmem-plain` BAR still leaves the window at the 32 GiB default. + A real GPU CVM grows it to the span OVMF assigned — several TiB for eight + B200s — which no dump-and-exit oracle can reproduce. Set an explicit + `qemu_pci_hole64_size` on GPU hosts; the explicit path is covered here. +- **More than one PXB.** `dstack-vmm` emits one `pxb-pcie` per GPU NUMA node, + but `MachineConfig` carries only `hugepages` and `num_gpus`, so a multi-node + topology cannot be expressed as a case at all. + The image is a differential-test oracle only. Production Rust code does not depend on it. The source revision and GPL license are recorded as OCI labels; the corresponding source is available from the repository and revision named @@ -13,6 +39,6 @@ in the labels. Build locally with: ```sh -docker build -t kvin/dstack-acpi-tables:qemu-11.1 \ +docker build -t kvin/dstack-acpi-tables:qemu-11.1-20260911 \ -f dstack/crates/qemu-acpi/reference/Dockerfile . ``` diff --git a/dstack/crates/qemu-acpi/scripts/differential-random.py b/dstack/crates/qemu-acpi/scripts/differential-random.py index 409cb4c29..87b49d1d2 100755 --- a/dstack/crates/qemu-acpi/scripts/differential-random.py +++ b/dstack/crates/qemu-acpi/scripts/differential-random.py @@ -39,6 +39,10 @@ class Case: smm: bool = False pic: bool = False pci_hole64_size: int = 0 + # How the host-shared directory is attached. dstack-mr accepts all three + # and models none of them, on the theory that each contributes exactly one + # PCI slot and the DSDT records only _ADR. These cases hold it to that. + share_mode: str = "9p" def fixed_cases(): @@ -58,6 +62,27 @@ def fixed_cases(): Case(hugepages=True, gpus=1), Case(hugepages=True, gpus=8, switches=4), Case(version="9.1.0", hugepages=True, gpus=1, hotplug_off=True), + # Root ports on pcie.0 carry ACPI hotplug AML whose BSEL values follow + # QEMU's reverse child-list walk, and the whole block disappears when + # bridge hotplug is off. Pin both branches with root ports present. + Case(gpus=8), + Case(gpus=8, hotplug_off=True), + Case(hugepages=True, gpus=8, switches=4, hotplug_off=True), + # The per-version cases above all run without root ports, so no version + # tier was pinned against the hotplug AML. Cross the two at the tier + # boundaries: pre-9.2 link triggering, the 9.2 cutoff, and the 11.1 + # serial IRQ change. + Case(version="8.0.0", gpus=8), + Case(version="9.2.0", gpus=8), + Case(version="11.1.0", gpus=8), + # Likewise the large-CPU cases are all GPU-less, so the CPU AML has + # never been pinned alongside root ports. 255 is the x2APIC boundary. + Case(cpus=255, gpus=8), + Case(cpus=256, gpus=8, switches=4), + # dstack-mr accepts three host sharing modes and models none of them. + Case(share_mode="vvfat"), + Case(share_mode="vhd"), + Case(share_mode="vhd", gpus=8, switches=2), ] return cases @@ -93,6 +118,7 @@ def random_case(rng): smm=rng.choice([False, True]), pic=rng.choice([False, True]), pci_hole64_size=rng.choice([0, 32 << 30, 1 << 40]), + share_mode=rng.choice(["9p", "vvfat", "vhd"]), ) @@ -127,12 +153,8 @@ def qemu_args(case): "-device", f"virtio-net-pci,netdev=net{index}", ] - args += [ - "-device", - "vhost-vsock-pci,guest-cid=3", - "-virtfs", - "local,path=/bin,mount_tag=host-shared,readonly=on,security_model=none,id=virtfs0", - ] + args += ["-device", "vhost-vsock-pci,guest-cid=3"] + args += host_share_args(case.share_mode) if case.root_verity: args += [ "-drive", @@ -186,6 +208,31 @@ def qemu_args(case): return args +def host_share_args(mode): + """Return the -device arguments dstack-vmm emits for each host sharing mode.""" + if mode == "9p": + return [ + "-virtfs", + "local,path=/bin,mount_tag=host-shared,readonly=on," + "security_model=none,id=virtfs0", + ] + if mode == "vvfat": + return [ + "-blockdev", + "driver=vvfat,node-name=vvfat0,read-only=on,dir=/tmp,label=SHARED", + "-device", + "virtio-blk-pci,drive=vvfat0", + ] + if mode == "vhd": + return [ + "-drive", + "file=/bin/sh,if=none,id=hd2,format=raw,readonly=on", + "-device", + "virtio-blk-pci,drive=hd2", + ] + raise RuntimeError(f"unknown host sharing mode {mode}") + + def command(kind, payload): data = struct.pack(" -SPDX-License-Identifier: Apache-2.0 - -This instrumentation does not alter ACPI construction. When -QEMU_ACPI_DUMP_DIR is set, it writes the three measured blobs and exits. -Without that variable, the existing DUMP_ACPI_TABLES behavior is unchanged. - ---- a/hw/i386/acpi-build.c -+++ b/hw/i386/acpi-build.c -@@ -2775,6 +2775,31 @@ void acpi_setup(void) - - #ifdef DUMP_ACPI_TABLES - { -+ const char *dump_dir = getenv("QEMU_ACPI_DUMP_DIR"); -+ if (dump_dir) { -+ struct { -+ const char *name; -+ GArray *blob; -+ } outputs[] = { -+ { "tables.bin", tables.table_data }, -+ { "loader.bin", tables.linker->cmd_blob }, -+ { "rsdp.bin", tables.rsdp }, -+ }; -+ -+ for (size_t i = 0; i < ARRAY_SIZE(outputs); i++) { -+ g_autofree char *path = g_build_filename(dump_dir, -+ outputs[i].name, -+ NULL); -+ g_autoptr(GError) error = NULL; -+ if (!g_file_set_contents(path, outputs[i].blob->data, -+ outputs[i].blob->len, &error)) { -+ error_report("failed to dump %s: %s", path, -+ error->message); -+ exit(1); -+ } -+ } -+ exit(0); -+ } - uint8_t *ptr = (uint8_t *)tables.table_data->data; - uint64_t size = tables.table_data->len; - int flags = fcntl(1, F_GETFL); diff --git a/dstack/crates/qemu-acpi/src/dsdt/gpe.rs b/dstack/crates/qemu-acpi/src/dsdt/gpe.rs index 46342124b..4a21e68c3 100644 --- a/dstack/crates/qemu-acpi/src/dsdt/gpe.rs +++ b/dstack/crates/qemu-acpi/src/dsdt/gpe.rs @@ -24,7 +24,7 @@ //! `_E01` is deliberately empty here: it is the PCI hotplug event, and the //! baseline machine has no hotplug-capable bridge for it to scan. -use acpi_tables::aml::{Method, MethodCall, Name, Path, Scope}; +use acpi_tables::aml::{Acquire, Method, MethodCall, Name, Path, Release, Scope}; use super::ops::emit_all; @@ -49,10 +49,18 @@ pub(crate) fn e02() -> Vec { )]) } -/// `Scope (_GPE) { Method (_E01, 0, NotSerialized) {} }`, the PCI hotplug -/// event. -pub(crate) fn e01() -> Vec { - let handler = Method::new(Path::new("_E01"), 0, false, vec![]); +/// `Scope (_GPE) { Method (_E01, 0, NotSerialized) { ... } }`, the PCI +/// hotplug event. QEMU leaves the method empty when no bus supplies PCNT. +pub(crate) fn e01(has_pcnt: bool) -> Vec { + let acquire = Acquire::new(Path::new("\\_SB_.PCI0.BLCK"), 0xffff); + let scan = MethodCall::new(Path::new("\\_SB_.PCI0.PCNT"), vec![]); + let release = Release::new(Path::new("\\_SB_.PCI0.BLCK")); + let children: Vec<&dyn acpi_tables::Aml> = if has_pcnt { + vec![&acquire, &scan, &release] + } else { + vec![] + }; + let handler = Method::new(Path::new("_E01"), 0, false, children); emit_all(&[&Scope::new(Path::new("_GPE"), vec![&handler])]) } @@ -70,6 +78,18 @@ mod tests { #[test] fn e01_matches_qemu() { - super::super::fixture::assert_region(&super::e01(), 8245, 8258); + super::super::fixture::assert_region(&super::e01(false), 8245, 8258); + } + + #[test] + fn e01_scans_root_port_buses_when_pcnt_exists() { + use sha2::{Digest, Sha256}; + + let e01 = super::e01(true); + assert_eq!(e01.len() - super::e01(false).len(), 51); + assert_eq!( + hex::encode(Sha256::digest(e01)), + "6e22ff760f3c9e6263713cd7590518d1fd5ca2cd2344e8856bb618d2bef6d470" + ); } } diff --git a/dstack/crates/qemu-acpi/src/dsdt/mod.rs b/dstack/crates/qemu-acpi/src/dsdt/mod.rs index 4f33fb9de..26eba15ea 100644 --- a/dstack/crates/qemu-acpi/src/dsdt/mod.rs +++ b/dstack/crates/qemu-acpi/src/dsdt/mod.rs @@ -101,14 +101,22 @@ pub(crate) fn body(config: &MachineConfig) -> Result, Error> { )); out.extend(sstate::build()); // 7732..7774 Scope(\) _S3/_S4/_S5 out.extend(fwcf::build()); // 7774..7834 Scope(\_SB.PCI0) FWCF + let pxb_devfn = has_pxb.then_some(0x80); out.extend(notify::build( regular_slots, root_ports, modern_serial_irq, - has_pxb.then_some(0x80), + pxb_devfn, + pci_hotplug, )); + let pcnt = pci_hotplug + .then(|| notify::pcnt(regular_slots, root_ports, pxb_devfn)) + .flatten(); + if let Some(pcnt) = &pcnt { + out.extend(pcnt); + } if pci_hotplug { - out.extend(gpe::e01()); + out.extend(gpe::e01(pcnt.is_some())); } Ok(out) } diff --git a/dstack/crates/qemu-acpi/src/dsdt/notify.rs b/dstack/crates/qemu-acpi/src/dsdt/notify.rs index 2039a5bfc..c52646324 100644 --- a/dstack/crates/qemu-acpi/src/dsdt/notify.rs +++ b/dstack/crates/qemu-acpi/src/dsdt/notify.rs @@ -7,8 +7,9 @@ //! //! QEMU walks every devfn of the bus (`build_append_pci_bus_devices`, //! `hw/acpi/pcihp.c`) and emits `Device (S) { Name (_ADR, ..) }` for the -//! populated ones, then lets each device append its own AML. Only the LPC -//! bridge does (`build_ich9_isa_aml`, `hw/isa/lpc_ich9.c`). +//! populated ones, then lets each device append its own AML. The LPC bridge +//! appends its ISA children; PCIe root ports append their secondary-bus PCI +//! hotplug slots and recursive notification methods. //! //! ```asl //! Scope (\_SB) { @@ -66,8 +67,9 @@ //! ``` use acpi_tables::aml::{ - Device, EISAName, Field, FieldAccessType, FieldEntry, FieldLockRule, FieldUpdateRule, Name, - OpRegion, OpRegionSpace, Path, ResourceTemplate, Scope, IO, + And, Arg, Device, EISAName, Field, FieldAccessType, FieldEntry, FieldLockRule, FieldUpdateRule, + If, Index, Local, Method, MethodCall, Name, Notify, One, OpRegion, OpRegionSpace, Package, + Path, ResourceTemplate, Return, Scope, Store, Zero, IO, }; use acpi_tables::{Aml, AmlSink}; @@ -85,46 +87,105 @@ pub(crate) fn build( root_port_count: u32, modern_serial_irq: bool, pxb_devfn: Option, + pci_hotplug: bool, ) -> Vec { let lpc = lpc_children(modern_serial_irq); + let root_ports = root_ports(slot_count, root_port_count, pxb_devfn); let mut devices = Vec::new(); for slot in 0..slot_count { - devices.push(((slot * 8) as u8, false)); + devices.push(((slot * 8) as u8, false, None)); } if let Some(devfn) = pxb_devfn { - devices.push((devfn, false)); + devices.push((devfn, false, None)); } - let mut slot = slot_count; - for _ in 0..root_port_count { - if pxb_devfn == Some((slot * 8) as u8) { - slot += 1; - } - devices.push(((slot * 8) as u8, true)); - slot += 1; + for &(devfn, bsel) in &root_ports { + devices.push((devfn, true, pci_hotplug.then_some(bsel))); } for &devfn in CHIPSET_DEVFNS { - devices.push((devfn, false)); + devices.push((devfn, false, None)); } - devices.sort_unstable_by_key(|(devfn, _)| *devfn); + devices.sort_unstable_by_key(|(devfn, _, _)| *devfn); let mut bus = Vec::new(); - for (devfn, root_port) in devices { - bus.extend(pci_device(devfn, root_port, &lpc)); + for (devfn, root_port, bsel) in devices { + bus.extend(pci_device(devfn, root_port, bsel, &lpc)); } Scope::raw(Path::new("\\_SB_"), Scope::raw(Path::new("PCI0"), bus)) } -fn pci_device(devfn: u8, root_port: bool, lpc: &[u8]) -> Vec { +/// Build the recursive notification methods QEMU appends after the root-bus +/// device descriptions. The host bridge has no BSEL; each hotplug-capable +/// root-port secondary bus has one, assigned in bus traversal order. +pub(crate) fn pcnt( + slot_count: u32, + root_port_count: u32, + pxb_devfn: Option, +) -> Option> { + let root_ports = root_ports(slot_count, root_port_count, pxb_devfn); + if root_ports.is_empty() { + return None; + } + + let mut children = Vec::new(); + for &(devfn, bsel) in root_ports.iter().rev() { + children.extend(child_pcnt(devfn, bsel)); + } + + let mut calls = Vec::new(); + for &(devfn, _) in root_ports.iter().rev() { + // `Path` cannot encode AML ParentPrefixChar. A zero-argument method + // invocation is just its NameString, which QEMU emits as + // `^S.PCNT` from the root PCNT method. + calls.push(parent_pcnt_call(devfn)); + } + let raw_calls: Vec<_> = calls.iter().map(|call| Raw(call)).collect(); + let call_refs = raw_calls.iter().map(|call| call as &dyn Aml).collect(); + children.extend(emit(&Method::new(Path::new("PCNT"), 0, false, call_refs))); + + Some(Scope::raw(Path::new("\\_SB_.PCI0"), children)) +} + +/// The `(devfn, BSEL)` of each root port on `pcie.0`, in slot order. +/// +/// QEMU numbers BSEL with a depth-first walk of the root bus's child list +/// (`acpi_set_bsel`, `hw/acpi/pcihp.c`), so the mapping depends on the order +/// dstack-vmm creates devices in: +/// +/// - root ports are created in ascending slot order, one endpoint at function +/// 0 each; +/// - PXB expanders are created before any root port, so their buses sit behind +/// the `pcie.0` root ports in the child list. Root ports under a PXB still +/// take BSEL values, but only after these ones, and QEMU emits no AML for +/// them. +/// +/// Reordering those devices in the VMM changes the measured DSDT. +fn root_ports(slot_count: u32, count: u32, pxb_devfn: Option) -> Vec<(u8, u32)> { + let mut ports = Vec::with_capacity(count as usize); + let mut slot = slot_count; + for index in 0..count { + if pxb_devfn == Some((slot * 8) as u8) { + slot += 1; + } + // QEMU inserts each secondary bus at the head of the root child list, + // then allocates BSEL values by walking that list. Root ports are + // created in ascending slot order, so their BSEL values run backward. + let bsel = count - index - 1; + ports.push(((slot * 8) as u8, bsel)); + slot += 1; + } + ports +} + +fn pci_device(devfn: u8, root_port: bool, bsel: Option, lpc: &[u8]) -> Vec { // QEMU names the device after the devfn but addresses it by the // ACPI 1.0b Table 6-2 PCI form: (device << 16) | function. let name = format!("S{devfn:02X}_"); let address = (u32::from(devfn >> 3) << 16) | u32::from(devfn & 0x07); let adr = Name::new(Path::new("_ADR"), &address); - let child_address = Name::new(Path::new("_ADR"), &0u8); - let child = root_port.then(|| emit(&Device::new(Path::new("S00_"), vec![&child_address]))); + let child = root_port.then(|| root_port_child(bsel)); let extra = Raw(if devfn == LPC_DEVFN { lpc } else { &[] }); let child = Raw(match child.as_deref() { Some(bytes) => bytes, @@ -133,6 +194,90 @@ fn pci_device(devfn: u8, root_port: bool, lpc: &[u8]) -> Vec { emit(&Device::new(Path::new(&name), vec![&adr, &extra, &child])) } +fn root_port_child(bsel: Option) -> Vec { + let child_address = Name::new(Path::new("_ADR"), &Zero {}); + let mut out = emit(&Device::new(Path::new("S00_"), vec![&child_address])); + let Some(bsel) = bsel else { + return out; + }; + + out.extend(emit(&Name::new(Path::new("BSEL"), &bsel))); + out.extend(hotplug_slot()); + out.extend(dvnt()); + out +} + +fn hotplug_slot() -> Vec { + let zero = Zero {}; + let one = One {}; + let asun = Name::new(Path::new("ASUN"), &zero); + + let local0 = Local(0); + let params = Package::new(vec![&zero, &zero]); + let init_params = Store::new(&local0, ¶ms); + let bus_slot = Index::new(&zero, &local0, &zero); + let bsel_name = Path::new("BSEL"); + let store_bus = Store::new(&bus_slot, &bsel_name); + let sun_slot = Index::new(&zero, &local0, &one); + let asun_name = Path::new("ASUN"); + let store_sun = Store::new(&sun_slot, &asun_name); + let (arg0, arg1, arg2, arg3) = (Arg(0), Arg(1), Arg(2), Arg(3)); + let pdsm = MethodCall::new(Path::new("PDSM"), vec![&arg0, &arg1, &arg2, &arg3, &local0]); + let ret = Return::new(&pdsm); + let dsm = Method::new( + Path::new("_DSM"), + 4, + true, + vec![&init_params, &store_bus, &store_sun, &ret], + ); + + let sun = Name::new(Path::new("_SUN"), &zero); + let bsel_name = Path::new("BSEL"); + let sun_name = Path::new("_SUN"); + let eject = MethodCall::new(Path::new("PCEJ"), vec![&bsel_name, &sun_name]); + let ej0 = Method::new(Path::new("_EJ0"), 1, false, vec![&eject]); + + emit(&Scope::new( + Path::new("S00_"), + vec![&asun, &dsm, &sun, &ej0], + )) +} + +fn dvnt() -> Vec { + let arg0 = Arg(0); + let arg1 = Arg(1); + let one = One {}; + let selected = And::new(&Zero {}, &arg0, &one); + let slot = Path::new("S00_"); + let notify = Notify::new(&slot, &arg1); + let branch = If::new(&selected, vec![¬ify]); + emit(&Method::new(Path::new("DVNT"), 2, false, vec![&branch])) +} + +fn child_pcnt(devfn: u8, bsel: u32) -> Vec { + let bnum = Path::new("BNUM"); + let pciu = Path::new("PCIU"); + let pcid = Path::new("PCID"); + let one = One {}; + let eject_request = 3u8; + let select_bus = Store::new(&bnum, &bsel); + let inserted = MethodCall::new(Path::new("DVNT"), vec![&pciu, &one]); + let removed = MethodCall::new(Path::new("DVNT"), vec![&pcid, &eject_request]); + let method = Method::new( + Path::new("PCNT"), + 0, + false, + vec![&select_bus, &inserted, &removed], + ); + Scope::raw(Path::new(&format!("S{devfn:02X}_")), emit(&method)) +} + +fn parent_pcnt_call(devfn: u8) -> Vec { + let mut call = vec![0x5e, 0x2e]; // ParentPrefixChar, DualNamePrefix + call.extend_from_slice(format!("S{devfn:02X}_PCNT").as_bytes()); + call +} + /// The children the ICH9 LPC bridge appends to its own device descriptor. fn lpc_children(modern_serial_irq: bool) -> Vec { // PCI-to-ISA interrupt routing registers in the bridge's config space. @@ -298,6 +443,90 @@ impl Aml for Irq { mod tests { #[test] fn matches_qemu() { - super::super::fixture::assert_region(&super::build(5, 0, true, None), 7834, 8245); + super::super::fixture::assert_region(&super::build(5, 0, true, None, true), 7834, 8245); + } + + #[test] + fn eight_root_ports_include_qemus_hotplug_aml() { + use sha2::{Digest, Sha256}; + + let hotplug_on = super::build(6, 8, true, None, true); + let hotplug_off = super::build(6, 8, true, None, false); + assert_eq!(hotplug_on.len() - hotplug_off.len(), 854); + + let pcnt = super::pcnt(6, 8, None).unwrap(); + assert_eq!(pcnt.len(), 411); + assert_eq!( + hex::encode(Sha256::digest(pcnt)), + "29f0fc0087802ef8886aa8eff10bbd78b471e50f8ce2d543861cd437a2806851" + ); + } + + #[test] + fn root_port_bsel_mapping_follows_qemus_reverse_child_walk() { + assert_eq!( + super::root_ports(6, 8, None), + vec![ + (0x30, 7), + (0x38, 6), + (0x40, 5), + (0x48, 4), + (0x50, 3), + (0x58, 2), + (0x60, 1), + (0x68, 0), + ] + ); + } + + #[test] + fn eight_root_port_devices_match_independent_qemu_encoding() { + use sha2::{Digest, Sha256}; + + let ports = super::root_ports(6, 8, None); + let mut hotplug_off = Vec::new(); + let mut hotplug_on = Vec::new(); + for (devfn, bsel) in ports { + hotplug_off.extend(super::pci_device(devfn, true, None, &[])); + hotplug_on.extend(super::pci_device(devfn, true, Some(bsel), &[])); + } + assert_eq!(hotplug_off.len(), 240); + assert_eq!(hotplug_on.len(), 1094); + assert_eq!( + hex::encode(Sha256::digest(hotplug_off)), + "7a107a6f0cf575cff4f70489c1b61a8c1ef1ae86a89de182ab0f412cea7bf80d" + ); + assert_eq!( + hex::encode(Sha256::digest(hotplug_on)), + "caec37e9162f026c9e36fe4952fb55f6aa7db09fc92e4e785914f84b05a705b3" + ); + } + + #[test] + fn root_port_hotplug_terms_match_independent_qemu_encoding() { + use sha2::{Digest, Sha256}; + + let base = super::root_port_child(None); + for (bsel, expected) in [ + ( + 0, + "2ab2603633fbf9eee2f7b9104d9f2510a55884520f1f02ddc41276fc9dfee90f", + ), + ( + 2, + "4b39e662fdf5334c777209850bb8984abb1dbda74f2f20bd807198e7683662ee", + ), + ] { + let port = super::root_port_child(Some(bsel)); + assert_eq!(hex::encode(Sha256::digest(&port[base.len()..])), expected); + } + } + + #[test] + fn root_port_hotplug_aml_is_conditional() { + let hotplug_off = super::build(6, 8, true, None, false); + let no_ports = super::build(6, 0, true, None, false); + assert_eq!(hotplug_off.len() - no_ports.len(), 8 * 30); + assert!(super::pcnt(6, 0, None).is_none()); } } diff --git a/dstack/crates/qemu-acpi/src/golden_tests.rs b/dstack/crates/qemu-acpi/src/golden_tests.rs index 7a13b4893..908db43a6 100644 --- a/dstack/crates/qemu-acpi/src/golden_tests.rs +++ b/dstack/crates/qemu-acpi/src/golden_tests.rs @@ -41,6 +41,19 @@ mod tests { Ok(()) } + #[test] + fn no_gpu_hotplug_off_matches_qemu_byte_for_byte() -> Result<(), Error> { + let mut c = config(0, 0); + c.hotplug_off = true; + let actual = build(&c)?; + let expected = include_bytes!("../fixtures/qemu-11.1-q35-hotplug-off-base.bin"); + assert_eq!(&actual.tables[..expected.len()], *expected); + assert!(actual.tables[expected.len()..] + .iter() + .all(|byte| *byte == 0)); + Ok(()) + } + #[test] fn numa_loader_and_rsdp_match_qemu_byte_for_byte() -> Result<(), Error> { let mut numa = config(1, 0); @@ -176,28 +189,28 @@ mod tests { 0, 1, 0, - "f47ab428541cb334c6de6e59e7fcf44a5db7b314e9dd4c643978968917eb25b2", + "f22f486b0e33ed0aad80e6cb26726652d671d3c58f521827825a26f069e499a3", ), ( 1, 0, 8, 0, - "ae3fefc72eb747cbff363e4f5ac7f3366f257849dc5f2719a9368303abaef4cc", + "037b07a2b0d6dc1d3b8370bb8d84564b74cd4f9a09163252c4a8b08d9ad0be70", ), ( 1, 0, 1, 1, - "8a48a13bc6041d73f7decce488054a8d25800cc82e11fa9bd1687e010ac9c9b0", + "6f1e598f7b9acbc24c33c573c81112b4e9df7d576d18e82d94b0253c4a69da71", ), ( 1, 0, 1, 4, - "2052ea73c74e1462947e600c95742e48cae0f7a84bc0ec79ab12f7a7818aec7a", + "e23873ce836a73783d043173f9757da1b36151a3a2987c8ab978e33570120eb9", ), ]; for (nics, volumes, gpus, switches, expected) in cases { diff --git a/dstack/verifier/src/verification.rs b/dstack/verifier/src/verification.rs index 6d9e56a3b..f14f96f3c 100644 --- a/dstack/verifier/src/verification.rs +++ b/dstack/verifier/src/verification.rs @@ -2462,6 +2462,56 @@ mod tests { .expect_err("an extra vCPU must change the ACPI tables"); } + /// Reproduces the VM shape from the captured eight-GPU QEMU 10.2.1 + /// attestation. The expected digests come from the CVM event log, not from + /// this generator. + /// + /// The loader digest only pins table sizes and offsets; the tables digest + /// pins the AML bytes themselves, including the root-port hotplug AML. + /// + /// The CVM ran with `pci_hole64_size = 0`, so QEMU sized the DSDT's 64-bit + /// PCI window from the BARs OVMF assigned to the eight GPUs instead. The + /// guest reported that window as `[mem 0x380000000000-0x3bc006013fff]`; + /// this test passes its length explicitly so that the `_CRS` bytes match. + #[test] + fn tdx_lite_acpi_matches_captured_eight_gpu_qemu_10_2_vm() { + let vm_config: VmConfig = serde_json::from_value(serde_json::json!({ + "cpu_count": 256, + "memory_size": 1_649_267_441_664u64, + "qemu_version": "10.2.1", + "pci_hole64_size": 0x3c0_0601_4000u64, + "hugepages": false, + "num_gpus": 8, + "num_nvswitches": 0, + "num_nics": 1, + "num_verity_volumes": 0, + "hotplug_off": false, + "host_share_mode": "9p", + "ovmf_variant": "pre202505", + "tdx_attestation_variant": "lite" + })) + .expect("captured VM config parses"); + + let actual = dstack_mr::tdx::expected_rtmr0_acpi_hashes( + &vm_config, + dstack_types::OvmfVariant::Pre202505, + ) + .expect("ACPI blobs are generated through the verifier measurement path"); + + assert_eq!( + hex::encode(actual.loader), + "01f02cba34d8f7213872ce341a587c636016a6268d0b9c2d2198058a8a709fe38cde80f67c808067444cb8fa11cdd27c" + ); + assert_eq!( + hex::encode(actual.rsdp), + "a5a7aa6b9b601386fab910a1840bbcef2a87c130e683b9630ebe53311fe1ed7cc80863f29f167350c147ae34082981e9" + ); + assert_eq!( + hex::encode(actual.tables), + "bbfdfdee5883455bce28fee44d807f2ed9999651e860aba5b2cd91d35040d7ef42ff36bb0fe5ace1299501e869519344" + ); + } + #[test] fn tdx_lite_acpi_hash_mismatch_names_the_table() { let expected = TdxRtmr0AcpiHashes {