fix(qemu-acpi): emit ACPI PCI hotplug AML for root-port buses

.github/workflows/qemu-acpi-differential.yml

@@ -32,7 +32,7 @@ jobs:
       run:
         working-directory: dstack
     env:
-      REFERENCE_IMAGE: kvin/dstack-acpi-tables@sha256:54e692d8c68c6f02dd7c655bf6de6e7f3ad7a43fded6ca00d8e998918108a3b4
+      REFERENCE_IMAGE: kvin/dstack-acpi-tables@sha256:98c42e609d84408cfb9a5e95e04f22058bb999ea62c72de43b9b2a18954d14e6
     steps:
       - uses: actions/checkout@v5
 

dstack/crates/qemu-acpi/fixtures/README.md

@@ -10,7 +10,7 @@ The fixtures in this directory were generated from:
 
 - repository: <https://github.com/kvinwang/qemu-tdx>
 - branch: `dstack-qemu-acpi-11.1-compat`
-- revision: `9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195`
+- revision: `0f3d3f6ed099e4cf0b79f59e8b6ba0083b7c414f`
 - dstack image inputs: `dstack-0.5.5/ovmf.fd` and
   `dstack-0.5.5/bzImage`
 
@@ -19,8 +19,7 @@ Build the reference in a clean build directory:
 ```bash
 git clone https://github.com/kvinwang/qemu-tdx.git qemu-tdx
 cd qemu-tdx
-git checkout 9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195
-git apply /path/to/qemu-acpi/scripts/qemu-dump-all-blobs.patch
+git checkout 0f3d3f6ed099e4cf0b79f59e8b6ba0083b7c414f
 mkdir build-acpi && cd build-acpi
 CFLAGS='-DDUMP_ACPI_TABLES -Wno-builtin-macro-redefined -D__DATE__="" -D__TIME__="" -D__TIMESTAMP__=""' \
 LDFLAGS='-Wl,--build-id=none' \
@@ -29,6 +28,13 @@ LDFLAGS='-Wl,--build-id=none' \
 ninja qemu-system-x86_64
 ```
 
+That revision samples the blobs after the PCI bridge `BSEL` properties are
+assigned, which QEMU only does during machine reset. Capturing from a build
+that dumps inside `acpi_setup()` yields tables missing every root-port hotplug
+method, so any fixture taken from a VM with PCIe root ports (GPU or NVSwitch
+passthrough) would be wrong. Fixtures for VMs with no root ports are
+unaffected either way.
+
 Run the complete three-blob differential matrix with:
 
 ```bash

dstack/crates/qemu-acpi/reference/Dockerfile

@@ -4,7 +4,7 @@
 FROM ubuntu:24.04 AS builder
 
 ARG QEMU_REPOSITORY=https://github.com/kvinwang/qemu-tdx.git
-ARG QEMU_REVISION=9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195
+ARG QEMU_REVISION=0f3d3f6ed099e4cf0b79f59e8b6ba0083b7c414f
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         ca-certificates git ninja-build pkg-config python3 python3-venv \
@@ -31,7 +31,7 @@ RUN CFLAGS='-O2 -DDUMP_ACPI_TABLES -Wno-builtin-macro-redefined -D__DATE__="" -D
 
 FROM ubuntu:24.04
 
-ARG QEMU_REVISION=9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195
+ARG QEMU_REVISION=0f3d3f6ed099e4cf0b79f59e8b6ba0083b7c414f
 LABEL org.opencontainers.image.source="https://github.com/kvinwang/qemu-tdx" \
       org.opencontainers.image.revision="${QEMU_REVISION}" \
       org.opencontainers.image.licenses="GPL-2.0-or-later"

dstack/crates/qemu-acpi/reference/README.md

@@ -1,10 +1,36 @@
 # dstack ACPI reference image
 
 This directory builds the production QEMU compatibility fork at revision
-`9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195` with its test-only
+`0f3d3f6ed099e4cf0b79f59e8b6ba0083b7c414f` with its test-only
 `DUMP_ACPI_TABLES` mode. The resulting command writes QEMU's 128 KiB
 `etc/acpi/tables` blob to standard output and exits without starting a VM.
 
+That revision samples the blob after the PCI bridge `BSEL` properties are
+assigned. QEMU only assigns them during machine reset and rebuilds the tables
+afterwards, so the blob a guest measures is the post-reset one. Earlier
+revisions dumped from inside `acpi_setup()`, which yielded a DSDT with no
+`BSEL` anywhere and therefore none of the root-port hotplug AML a GPU CVM
+measures — an oracle built from them silently agreed with a generator that
+omitted those terms.
+
+## What this oracle cannot test
+
+Two measured inputs are out of its reach, so a green differential run says
+nothing about them:
+
+- **The 64-bit PCI window (`_CRS`) when `pci_hole64_size` is left at 0.** QEMU
+  derives that window from `pci_bus_get_w64_range()`, which only sees BARs the
+  guest firmware has already assigned. The oracle exits before any firmware
+  runs, so the range is always empty and the window always falls back to the
+  configured size. Adding a device with a large 64-bit BAR does not help: a
+  64 GiB `ivshmem-plain` BAR still leaves the window at the 32 GiB default.
+  A real GPU CVM grows it to the span OVMF assigned — several TiB for eight
+  B200s — which no dump-and-exit oracle can reproduce. Set an explicit
+  `qemu_pci_hole64_size` on GPU hosts; the explicit path is covered here.
+- **More than one PXB.** `dstack-vmm` emits one `pxb-pcie` per GPU NUMA node,
+  but `MachineConfig` carries only `hugepages` and `num_gpus`, so a multi-node
+  topology cannot be expressed as a case at all.
+
 The image is a differential-test oracle only. Production Rust code does not
 depend on it. The source revision and GPL license are recorded as OCI labels;
 the corresponding source is available from the repository and revision named
@@ -13,6 +39,6 @@ in the labels.
 Build locally with:
 
 ```sh
-docker build -t kvin/dstack-acpi-tables:qemu-11.1 \
+docker build -t kvin/dstack-acpi-tables:qemu-11.1-20260911 \
   -f dstack/crates/qemu-acpi/reference/Dockerfile .
 ```

dstack/crates/qemu-acpi/scripts/differential-random.py

@@ -39,6 +39,10 @@ class Case:
     smm: bool = False
     pic: bool = False
     pci_hole64_size: int = 0
+    # How the host-shared directory is attached. dstack-mr accepts all three
+    # and models none of them, on the theory that each contributes exactly one
+    # PCI slot and the DSDT records only _ADR. These cases hold it to that.
+    share_mode: str = "9p"
 
 
 def fixed_cases():
@@ -58,6 +62,27 @@ def fixed_cases():
         Case(hugepages=True, gpus=1),
         Case(hugepages=True, gpus=8, switches=4),
         Case(version="9.1.0", hugepages=True, gpus=1, hotplug_off=True),
+        # Root ports on pcie.0 carry ACPI hotplug AML whose BSEL values follow
+        # QEMU's reverse child-list walk, and the whole block disappears when
+        # bridge hotplug is off. Pin both branches with root ports present.
+        Case(gpus=8),
+        Case(gpus=8, hotplug_off=True),
+        Case(hugepages=True, gpus=8, switches=4, hotplug_off=True),
+        # The per-version cases above all run without root ports, so no version
+        # tier was pinned against the hotplug AML. Cross the two at the tier
+        # boundaries: pre-9.2 link triggering, the 9.2 cutoff, and the 11.1
+        # serial IRQ change.
+        Case(version="8.0.0", gpus=8),
+        Case(version="9.2.0", gpus=8),
+        Case(version="11.1.0", gpus=8),
+        # Likewise the large-CPU cases are all GPU-less, so the CPU AML has
+        # never been pinned alongside root ports. 255 is the x2APIC boundary.
+        Case(cpus=255, gpus=8),
+        Case(cpus=256, gpus=8, switches=4),
+        # dstack-mr accepts three host sharing modes and models none of them.
+        Case(share_mode="vvfat"),
+        Case(share_mode="vhd"),
+        Case(share_mode="vhd", gpus=8, switches=2),
     ]
     return cases
 
@@ -93,6 +118,7 @@ def random_case(rng):
         smm=rng.choice([False, True]),
         pic=rng.choice([False, True]),
         pci_hole64_size=rng.choice([0, 32 << 30, 1 << 40]),
+        share_mode=rng.choice(["9p", "vvfat", "vhd"]),
     )
 
 
@@ -127,12 +153,8 @@ def qemu_args(case):
             "-device",
             f"virtio-net-pci,netdev=net{index}",
         ]
-    args += [
-        "-device",
-        "vhost-vsock-pci,guest-cid=3",
-        "-virtfs",
-        "local,path=/bin,mount_tag=host-shared,readonly=on,security_model=none,id=virtfs0",
-    ]
+    args += ["-device", "vhost-vsock-pci,guest-cid=3"]
+    args += host_share_args(case.share_mode)
     if case.root_verity:
         args += [
             "-drive",
@@ -186,6 +208,31 @@ def qemu_args(case):
     return args
 
 
+def host_share_args(mode):
+    """Return the -device arguments dstack-vmm emits for each host sharing mode."""
+    if mode == "9p":
+        return [
+            "-virtfs",
+            "local,path=/bin,mount_tag=host-shared,readonly=on,"
+            "security_model=none,id=virtfs0",
+        ]
+    if mode == "vvfat":
+        return [
+            "-blockdev",
+            "driver=vvfat,node-name=vvfat0,read-only=on,dir=/tmp,label=SHARED",
+            "-device",
+            "virtio-blk-pci,drive=vvfat0",
+        ]
+    if mode == "vhd":
+        return [
+            "-drive",
+            "file=/bin/sh,if=none,id=hd2,format=raw,readonly=on",
+            "-device",
+            "virtio-blk-pci,drive=hd2",
+        ]
+    raise RuntimeError(f"unknown host sharing mode {mode}")
+
+
 def command(kind, payload):
     data = struct.pack("<I", kind) + payload
     return data + bytes(LOADER_COMMAND_SIZE - len(data))

dstack/crates/qemu-acpi/src/dsdt/gpe.rs

@@ -24,7 +24,7 @@
 //! `_E01` is deliberately empty here: it is the PCI hotplug event, and the
 //! baseline machine has no hotplug-capable bridge for it to scan.
 
-use acpi_tables::aml::{Method, MethodCall, Name, Path, Scope};
+use acpi_tables::aml::{Acquire, Method, MethodCall, Name, Path, Release, Scope};
 
 use super::ops::emit_all;
 
@@ -49,10 +49,18 @@ pub(crate) fn e02() -> Vec<u8> {
     )])
 }
 
-/// `Scope (_GPE) { Method (_E01, 0, NotSerialized) {} }`, the PCI hotplug
-/// event.
-pub(crate) fn e01() -> Vec<u8> {
-    let handler = Method::new(Path::new("_E01"), 0, false, vec![]);
+/// `Scope (_GPE) { Method (_E01, 0, NotSerialized) { ... } }`, the PCI
+/// hotplug event. QEMU leaves the method empty when no bus supplies PCNT.
+pub(crate) fn e01(has_pcnt: bool) -> Vec<u8> {
+    let acquire = Acquire::new(Path::new("\\_SB_.PCI0.BLCK"), 0xffff);
+    let scan = MethodCall::new(Path::new("\\_SB_.PCI0.PCNT"), vec![]);
+    let release = Release::new(Path::new("\\_SB_.PCI0.BLCK"));
+    let children: Vec<&dyn acpi_tables::Aml> = if has_pcnt {
+        vec![&acquire, &scan, &release]
+    } else {
+        vec![]
+    };
+    let handler = Method::new(Path::new("_E01"), 0, false, children);
     emit_all(&[&Scope::new(Path::new("_GPE"), vec![&handler])])
 }
 
@@ -70,6 +78,18 @@ mod tests {
 
     #[test]
     fn e01_matches_qemu() {
-        super::super::fixture::assert_region(&super::e01(), 8245, 8258);
+        super::super::fixture::assert_region(&super::e01(false), 8245, 8258);
+    }
+
+    #[test]
+    fn e01_scans_root_port_buses_when_pcnt_exists() {
+        use sha2::{Digest, Sha256};
+
+        let e01 = super::e01(true);
+        assert_eq!(e01.len() - super::e01(false).len(), 51);
+        assert_eq!(
+            hex::encode(Sha256::digest(e01)),
+            "6e22ff760f3c9e6263713cd7590518d1fd5ca2cd2344e8856bb618d2bef6d470"
+        );
     }
 }

dstack/crates/qemu-acpi/src/dsdt/mod.rs

@@ -101,14 +101,22 @@ pub(crate) fn body(config: &MachineConfig) -> Result<Vec<u8>, Error> {
     ));
     out.extend(sstate::build()); //      7732..7774  Scope(\) _S3/_S4/_S5
     out.extend(fwcf::build()); //        7774..7834  Scope(\_SB.PCI0) FWCF
+    let pxb_devfn = has_pxb.then_some(0x80);
     out.extend(notify::build(
         regular_slots,
         root_ports,
         modern_serial_irq,
-        has_pxb.then_some(0x80),
+        pxb_devfn,
+        pci_hotplug,
     ));
+    let pcnt = pci_hotplug
+        .then(|| notify::pcnt(regular_slots, root_ports, pxb_devfn))
+        .flatten();
+    if let Some(pcnt) = &pcnt {
+        out.extend(pcnt);
+    }
     if pci_hotplug {
-        out.extend(gpe::e01());
+        out.extend(gpe::e01(pcnt.is_some()));
     }
     Ok(out)
 }