openvmm_entry/ttrpc: expose PCIe NUMA and ACS config

Cargo.lock

@@ -10546,6 +10546,7 @@ dependencies = [
  "get_resources",
  "guid",
  "hyperv_ic_resources",
+ "initrd_cpio",
  "inspect",
  "jiff",
  "kmsg",

openvmm/membacking/src/mapping_manager/manager.rs

@@ -56,18 +56,24 @@ impl MappingManager {
     /// that is enforced dynamically by the manager task as mappings and mappers
     /// come and go (private RAM may be added later, e.g. via hotplug) rather
     /// than captured here at construction time.
+    ///
+    /// `soft_large_pages` reports whether the partition can resolve guest memory
+    /// faults; it gates the Windows soft-large-page (deferred-protect) path on
+    /// the primary mapper, since that scheme relies on the partition delivering
+    /// the first write fault back to the VMM.
     pub async fn new(
         spawn: impl Spawn,
         max_addr: u64,
         minimum_va_alignment: Option<usize>,
+        soft_large_pages: bool,
     ) -> Result<(Self, Arc<VaMapper>), VaMapperError> {
         let this = Self::new_bare(spawn, max_addr, minimum_va_alignment);
         // Create the primary mapper as part of construction. Being first, it is
         // the instance the shared cache hands to every later `new_mapper` in
         // this process.
         let primary = this
             .client()
-            .get_or_create_mapper(true, MapperRole::Primary)
+            .get_or_create_mapper(true, MapperRole::Primary { soft_large_pages })
             .await?;
         Ok((this, primary))
     }
@@ -1481,7 +1487,9 @@ mod tests {
             None,
             None,
             true, // eager
-            MapperRole::Primary,
+            MapperRole::Primary {
+                soft_large_pages: true,
+            },
         );
         let (mapper, _) = futures::join!(mapper_future, async {
             let msg = req_recv.recv().await.unwrap();
@@ -1519,7 +1527,9 @@ mod tests {
             None,
             None,
             true, // eager
-            MapperRole::Primary,
+            MapperRole::Primary {
+                soft_large_pages: true,
+            },
         );
         let (mapper, mapper_req_send) = futures::join!(mapper_future, async {
             let msg = req_recv.recv().await.unwrap();

openvmm/membacking/src/mapping_manager/va_mapper.rs

@@ -117,8 +117,14 @@ enum FaultError {
 /// handled correctly.
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 pub(crate) enum MapperRole {
-    /// The single loader/partition mapper; eligible for soft large pages.
-    Primary,
+    /// The single loader/partition mapper. Eligible for soft large pages only
+    /// when `soft_large_pages` is set, i.e. the partition can resolve guest
+    /// memory faults: the deferred-protect scheme maps guest RAM read-only and
+    /// relies on the partition delivering the first write fault back to the VMM
+    /// to upgrade it. Backends that can't resolve faults (e.g. the aarch64 WHP
+    /// memory-intercept handler) must clear this and use plain read-write 4 KB
+    /// pages instead.
+    Primary { soft_large_pages: bool },
     /// Any other mapper; plain read-write 4 KB pages.
     Secondary,
 }
@@ -277,11 +283,14 @@ struct MapperInner {
     /// succeeds because the mapping is already established. The flag
     /// is eventually updated after `SetEager` is processed.
     eager: AtomicBool,
-    /// Whether this is the **primary** mapper — the one the partition and the
-    /// loader run against. Soft large pages (Windows) are only worthwhile here,
-    /// since this is the mapping whose host backing drives the guest's SLAT.
-    /// Secondary mappers use plain read-write 4 KB pages. See [`MapperRole`].
-    primary: bool,
+    /// Whether this mapper is eligible for the Windows soft-large-page
+    /// (deferred-protect) path. True only for the **primary** mapper — the one
+    /// the partition and loader run against, whose host backing drives the
+    /// guest's SLAT — and only when the partition can resolve guest memory
+    /// faults. Secondary mappers, and partitions without fault resolution (e.g.
+    /// the aarch64 WHP backend), use plain read-write 4 KB pages. See
+    /// [`MapperRole`].
+    soft_large_pages: bool,
     req_send: mesh::Sender<MappingRequest>,
 }
 
@@ -373,14 +382,18 @@ impl MapperTask {
     /// Establishes a mapping in the VA space, dispatching on how it is backed.
     fn map(&self, params: MappingParams) -> Result<(), MappingError> {
         // Soft large pages are a Windows-only, THP-only feature that matters only
-        // on the **primary** mapper: the partition and the loader run against its
-        // VA, so that is the only place a large host backing yields a 2 MB SLAT
-        // entry. Non-primary (device/DMA) mappers, read-only mappings (e.g. ROM),
-        // non-THP ranges, and non-Windows hosts use plain read-write 4 KB pages.
+        // on the **primary** mapper whose partition can resolve guest memory
+        // faults: the partition and loader run against its VA, so that is the
+        // only place a large host backing yields a 2 MB SLAT entry, and the
+        // deferred-protect scheme it uses maps guest RAM read-only and relies on
+        // the partition delivering the first write fault back to the VMM to
+        // upgrade it. Non-primary (device/DMA) mappers, read-only mappings (e.g.
+        // ROM), non-THP ranges, partitions that can't resolve faults (e.g.
+        // aarch64 WHP), and non-Windows hosts use plain read-write 4 KB pages.
         let soft_lp = cfg!(windows)
             && params.policy.transparent_hugepages
             && params.writable
-            && self.inner.primary;
+            && self.inner.soft_large_pages;
 
         // Deferred protect (map read-only, populate lazily on the first write
         // fault) drives the *lazy* soft-LP path. When the range is prefetched it
@@ -791,7 +804,12 @@ impl VaMapper {
             waiters: Mutex::new(Some(Vec::new())),
             mappings: RwLock::new(RangeMap::new()),
             eager: AtomicBool::new(eager),
-            primary: role == MapperRole::Primary,
+            soft_large_pages: matches!(
+                role,
+                MapperRole::Primary {
+                    soft_large_pages: true
+                }
+            ),
             req_send,
         });
 

openvmm/membacking/src/memory_manager/mod.rs

@@ -570,10 +570,14 @@ impl GuestMemoryBuilder {
 
         // The primary mapper is created as part of `MappingManager::new`: it is
         // the loader's target and the partition's fault resolver.
-        let (mapping_manager, va_mapper) =
-            MappingManager::new(&spawner, max_addr, max_hugepage_size)
-                .await
-                .map_err(MemoryBuildError::VaMapper)?;
+        let (mapping_manager, va_mapper) = MappingManager::new(
+            &spawner,
+            max_addr,
+            max_hugepage_size,
+            self.supports_memory_fault_resolution,
+        )
+        .await
+        .map_err(MemoryBuildError::VaMapper)?;
 
         let region_manager = RegionManager::new(&spawner, mapping_manager.client().clone());
 

openvmm/openvmm_entry/src/ttrpc/mod.rs

@@ -62,6 +62,7 @@ use openvmm_defs::config::NumaDistance;
 use openvmm_defs::config::NumaNode;
 use openvmm_defs::config::NumaTopology;
 use openvmm_defs::config::PcieDeviceConfig;
+use openvmm_defs::config::PcieGenericInitiatorConfig;
 use openvmm_defs::config::PcieMmioRangeConfig;
 use openvmm_defs::config::PciePortConfig;
 use openvmm_defs::config::PcieRootComplexConfig;
@@ -785,22 +786,21 @@ impl VmService {
 
         // Build the PCIe topology (root complexes, switches, and the devices
         // attached behind their ports).
-        let (pcie_root_complexes, pcie_switches, pcie_devices) =
-            if let Some(pcie) = req_config.pcie.take() {
-                build_pcie_topology(pcie, &registry).await?
-            } else {
-                (Vec::new(), Vec::new(), Vec::new())
-            };
+        let pcie = if let Some(pcie) = req_config.pcie.take() {
+            build_pcie_topology(pcie, &registry).await?
+        } else {
+            BuiltPcieTopology::default()
+        };
 
         let mut config = Config {
             // TODO: devices, other stuff
             load_mode,
             ide_disks: vec![],
             floppy_disks: vec![],
-            pcie_root_complexes,
-            pcie_devices,
-            pcie_switches,
-            pcie_generic_initiators: vec![],
+            pcie_root_complexes: pcie.root_complexes,
+            pcie_devices: pcie.devices,
+            pcie_switches: pcie.switches,
+            pcie_generic_initiators: pcie.generic_initiators,
             vpci_devices: vec![],
             numa,
             chipset: chipset.chipset,
@@ -1484,16 +1484,21 @@ fn pcie_mmio_range_config(size: u64, base: Option<u64>) -> anyhow::Result<PcieMm
 /// a list of root complexes (each carrying its root ports), a list of switches
 /// (each referencing its parent port), and a list of devices (each referencing
 /// the port it sits behind).
+#[derive(Default)]
+struct BuiltPcieTopology {
+    root_complexes: Vec<PcieRootComplexConfig>,
+    switches: Vec<PcieSwitchConfig>,
+    devices: Vec<PcieDeviceConfig>,
+    generic_initiators: Vec<PcieGenericInitiatorConfig>,
+}
+
 async fn build_pcie_topology(
     topology: vmservice::PcieTopologyConfig,
     registry: &FdRegistry,
-) -> anyhow::Result<(
-    Vec<PcieRootComplexConfig>,
-    Vec<PcieSwitchConfig>,
-    Vec<PcieDeviceConfig>,
-)> {
+) -> anyhow::Result<BuiltPcieTopology> {
     let vmservice::PcieTopologyConfig {
         root_complexes: proto_root_complexes,
+        generic_initiators,
     } = topology;
     let mut root_complexes = Vec::new();
     let mut switches = Vec::new();
@@ -1522,14 +1527,17 @@ async fn build_pcie_topology(
                 hotplug,
                 attached,
                 devfn,
+                acs_capabilities_supported,
             } = root_port;
             ports.push(PciePortConfig {
                 name: port_name.clone(),
                 devfn: devfn
                     .map(|d| d.try_into().context("devfn out of range"))
                     .transpose()?,
                 hotplug,
-                acs_capabilities_supported: None,
+                acs_capabilities_supported: acs_capabilities_supported
+                    .map(|acs| acs.try_into().context("ACS capability mask out of range"))
+                    .transpose()?,
                 cxl: false,
             });
             if let Some(attached) = attached {
@@ -1562,7 +1570,20 @@ async fn build_pcie_topology(
         });
     }
 
-    Ok((root_complexes, switches, devices))
+    let generic_initiators = generic_initiators
+        .into_iter()
+        .map(|initiator| PcieGenericInitiatorConfig {
+            port_name: initiator.port_name,
+            node: initiator.node,
+        })
+        .collect();
+
+    Ok(BuiltPcieTopology {
+        root_complexes,
+        switches,
+        devices,
+        generic_initiators,
+    })
 }
 
 /// Walks a single proto `PcieAttachment` (the thing behind one port): either an
@@ -1591,14 +1612,17 @@ fn walk_pcie_attachment(
                     hotplug,
                     attached,
                     devfn,
+                    acs_capabilities_supported,
                 } = downstream;
                 ports.push(PciePortConfig {
                     name: downstream_name.clone(),
                     devfn: devfn
                         .map(|d| d.try_into().context("devfn out of range"))
                         .transpose()?,
                     hotplug,
-                    acs_capabilities_supported: None,
+                    acs_capabilities_supported: acs_capabilities_supported
+                        .map(|acs| acs.try_into().context("ACS capability mask out of range"))
+                        .transpose()?,
                     cxl: false,
                 });
                 if let Some(attached) = attached {

openvmm/openvmm_entry/src/vm_controller.rs

@@ -23,6 +23,7 @@ use std::path::Path;
 use std::path::PathBuf;
 use std::pin::pin;
 use std::sync::Arc;
+use std::time::Duration;
 use std::time::Instant;
 use vmm_core_defs::HaltReason;
 
@@ -303,6 +304,12 @@ impl VmController {
                             }
                         }
                         GuestPowerAction::Halt => {
+                            // The VM is kept stopped for inspection (e.g. a
+                            // fatal error or crash). Dump the full inspect
+                            // graph to the log so VM state is captured for
+                            // post-mortem diagnosis even when no client is
+                            // attached to inspect it.
+                            self.dump_inspect_graph().await;
                             event_send.send(VmControllerEvent::GuestHalt(format!("{reason:?}")));
                         }
                     }
@@ -442,6 +449,28 @@ impl VmController {
         deferred.inspect(obj);
     }
 
+    /// Collects the full host inspect graph and logs it. Invoked when the
+    /// guest halts for a reason that keeps the VM stopped (e.g. a fatal error
+    /// or crash), so that VM state (memory layout, device state, etc.) is
+    /// captured in the log for post-mortem diagnosis even when no client is
+    /// attached to inspect it.
+    async fn dump_inspect_graph(&mut self) {
+        let obj = inspect::adhoc_mut(|req| {
+            let mut resp = req.respond();
+            resp.field("mesh", &self.mesh)
+                .field("vm", &self.vm_worker)
+                .field("vnc", self.vnc_worker.as_ref())
+                .field("gdb", self.gdb_worker.as_ref());
+        });
+        let mut inspection = inspect::inspect("", obj);
+        let mut ctx = mesh::CancelContext::new().with_timeout(Duration::from_secs(5));
+        if ctx.until_cancelled(inspection.resolve()).await.is_err() {
+            tracing::warn!("timed out collecting inspect graph on halt");
+        }
+        let node = inspection.results();
+        tracing::error!("VM inspect graph at halt:\n{node}");
+    }
+
     async fn handle_save_snapshot(&self, dir: &Path) -> anyhow::Result<()> {
         let memory_file_path = self
             .memory_backing_file

openvmm/openvmm_ttrpc_vmservice/src/vmservice.proto

@@ -370,6 +370,16 @@ message NumaDistance {
 message PcieTopologyConfig {
     // The PCIe root complexes in the VM.
     repeated PcieRootComplex root_complexes = 1;
+    // Devices that act as initiators for device-only NUMA nodes.
+    repeated PcieGenericInitiator generic_initiators = 2;
+}
+
+// Associates the device behind a PCIe port with a NUMA proximity domain.
+message PcieGenericInitiator {
+    // Port directly upstream of the initiator device.
+    string port_name = 1;
+    // NUMA node for which the device is an initiator.
+    uint32 node = 2;
 }
 
 // A PCIe root complex.
@@ -411,6 +421,8 @@ message PciePort {
     // Device/function (`device << 3 | function`) to place this port at on its
     // bus. Absent => lowest available devfn.
     optional uint32 devfn = 4;
+    // ACS capability bits exposed by this port. Absent => default capabilities.
+    optional uint32 acs_capabilities_supported = 5;
 }
 
 // A PCIe switch: a set of downstream ports, each of which may have its own

vmm_core/state_unit/src/lib.rs

@@ -318,18 +318,22 @@ impl Inspect for Inner {
                 let mut resp = req.respond();
                 if !unit.dependencies.is_empty() {
                     resp.field_with("dependencies", || {
+                        // Removing a unit leaves stale ids in other units'
+                        // dependency lists (consumers tolerate this via
+                        // `get`), so skip any that no longer resolve.
                         unit.dependencies
                             .iter()
-                            .map(|id| self.units[id].name.as_ref())
+                            .filter_map(|id| self.units.get(id).map(|u| u.name.as_ref()))
                             .collect::<Vec<_>>()
                             .join(",")
                     });
                 }
                 if !unit.dependents.is_empty() {
                     resp.field_with("dependents", || {
+                        // See the note above: skip stale ids for removed units.
                         unit.dependents
                             .iter()
-                            .map(|id| self.units[id].name.as_ref())
+                            .filter_map(|id| self.units.get(id).map(|u| u.name.as_ref()))
                             .collect::<Vec<_>>()
                             .join(",")
                     });

vmm_core/virt_whp/src/lib.rs

@@ -967,9 +967,16 @@ impl ProtoPartition for WhpProtoPartition<'_> {
 
     fn supports_memory_fault_resolution(&self) -> bool {
         // WHP forwards guest memory-access faults back to the VMM, so the
-        // memory backing can resolve them on demand (soft large pages, lazy
-        // commit).
-        true
+        // memory backing can use the soft-large-page deferred-protect scheme:
+        // map guest RAM read-only and resolve the first write fault on demand.
+        //
+        // The aarch64 memory-intercept handler does not yet implement RAM
+        // fault population (unlike x86), so report that faults cannot be
+        // resolved there. The memory backing then maps RAM plain read-write
+        // instead of deferred-protect read-only, so first-write accesses never
+        // fault into the VMM \u2014 which the aarch64 handler would otherwise treat
+        // as fatal (e.g. a `DC ZVA` to a not-yet-populated page).
+        cfg!(guest_arch = "x86_64")
     }
 
     fn build(