openvmm_entry/ttrpc: expose PCIe NUMA and ACS config

Cargo.lock

@@ -10546,6 +10546,7 @@ dependencies = [
  "get_resources",
  "guid",
  "hyperv_ic_resources",
+ "initrd_cpio",
  "inspect",
  "jiff",
  "kmsg",

openvmm/openvmm_entry/src/ttrpc/mod.rs

@@ -62,6 +62,7 @@ use openvmm_defs::config::NumaDistance;
 use openvmm_defs::config::NumaNode;
 use openvmm_defs::config::NumaTopology;
 use openvmm_defs::config::PcieDeviceConfig;
+use openvmm_defs::config::PcieGenericInitiatorConfig;
 use openvmm_defs::config::PcieMmioRangeConfig;
 use openvmm_defs::config::PciePortConfig;
 use openvmm_defs::config::PcieRootComplexConfig;
@@ -785,22 +786,21 @@ impl VmService {
 
         // Build the PCIe topology (root complexes, switches, and the devices
         // attached behind their ports).
-        let (pcie_root_complexes, pcie_switches, pcie_devices) =
-            if let Some(pcie) = req_config.pcie.take() {
-                build_pcie_topology(pcie, &registry).await?
-            } else {
-                (Vec::new(), Vec::new(), Vec::new())
-            };
+        let pcie = if let Some(pcie) = req_config.pcie.take() {
+            build_pcie_topology(pcie, &registry).await?
+        } else {
+            BuiltPcieTopology::default()
+        };
 
         let mut config = Config {
             // TODO: devices, other stuff
             load_mode,
             ide_disks: vec![],
             floppy_disks: vec![],
-            pcie_root_complexes,
-            pcie_devices,
-            pcie_switches,
-            pcie_generic_initiators: vec![],
+            pcie_root_complexes: pcie.root_complexes,
+            pcie_devices: pcie.devices,
+            pcie_switches: pcie.switches,
+            pcie_generic_initiators: pcie.generic_initiators,
             vpci_devices: vec![],
             numa,
             chipset: chipset.chipset,
@@ -1484,16 +1484,21 @@ fn pcie_mmio_range_config(size: u64, base: Option<u64>) -> anyhow::Result<PcieMm
 /// a list of root complexes (each carrying its root ports), a list of switches
 /// (each referencing its parent port), and a list of devices (each referencing
 /// the port it sits behind).
+#[derive(Default)]
+struct BuiltPcieTopology {
+    root_complexes: Vec<PcieRootComplexConfig>,
+    switches: Vec<PcieSwitchConfig>,
+    devices: Vec<PcieDeviceConfig>,
+    generic_initiators: Vec<PcieGenericInitiatorConfig>,
+}
+
 async fn build_pcie_topology(
     topology: vmservice::PcieTopologyConfig,
     registry: &FdRegistry,
-) -> anyhow::Result<(
-    Vec<PcieRootComplexConfig>,
-    Vec<PcieSwitchConfig>,
-    Vec<PcieDeviceConfig>,
-)> {
+) -> anyhow::Result<BuiltPcieTopology> {
     let vmservice::PcieTopologyConfig {
         root_complexes: proto_root_complexes,
+        generic_initiators,
     } = topology;
     let mut root_complexes = Vec::new();
     let mut switches = Vec::new();
@@ -1522,14 +1527,17 @@ async fn build_pcie_topology(
                 hotplug,
                 attached,
                 devfn,
+                acs_capabilities_supported,
             } = root_port;
             ports.push(PciePortConfig {
                 name: port_name.clone(),
                 devfn: devfn
                     .map(|d| d.try_into().context("devfn out of range"))
                     .transpose()?,
                 hotplug,
-                acs_capabilities_supported: None,
+                acs_capabilities_supported: acs_capabilities_supported
+                    .map(|acs| acs.try_into().context("ACS capability mask out of range"))
+                    .transpose()?,
                 cxl: false,
             });
             if let Some(attached) = attached {
@@ -1562,7 +1570,20 @@ async fn build_pcie_topology(
         });
     }
 
-    Ok((root_complexes, switches, devices))
+    let generic_initiators = generic_initiators
+        .into_iter()
+        .map(|initiator| PcieGenericInitiatorConfig {
+            port_name: initiator.port_name,
+            node: initiator.node,
+        })
+        .collect();
+
+    Ok(BuiltPcieTopology {
+        root_complexes,
+        switches,
+        devices,
+        generic_initiators,
+    })
 }
 
 /// Walks a single proto `PcieAttachment` (the thing behind one port): either an
@@ -1591,14 +1612,17 @@ fn walk_pcie_attachment(
                     hotplug,
                     attached,
                     devfn,
+                    acs_capabilities_supported,
                 } = downstream;
                 ports.push(PciePortConfig {
                     name: downstream_name.clone(),
                     devfn: devfn
                         .map(|d| d.try_into().context("devfn out of range"))
                         .transpose()?,
                     hotplug,
-                    acs_capabilities_supported: None,
+                    acs_capabilities_supported: acs_capabilities_supported
+                        .map(|acs| acs.try_into().context("ACS capability mask out of range"))
+                        .transpose()?,
                     cxl: false,
                 });
                 if let Some(attached) = attached {

openvmm/openvmm_entry/src/vm_controller.rs

@@ -23,6 +23,7 @@ use std::path::Path;
 use std::path::PathBuf;
 use std::pin::pin;
 use std::sync::Arc;
+use std::time::Duration;
 use std::time::Instant;
 use vmm_core_defs::HaltReason;
 
@@ -303,6 +304,12 @@ impl VmController {
                             }
                         }
                         GuestPowerAction::Halt => {
+                            // The VM is kept stopped for inspection (e.g. a
+                            // fatal error or crash). Dump the full inspect
+                            // graph to the log so VM state is captured for
+                            // post-mortem diagnosis even when no client is
+                            // attached to inspect it.
+                            self.dump_inspect_graph().await;
                             event_send.send(VmControllerEvent::GuestHalt(format!("{reason:?}")));
                         }
                     }
@@ -442,6 +449,28 @@ impl VmController {
         deferred.inspect(obj);
     }
 
+    /// Collects the full host inspect graph and logs it. Invoked when the
+    /// guest halts for a reason that keeps the VM stopped (e.g. a fatal error
+    /// or crash), so that VM state (memory layout, device state, etc.) is
+    /// captured in the log for post-mortem diagnosis even when no client is
+    /// attached to inspect it.
+    async fn dump_inspect_graph(&mut self) {
+        let obj = inspect::adhoc_mut(|req| {
+            let mut resp = req.respond();
+            resp.field("mesh", &self.mesh)
+                .field("vm", &self.vm_worker)
+                .field("vnc", self.vnc_worker.as_ref())
+                .field("gdb", self.gdb_worker.as_ref());
+        });
+        let mut inspection = inspect::inspect("", obj);
+        let mut ctx = mesh::CancelContext::new().with_timeout(Duration::from_secs(5));
+        if ctx.until_cancelled(inspection.resolve()).await.is_err() {
+            tracing::warn!("timed out collecting inspect graph on halt");
+        }
+        let node = inspection.results();
+        tracing::error!("VM inspect graph at halt:\n{node}");
+    }
+
     async fn handle_save_snapshot(&self, dir: &Path) -> anyhow::Result<()> {
         let memory_file_path = self
             .memory_backing_file

openvmm/openvmm_ttrpc_vmservice/src/vmservice.proto

@@ -370,6 +370,16 @@ message NumaDistance {
 message PcieTopologyConfig {
     // The PCIe root complexes in the VM.
     repeated PcieRootComplex root_complexes = 1;
+    // Devices that act as initiators for device-only NUMA nodes.
+    repeated PcieGenericInitiator generic_initiators = 2;
+}
+
+// Associates the device behind a PCIe port with a NUMA proximity domain.
+message PcieGenericInitiator {
+    // Port directly upstream of the initiator device.
+    string port_name = 1;
+    // NUMA node for which the device is an initiator.
+    uint32 node = 2;
 }
 
 // A PCIe root complex.
@@ -411,6 +421,8 @@ message PciePort {
     // Device/function (`device << 3 | function`) to place this port at on its
     // bus. Absent => lowest available devfn.
     optional uint32 devfn = 4;
+    // ACS capability bits exposed by this port. Absent => default capabilities.
+    optional uint32 acs_capabilities_supported = 5;
 }
 
 // A PCIe switch: a set of downstream ports, each of which may have its own

vmm_core/virt_whp/src/vp.rs

@@ -1848,7 +1848,11 @@ mod aarch64 {
                     let iss = IssDataAbort::from(syndrome.iss());
                     if !iss.isv() {
                         return Err(dev.fatal_error(
-                            anyhow::anyhow!("can't handle data abort without isv: {iss:?}").into(),
+                            anyhow::anyhow!(
+                                "can't handle data abort without isv: {iss:?} ({})",
+                                self.memory_access_details(message)
+                            )
+                            .into(),
                         ));
                     }
                     let len = 1 << iss.sas();
@@ -1886,13 +1890,88 @@ mod aarch64 {
                 }
                 ec => {
                     return Err(dev.fatal_error(
-                        anyhow::anyhow!("unknown memory access exception: {ec:?}").into(),
+                        anyhow::anyhow!(
+                            "unknown memory access exception: {ec:?} ({})",
+                            self.memory_access_details(message)
+                        )
+                        .into(),
                     ));
                 }
             }
             Ok(())
         }
 
+        /// Formats a memory intercept for diagnostics, including the faulting
+        /// instruction.
+        fn memory_access_details(&self, message: &hvdef::HvArm64MemoryInterceptMessage) -> String {
+            let instruction = match self.faulting_instruction(message) {
+                Some(instruction) => format!("{instruction:#010x}"),
+                None => "unavailable".to_string(),
+            };
+            format!(
+                "gpa={:#x} pc={:#x} access={:?} syndrome={:#x} instruction={}",
+                message.guest_physical_address,
+                message.header.pc,
+                message.header.intercept_access_type,
+                message.syndrome,
+                instruction,
+            )
+        }
+
+        /// Returns the faulting instruction word. Prefers the bytes provided in
+        /// the intercept message, but when the hypervisor doesn't provide them
+        /// (as happens for unmapped-GPA intercepts, which arrive with an empty
+        /// syndrome and no instruction bytes) falls back to reading guest
+        /// memory by translating the faulting PC.
+        fn faulting_instruction(
+            &self,
+            message: &hvdef::HvArm64MemoryInterceptMessage,
+        ) -> Option<u32> {
+            if message.instruction_byte_count >= 4 {
+                return Some(u32::from_le_bytes(message.instruction_bytes));
+            }
+            match self.read_guest_instruction(message.header.pc) {
+                Ok(instruction) => Some(instruction),
+                Err(err) => {
+                    tracing::warn!(
+                        error = err.as_ref() as &dyn std::error::Error,
+                        pc = message.header.pc,
+                        "failed to read faulting instruction from guest memory"
+                    );
+                    None
+                }
+            }
+        }
+
+        /// Reads the 4-byte instruction word at the given guest virtual address
+        /// by walking the guest's page tables and reading guest memory.
+        fn read_guest_instruction(&self, pc: u64) -> anyhow::Result<u32> {
+            use virt_support_aarch64emu::translate;
+
+            let processor = self.current_whp();
+            let registers = translate::TranslationRegisters {
+                cpsr: processor.get_register(whp::Register64::Cpsr)?.into(),
+                sctlr: processor.get_register(whp::Register64::Sctlr)?.into(),
+                tcr: processor.get_register(whp::Register64::Tcr)?.into(),
+                ttbr0: processor.get_register(whp::Register64::Ttbr0)?,
+                ttbr1: processor.get_register(whp::Register64::Ttbr1)?,
+                syndrome: 0,
+                encryption_mode: translate::EncryptionMode::None,
+            };
+            let flags = translate::TranslateFlags {
+                validate_execute: false,
+                validate_read: false,
+                validate_write: false,
+                privilege_check: translate::TranslatePrivilegeCheck::None,
+                set_page_table_bits: false,
+            };
+            let gpa =
+                translate::translate_gva_to_gpa(&self.vp.partition.gm, pc, &registers, flags)?;
+            let mut bytes = [0; 4];
+            self.vp.partition.gm.read_at(gpa, &mut bytes)?;
+            Ok(u32::from_le_bytes(bytes))
+        }
+
         /// Handle a reset from the hypervisor-handled PSCI call.
         fn handle_reset(&mut self, info: &hvdef::HvArm64ResetInterceptMessage) -> VpHaltReason {
             match info.reset_type {

vmm_tests/vmm_tests/Cargo.toml

@@ -28,6 +28,7 @@ tmk_tests.workspace = true
 petri_artifacts_common.workspace = true
 petri.workspace = true
 inspect.workspace = true
+initrd_cpio.workspace = true
 
 get_resources.workspace = true
 openvmm_defs.workspace = true