From 74f0763386850786edf5dd074bab22f003b05c3e Mon Sep 17 00:00:00 2001 From: John Starks Date: Thu, 23 Jul 2026 10:36:22 +0000 Subject: [PATCH 1/7] openvmm_entry/ttrpc: expose PCIe NUMA and ACS config GB200-style device topologies need the management interface to describe PCIe devices as ACPI Generic Initiators and control the ACS capabilities advertised by individual ports. Extend the VM service schema and topology conversion so these settings reach the canonical OpenVMM configuration. Use pipette in the ttrpc integration test to verify the resulting SRAT Generic Initiator entry and ACS capability directly from inside the guest. --- Cargo.lock | 1 + openvmm/openvmm_entry/src/ttrpc/mod.rs | 60 ++++-- .../src/vmservice.proto | 12 ++ vmm_tests/vmm_tests/Cargo.toml | 1 + vmm_tests/vmm_tests/tests/tests/ttrpc.rs | 182 +++++++++++++++++- 5 files changed, 231 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73fa47397b..0358101273 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10546,6 +10546,7 @@ dependencies = [ "get_resources", "guid", "hyperv_ic_resources", + "initrd_cpio", "inspect", "jiff", "kmsg", diff --git a/openvmm/openvmm_entry/src/ttrpc/mod.rs b/openvmm/openvmm_entry/src/ttrpc/mod.rs index 1e299f0317..7e1e253512 100644 --- a/openvmm/openvmm_entry/src/ttrpc/mod.rs +++ b/openvmm/openvmm_entry/src/ttrpc/mod.rs @@ -62,6 +62,7 @@ use openvmm_defs::config::NumaDistance; use openvmm_defs::config::NumaNode; use openvmm_defs::config::NumaTopology; use openvmm_defs::config::PcieDeviceConfig; +use openvmm_defs::config::PcieGenericInitiatorConfig; use openvmm_defs::config::PcieMmioRangeConfig; use openvmm_defs::config::PciePortConfig; use openvmm_defs::config::PcieRootComplexConfig; @@ -785,22 +786,21 @@ impl VmService { // Build the PCIe topology (root complexes, switches, and the devices // attached behind their ports). - let (pcie_root_complexes, pcie_switches, pcie_devices) = - if let Some(pcie) = req_config.pcie.take() { - build_pcie_topology(pcie, ®istry).await? - } else { - (Vec::new(), Vec::new(), Vec::new()) - }; + let pcie = if let Some(pcie) = req_config.pcie.take() { + build_pcie_topology(pcie, ®istry).await? + } else { + BuiltPcieTopology::default() + }; let mut config = Config { // TODO: devices, other stuff load_mode, ide_disks: vec![], floppy_disks: vec![], - pcie_root_complexes, - pcie_devices, - pcie_switches, - pcie_generic_initiators: vec![], + pcie_root_complexes: pcie.root_complexes, + pcie_devices: pcie.devices, + pcie_switches: pcie.switches, + pcie_generic_initiators: pcie.generic_initiators, vpci_devices: vec![], numa, chipset: chipset.chipset, @@ -1484,16 +1484,21 @@ fn pcie_mmio_range_config(size: u64, base: Option) -> anyhow::Result, + switches: Vec, + devices: Vec, + generic_initiators: Vec, +} + async fn build_pcie_topology( topology: vmservice::PcieTopologyConfig, registry: &FdRegistry, -) -> anyhow::Result<( - Vec, - Vec, - Vec, -)> { +) -> anyhow::Result { let vmservice::PcieTopologyConfig { root_complexes: proto_root_complexes, + generic_initiators, } = topology; let mut root_complexes = Vec::new(); let mut switches = Vec::new(); @@ -1522,6 +1527,7 @@ async fn build_pcie_topology( hotplug, attached, devfn, + acs_capabilities_supported, } = root_port; ports.push(PciePortConfig { name: port_name.clone(), @@ -1529,7 +1535,9 @@ async fn build_pcie_topology( .map(|d| d.try_into().context("devfn out of range")) .transpose()?, hotplug, - acs_capabilities_supported: None, + acs_capabilities_supported: acs_capabilities_supported + .map(|acs| acs.try_into().context("ACS capability mask out of range")) + .transpose()?, cxl: false, }); if let Some(attached) = attached { @@ -1562,7 +1570,20 @@ async fn build_pcie_topology( }); } - Ok((root_complexes, switches, devices)) + let generic_initiators = generic_initiators + .into_iter() + .map(|initiator| PcieGenericInitiatorConfig { + port_name: initiator.port_name, + node: initiator.node, + }) + .collect(); + + Ok(BuiltPcieTopology { + root_complexes, + switches, + devices, + generic_initiators, + }) } /// Walks a single proto `PcieAttachment` (the thing behind one port): either an @@ -1591,6 +1612,7 @@ fn walk_pcie_attachment( hotplug, attached, devfn, + acs_capabilities_supported, } = downstream; ports.push(PciePortConfig { name: downstream_name.clone(), @@ -1598,7 +1620,9 @@ fn walk_pcie_attachment( .map(|d| d.try_into().context("devfn out of range")) .transpose()?, hotplug, - acs_capabilities_supported: None, + acs_capabilities_supported: acs_capabilities_supported + .map(|acs| acs.try_into().context("ACS capability mask out of range")) + .transpose()?, cxl: false, }); if let Some(attached) = attached { diff --git a/openvmm/openvmm_ttrpc_vmservice/src/vmservice.proto b/openvmm/openvmm_ttrpc_vmservice/src/vmservice.proto index 32f55df524..ae5786bc59 100644 --- a/openvmm/openvmm_ttrpc_vmservice/src/vmservice.proto +++ b/openvmm/openvmm_ttrpc_vmservice/src/vmservice.proto @@ -370,6 +370,16 @@ message NumaDistance { message PcieTopologyConfig { // The PCIe root complexes in the VM. repeated PcieRootComplex root_complexes = 1; + // Devices that act as initiators for device-only NUMA nodes. + repeated PcieGenericInitiator generic_initiators = 2; +} + +// Associates the device behind a PCIe port with a NUMA proximity domain. +message PcieGenericInitiator { + // Port directly upstream of the initiator device. + string port_name = 1; + // NUMA node for which the device is an initiator. + uint32 node = 2; } // A PCIe root complex. @@ -411,6 +421,8 @@ message PciePort { // Device/function (`device << 3 | function`) to place this port at on its // bus. Absent => lowest available devfn. optional uint32 devfn = 4; + // ACS capability bits exposed by this port. Absent => default capabilities. + optional uint32 acs_capabilities_supported = 5; } // A PCIe switch: a set of downstream ports, each of which may have its own diff --git a/vmm_tests/vmm_tests/Cargo.toml b/vmm_tests/vmm_tests/Cargo.toml index e5b7783fe8..11dfdc203a 100644 --- a/vmm_tests/vmm_tests/Cargo.toml +++ b/vmm_tests/vmm_tests/Cargo.toml @@ -28,6 +28,7 @@ tmk_tests.workspace = true petri_artifacts_common.workspace = true petri.workspace = true inspect.workspace = true +initrd_cpio.workspace = true get_resources.workspace = true openvmm_defs.workspace = true diff --git a/vmm_tests/vmm_tests/tests/tests/ttrpc.rs b/vmm_tests/vmm_tests/tests/tests/ttrpc.rs index 9d972469af..bdb160c23b 100644 --- a/vmm_tests/vmm_tests/tests/tests/ttrpc.rs +++ b/vmm_tests/vmm_tests/tests/tests/ttrpc.rs @@ -22,7 +22,9 @@ use pal_async::socket::PolledSocket; use pal_async::task::Spawn; use pal_async::task::Task; use petri::ResolvedArtifact; +use petri::pipette::cmd; use petri_artifacts_vmm_test::artifacts; +use std::io::Write; use std::path::Path; use std::process::Stdio; use std::time::Duration; @@ -33,12 +35,20 @@ petri::test!(test_ttrpc_interface, |resolver| { let openvmm = resolver.require(artifacts::OPENVMM_NATIVE); let kernel = resolver.require(artifacts::loadable::LINUX_DIRECT_TEST_KERNEL_NATIVE); let initrd = resolver.require(artifacts::loadable::LINUX_DIRECT_TEST_INITRD_NATIVE); - Some([openvmm.erase(), kernel.erase(), initrd.erase()]) + let pipette = match petri_artifacts_common::tags::MachineArch::host() { + petri_artifacts_common::tags::MachineArch::X86_64 => resolver + .require(petri_artifacts_common::artifacts::PIPETTE_LINUX_X64) + .erase(), + petri_artifacts_common::tags::MachineArch::Aarch64 => resolver + .require(petri_artifacts_common::artifacts::PIPETTE_LINUX_AARCH64) + .erase(), + }; + Some([openvmm.erase(), kernel.erase(), initrd.erase(), pipette]) }); fn test_ttrpc_interface( params: petri::PetriTestParams<'_>, - [openvmm, kernel_path, initrd_path]: [ResolvedArtifact; 3], + [openvmm, kernel_path, initrd_path, pipette_path]: [ResolvedArtifact; 4], ) -> anyhow::Result<()> { // All temporary files for this test live under a single temp directory // that is cleaned up automatically when it is dropped at the end of the @@ -47,6 +57,13 @@ fn test_ttrpc_interface( let socket_path = tempdir.path().join("ttrpc.sock"); let pidfile_path = tempdir.path().join("openvmm.pid"); + let initrd = std::fs::read(initrd_path.get()).context("failed to read test initrd")?; + let pipette = std::fs::read(pipette_path.get()).context("failed to read pipette")?; + let pipette_initrd = initrd_cpio::inject_into_initrd(&initrd, "pipette", &pipette, 0o100755) + .context("failed to inject pipette into test initrd")?; + let mut pipette_initrd_file = tempfile::NamedTempFile::new()?; + pipette_initrd_file.write_all(&pipette_initrd)?; + // The serial console device differs by architecture: x86 exposes a 16550 // UART as `ttyS0`, while aarch64 exposes a PL011 UART as `ttyAMA0`. let console = match petri_artifacts_common::tags::MachineArch::host() { @@ -128,6 +145,17 @@ fn test_ttrpc_interface( let console_path = tempdir.path().join(format!("console-{i}.sock")); let virtiofs_root = tempdir.path().join(format!("virtiofs-{i}")); std::fs::create_dir_all(&virtiofs_root)?; + let hvsocket_path = tempdir.path().join(format!("hvsocket-{i}")); + let pipette_listener = if i == 0 { + let path = format!( + "{}_{}", + hvsocket_path.to_string_lossy(), + pipette_client::PIPETTE_PORT + ); + Some(UnixListener::bind(path)?) + } else { + None + }; let consomme_nic_id = Guid::new_random().to_string(); @@ -166,6 +194,7 @@ fn test_ttrpc_interface( read_only: false, }), ))), + acs_capabilities_supported: Some(1), devfn: None, }, vmservice::PciePort { @@ -173,6 +202,7 @@ fn test_ttrpc_interface( hotplug: false, attached: None, devfn: None, + acs_capabilities_supported: None, }, ], }; @@ -267,6 +297,10 @@ fn test_ttrpc_interface( }), Some(vmservice::PcieTopologyConfig { root_complexes: vec![root_complex], + generic_initiators: vec![vmservice::PcieGenericInitiator { + port_name: "sw0-dp0".to_string(), + node: 1, + }], }), ) } else { @@ -284,7 +318,20 @@ fn test_ttrpc_interface( ) }; - let guest_command = if i == 1 { "sleep 30" } else { "poweroff -f" }; + let (boot_initrd_path, kernel_cmdline) = if i == 0 { + ( + pipette_initrd_file.path(), + format!( + "console={console} rdinit=/pipette panic=-1 initcall_blacklist=virtio_vsock_init" + ), + ) + } else { + let guest_command = if i == 1 { "sleep 30" } else { "poweroff -f" }; + ( + initrd_path.get(), + format!("console={console} rdinit=/bin/busybox panic=-1 -- {guest_command}"), + ) + }; client .call() @@ -299,10 +346,8 @@ fn test_ttrpc_interface( boot_config: Some(vmservice::vm_config::BootConfig::DirectBoot( vmservice::DirectBoot { kernel_path: kernel_path.get().to_string_lossy().to_string(), - initrd_path: initrd_path.get().to_string_lossy().to_string(), - kernel_cmdline: format!( - "console={console} rdinit=/bin/busybox panic=-1 -- {guest_command}" - ), + initrd_path: boot_initrd_path.to_string_lossy().to_string(), + kernel_cmdline, }, )), serial_config: Some(vmservice::SerialConfig { @@ -334,6 +379,9 @@ fn test_ttrpc_interface( }], ..Default::default() }), + hvsocket_config: (i == 0).then(|| vmservice::HvSocketConfig { + path: hvsocket_path.to_string_lossy().to_string(), + }), ..Default::default() }), log_id: String::new(), @@ -484,6 +532,20 @@ fn test_ttrpc_interface( "after ResumeVm, expected RUNNING" ); + if let Some(listener) = pipette_listener { + let mut listener = PolledSocket::new(&driver, listener)?; + let (conn, _) = listener.accept().await?; + let conn = PolledSocket::new(&driver, conn)?; + let agent = pipette_client::PipetteClient::new( + &driver, + conn, + params.logger.output_dir(), + ) + .await?; + validate_pcie_config(&agent).await?; + agent.power_off().await?; + } + waiter.await.unwrap(); let props = query_props().await.unwrap(); @@ -611,6 +673,7 @@ fn pcie_root_port( hotplug, attached, devfn: None, + acs_capabilities_supported: None, } } @@ -723,3 +786,108 @@ fn file_disk(path: &Path) -> vmservice::DiskBackend { })), } } + +async fn validate_pcie_config(agent: &pipette_client::PipetteClient) -> anyhow::Result<()> { + let sh = agent.unix_shell(); + let devices = cmd!(sh, "ls /sys/bus/pci/devices").read().await?; + let mut device = None; + for bdf in devices.split_whitespace() { + let class = sh + .read_file(format!("/sys/bus/pci/devices/{bdf}/class")) + .await?; + if class.trim() == "0x010000" { + device = Some(bdf); + break; + } + } + let device = device.context("virtio-blk PCI device not found")?; + + let mut bdf = device.split([':', '.']); + let segment = u16::from_str_radix(bdf.next().context("missing PCI segment")?, 16)?; + let bus = u8::from_str_radix(bdf.next().context("missing PCI bus")?, 16)?; + let device_number = u8::from_str_radix(bdf.next().context("missing PCI device")?, 16)?; + let function = u8::from_str_radix(bdf.next().context("missing PCI function")?, 16)?; + anyhow::ensure!(bdf.next().is_none(), "invalid PCI BDF {device}"); + + let srat = agent.read_file("/sys/firmware/acpi/tables/SRAT").await?; + anyhow::ensure!( + srat.get(..4) == Some(b"SRAT"), + "guest SRAT has an invalid signature" + ); + let mut offset = 48; + let mut found_generic_initiator = false; + while offset + 2 <= srat.len() { + let entry_len = srat[offset + 1] as usize; + anyhow::ensure!( + entry_len >= 2 && offset + entry_len <= srat.len(), + "guest SRAT contains an invalid entry at offset {offset:#x}" + ); + if srat[offset] == 5 && entry_len == 32 { + let proximity_domain = + u32::from_le_bytes(srat[offset + 4..offset + 8].try_into().unwrap()); + let entry_segment = + u16::from_le_bytes(srat[offset + 8..offset + 10].try_into().unwrap()); + let entry_bus = srat[offset + 10]; + let entry_devfn = srat[offset + 11]; + let flags = u32::from_le_bytes(srat[offset + 24..offset + 28].try_into().unwrap()); + if srat[offset + 3] == 1 + && proximity_domain == 1 + && entry_segment == segment + && entry_bus == bus + && entry_devfn == (device_number << 3) | function + && flags & 1 != 0 + { + found_generic_initiator = true; + break; + } + } + offset += entry_len; + } + anyhow::ensure!( + found_generic_initiator, + "guest SRAT has no enabled Generic Initiator entry for {device} on NUMA node 1" + ); + + let device_path = cmd!(sh, "readlink -f /sys/bus/pci/devices/{device}") + .read() + .await?; + let port_path = Path::new(device_path.trim()) + .parent() + .and_then(Path::to_str) + .context("PCI device has no parent port")?; + let config = sh.read_file_raw(format!("{port_path}/config")).await?; + let mut capability_offset = 0x100; + let acs_offset = loop { + let header = u32::from_le_bytes( + config + .get(capability_offset..capability_offset + 4) + .context("invalid PCIe extended capability offset")? + .try_into() + .unwrap(), + ); + let capability_id = header as u16; + if capability_id == 0x000d { + break capability_offset; + } + + let next_offset = (header >> 20) as usize; + anyhow::ensure!( + next_offset > capability_offset && next_offset % 4 == 0, + "parent port has no ACS capability" + ); + capability_offset = next_offset; + }; + let acs_capabilities = u16::from_le_bytes( + config + .get(acs_offset + 4..acs_offset + 6) + .context("parent port has no ACS capability register")? + .try_into() + .unwrap(), + ); + anyhow::ensure!( + acs_capabilities == 1, + "expected ACS capability mask 0x0001, got {acs_capabilities:#06x}" + ); + + Ok(()) +} From eb0a0067f3e0d92fcadb2d479392fc96031531cf Mon Sep 17 00:00:00 2001 From: John Starks Date: Thu, 23 Jul 2026 10:53:22 +0000 Subject: [PATCH 2/7] fix --- vmm_tests/vmm_tests/tests/tests/ttrpc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vmm_tests/vmm_tests/tests/tests/ttrpc.rs b/vmm_tests/vmm_tests/tests/tests/ttrpc.rs index bdb160c23b..c069b38050 100644 --- a/vmm_tests/vmm_tests/tests/tests/ttrpc.rs +++ b/vmm_tests/vmm_tests/tests/tests/ttrpc.rs @@ -872,7 +872,7 @@ async fn validate_pcie_config(agent: &pipette_client::PipetteClient) -> anyhow:: let next_offset = (header >> 20) as usize; anyhow::ensure!( - next_offset > capability_offset && next_offset % 4 == 0, + next_offset > capability_offset && next_offset.is_multiple_of(4), "parent port has no ACS capability" ); capability_offset = next_offset; From 09f26e6156ce4be0518da693c34f2cfcd3fba586 Mon Sep 17 00:00:00 2001 From: John Starks Date: Fri, 24 Jul 2026 02:55:46 -0700 Subject: [PATCH 3/7] diag --- vmm_core/virt_whp/src/vp.rs | 83 ++++++++++++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 2 deletions(-) diff --git a/vmm_core/virt_whp/src/vp.rs b/vmm_core/virt_whp/src/vp.rs index d713c5ee86..deca2289fa 100644 --- a/vmm_core/virt_whp/src/vp.rs +++ b/vmm_core/virt_whp/src/vp.rs @@ -1848,7 +1848,11 @@ mod aarch64 { let iss = IssDataAbort::from(syndrome.iss()); if !iss.isv() { return Err(dev.fatal_error( - anyhow::anyhow!("can't handle data abort without isv: {iss:?}").into(), + anyhow::anyhow!( + "can't handle data abort without isv: {iss:?} ({})", + self.memory_access_details(message) + ) + .into(), )); } let len = 1 << iss.sas(); @@ -1886,13 +1890,88 @@ mod aarch64 { } ec => { return Err(dev.fatal_error( - anyhow::anyhow!("unknown memory access exception: {ec:?}").into(), + anyhow::anyhow!( + "unknown memory access exception: {ec:?} ({})", + self.memory_access_details(message) + ) + .into(), )); } } Ok(()) } + /// Formats a memory intercept for diagnostics, including the faulting + /// instruction. + fn memory_access_details(&self, message: &hvdef::HvArm64MemoryInterceptMessage) -> String { + let instruction = match self.faulting_instruction(message) { + Some(instruction) => format!("{instruction:#010x}"), + None => "unavailable".to_string(), + }; + format!( + "gpa={:#x} pc={:#x} access={:?} syndrome={:#x} instruction={}", + message.guest_physical_address, + message.header.pc, + message.header.intercept_access_type, + message.syndrome, + instruction, + ) + } + + /// Returns the faulting instruction word. Prefers the bytes provided in + /// the intercept message, but when the hypervisor doesn't provide them + /// (as happens for unmapped-GPA intercepts, which arrive with an empty + /// syndrome and no instruction bytes) falls back to reading guest + /// memory by translating the faulting PC. + fn faulting_instruction( + &self, + message: &hvdef::HvArm64MemoryInterceptMessage, + ) -> Option { + if message.instruction_byte_count >= 4 { + return Some(u32::from_le_bytes(message.instruction_bytes)); + } + match self.read_guest_instruction(message.header.pc) { + Ok(instruction) => Some(instruction), + Err(err) => { + tracing::warn!( + error = err.as_ref() as &dyn std::error::Error, + pc = message.header.pc, + "failed to read faulting instruction from guest memory" + ); + None + } + } + } + + /// Reads the 4-byte instruction word at the given guest virtual address + /// by walking the guest's page tables and reading guest memory. + fn read_guest_instruction(&self, pc: u64) -> anyhow::Result { + use virt_support_aarch64emu::translate; + + let processor = self.current_whp(); + let registers = translate::TranslationRegisters { + cpsr: processor.get_register(whp::Register64::Cpsr)?.into(), + sctlr: processor.get_register(whp::Register64::Sctlr)?.into(), + tcr: processor.get_register(whp::Register64::Tcr)?.into(), + ttbr0: processor.get_register(whp::Register64::Ttbr0)?, + ttbr1: processor.get_register(whp::Register64::Ttbr1)?, + syndrome: 0, + encryption_mode: translate::EncryptionMode::None, + }; + let flags = translate::TranslateFlags { + validate_execute: false, + validate_read: false, + validate_write: false, + privilege_check: translate::TranslatePrivilegeCheck::None, + set_page_table_bits: false, + }; + let gpa = + translate::translate_gva_to_gpa(&self.vp.partition.gm, pc, ®isters, flags)?; + let mut bytes = [0; 4]; + self.vp.partition.gm.read_at(gpa, &mut bytes)?; + Ok(u32::from_le_bytes(bytes)) + } + /// Handle a reset from the hypervisor-handled PSCI call. fn handle_reset(&mut self, info: &hvdef::HvArm64ResetInterceptMessage) -> VpHaltReason { match info.reset_type { From 84d6bcd4e04b50b5856f0bc9487438ee57e681a8 Mon Sep 17 00:00:00 2001 From: John Starks Date: Fri, 24 Jul 2026 04:36:06 -0700 Subject: [PATCH 4/7] diag --- openvmm/openvmm_entry/src/vm_controller.rs | 29 ++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/openvmm/openvmm_entry/src/vm_controller.rs b/openvmm/openvmm_entry/src/vm_controller.rs index 564354775d..a40f35da2f 100644 --- a/openvmm/openvmm_entry/src/vm_controller.rs +++ b/openvmm/openvmm_entry/src/vm_controller.rs @@ -23,6 +23,7 @@ use std::path::Path; use std::path::PathBuf; use std::pin::pin; use std::sync::Arc; +use std::time::Duration; use std::time::Instant; use vmm_core_defs::HaltReason; @@ -303,6 +304,12 @@ impl VmController { } } GuestPowerAction::Halt => { + // The VM is kept stopped for inspection (e.g. a + // fatal error or crash). Dump the full inspect + // graph to the log so VM state is captured for + // post-mortem diagnosis even when no client is + // attached to inspect it. + self.dump_inspect_graph().await; event_send.send(VmControllerEvent::GuestHalt(format!("{reason:?}"))); } } @@ -442,6 +449,28 @@ impl VmController { deferred.inspect(obj); } + /// Collects the full host inspect graph and logs it. Invoked when the + /// guest halts for a reason that keeps the VM stopped (e.g. a fatal error + /// or crash), so that VM state (memory layout, device state, etc.) is + /// captured in the log for post-mortem diagnosis even when no client is + /// attached to inspect it. + async fn dump_inspect_graph(&mut self) { + let obj = inspect::adhoc_mut(|req| { + let mut resp = req.respond(); + resp.field("mesh", &self.mesh) + .field("vm", &self.vm_worker) + .field("vnc", self.vnc_worker.as_ref()) + .field("gdb", self.gdb_worker.as_ref()); + }); + let mut inspection = inspect::inspect("", obj); + let mut ctx = mesh::CancelContext::new().with_timeout(Duration::from_secs(5)); + if ctx.until_cancelled(inspection.resolve()).await.is_err() { + tracing::warn!("timed out collecting inspect graph on halt"); + } + let node = inspection.results(); + tracing::error!("VM inspect graph at halt:\n{node}"); + } + async fn handle_save_snapshot(&self, dir: &Path) -> anyhow::Result<()> { let memory_file_path = self .memory_backing_file From 7835352a49e2de90f12dd0e731764b1e3ac56333 Mon Sep 17 00:00:00 2001 From: John Starks Date: Fri, 24 Jul 2026 04:44:05 -0700 Subject: [PATCH 5/7] fix state unit inspect --- vmm_core/state_unit/src/lib.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index fa98780985..1cfd09a304 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -318,18 +318,22 @@ impl Inspect for Inner { let mut resp = req.respond(); if !unit.dependencies.is_empty() { resp.field_with("dependencies", || { + // Removing a unit leaves stale ids in other units' + // dependency lists (consumers tolerate this via + // `get`), so skip any that no longer resolve. unit.dependencies .iter() - .map(|id| self.units[id].name.as_ref()) + .filter_map(|id| self.units.get(id).map(|u| u.name.as_ref())) .collect::>() .join(",") }); } if !unit.dependents.is_empty() { resp.field_with("dependents", || { + // See the note above: skip stale ids for removed units. unit.dependents .iter() - .map(|id| self.units[id].name.as_ref()) + .filter_map(|id| self.units.get(id).map(|u| u.name.as_ref())) .collect::>() .join(",") }); From 5d8ed12a7b210b48376203737699f978b73b305b Mon Sep 17 00:00:00 2001 From: John Starks Date: Fri, 24 Jul 2026 04:57:34 -0700 Subject: [PATCH 6/7] ah ha --- vmm_core/virt_whp/src/lib.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/vmm_core/virt_whp/src/lib.rs b/vmm_core/virt_whp/src/lib.rs index 3019f17aee..17bb2173dc 100644 --- a/vmm_core/virt_whp/src/lib.rs +++ b/vmm_core/virt_whp/src/lib.rs @@ -969,7 +969,14 @@ impl ProtoPartition for WhpProtoPartition<'_> { // WHP forwards guest memory-access faults back to the VMM, so the // memory backing can resolve them on demand (soft large pages, lazy // commit). - true + // + // The aarch64 memory-intercept handler does not yet implement RAM + // fault population (unlike x86), so report that faults cannot be + // resolved there. This keeps the memory backing eager and avoids + // intercepts on lazily-mapped RAM, which the aarch64 handler would + // otherwise treat as fatal (e.g. a `DC ZVA` to a not-yet-populated + // page). + cfg!(guest_arch = "x86_64") } fn build( From 1006c09add77d70d29935590484c3eda5086cee0 Mon Sep 17 00:00:00 2001 From: John Starks Date: Fri, 24 Jul 2026 05:46:09 -0700 Subject: [PATCH 7/7] whoops --- .../membacking/src/mapping_manager/manager.rs | 16 +++++-- .../src/mapping_manager/va_mapper.rs | 44 +++++++++++++------ openvmm/membacking/src/memory_manager/mod.rs | 12 +++-- vmm_core/virt_whp/src/lib.rs | 12 ++--- 4 files changed, 58 insertions(+), 26 deletions(-) diff --git a/openvmm/membacking/src/mapping_manager/manager.rs b/openvmm/membacking/src/mapping_manager/manager.rs index 67ec81211f..6ead9ccf18 100644 --- a/openvmm/membacking/src/mapping_manager/manager.rs +++ b/openvmm/membacking/src/mapping_manager/manager.rs @@ -56,10 +56,16 @@ impl MappingManager { /// that is enforced dynamically by the manager task as mappings and mappers /// come and go (private RAM may be added later, e.g. via hotplug) rather /// than captured here at construction time. + /// + /// `soft_large_pages` reports whether the partition can resolve guest memory + /// faults; it gates the Windows soft-large-page (deferred-protect) path on + /// the primary mapper, since that scheme relies on the partition delivering + /// the first write fault back to the VMM. pub async fn new( spawn: impl Spawn, max_addr: u64, minimum_va_alignment: Option, + soft_large_pages: bool, ) -> Result<(Self, Arc), VaMapperError> { let this = Self::new_bare(spawn, max_addr, minimum_va_alignment); // Create the primary mapper as part of construction. Being first, it is @@ -67,7 +73,7 @@ impl MappingManager { // this process. let primary = this .client() - .get_or_create_mapper(true, MapperRole::Primary) + .get_or_create_mapper(true, MapperRole::Primary { soft_large_pages }) .await?; Ok((this, primary)) } @@ -1481,7 +1487,9 @@ mod tests { None, None, true, // eager - MapperRole::Primary, + MapperRole::Primary { + soft_large_pages: true, + }, ); let (mapper, _) = futures::join!(mapper_future, async { let msg = req_recv.recv().await.unwrap(); @@ -1519,7 +1527,9 @@ mod tests { None, None, true, // eager - MapperRole::Primary, + MapperRole::Primary { + soft_large_pages: true, + }, ); let (mapper, mapper_req_send) = futures::join!(mapper_future, async { let msg = req_recv.recv().await.unwrap(); diff --git a/openvmm/membacking/src/mapping_manager/va_mapper.rs b/openvmm/membacking/src/mapping_manager/va_mapper.rs index 46d5ca5f36..be197160ae 100644 --- a/openvmm/membacking/src/mapping_manager/va_mapper.rs +++ b/openvmm/membacking/src/mapping_manager/va_mapper.rs @@ -117,8 +117,14 @@ enum FaultError { /// handled correctly. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) enum MapperRole { - /// The single loader/partition mapper; eligible for soft large pages. - Primary, + /// The single loader/partition mapper. Eligible for soft large pages only + /// when `soft_large_pages` is set, i.e. the partition can resolve guest + /// memory faults: the deferred-protect scheme maps guest RAM read-only and + /// relies on the partition delivering the first write fault back to the VMM + /// to upgrade it. Backends that can't resolve faults (e.g. the aarch64 WHP + /// memory-intercept handler) must clear this and use plain read-write 4 KB + /// pages instead. + Primary { soft_large_pages: bool }, /// Any other mapper; plain read-write 4 KB pages. Secondary, } @@ -277,11 +283,14 @@ struct MapperInner { /// succeeds because the mapping is already established. The flag /// is eventually updated after `SetEager` is processed. eager: AtomicBool, - /// Whether this is the **primary** mapper — the one the partition and the - /// loader run against. Soft large pages (Windows) are only worthwhile here, - /// since this is the mapping whose host backing drives the guest's SLAT. - /// Secondary mappers use plain read-write 4 KB pages. See [`MapperRole`]. - primary: bool, + /// Whether this mapper is eligible for the Windows soft-large-page + /// (deferred-protect) path. True only for the **primary** mapper — the one + /// the partition and loader run against, whose host backing drives the + /// guest's SLAT — and only when the partition can resolve guest memory + /// faults. Secondary mappers, and partitions without fault resolution (e.g. + /// the aarch64 WHP backend), use plain read-write 4 KB pages. See + /// [`MapperRole`]. + soft_large_pages: bool, req_send: mesh::Sender, } @@ -373,14 +382,18 @@ impl MapperTask { /// Establishes a mapping in the VA space, dispatching on how it is backed. fn map(&self, params: MappingParams) -> Result<(), MappingError> { // Soft large pages are a Windows-only, THP-only feature that matters only - // on the **primary** mapper: the partition and the loader run against its - // VA, so that is the only place a large host backing yields a 2 MB SLAT - // entry. Non-primary (device/DMA) mappers, read-only mappings (e.g. ROM), - // non-THP ranges, and non-Windows hosts use plain read-write 4 KB pages. + // on the **primary** mapper whose partition can resolve guest memory + // faults: the partition and loader run against its VA, so that is the + // only place a large host backing yields a 2 MB SLAT entry, and the + // deferred-protect scheme it uses maps guest RAM read-only and relies on + // the partition delivering the first write fault back to the VMM to + // upgrade it. Non-primary (device/DMA) mappers, read-only mappings (e.g. + // ROM), non-THP ranges, partitions that can't resolve faults (e.g. + // aarch64 WHP), and non-Windows hosts use plain read-write 4 KB pages. let soft_lp = cfg!(windows) && params.policy.transparent_hugepages && params.writable - && self.inner.primary; + && self.inner.soft_large_pages; // Deferred protect (map read-only, populate lazily on the first write // fault) drives the *lazy* soft-LP path. When the range is prefetched it @@ -791,7 +804,12 @@ impl VaMapper { waiters: Mutex::new(Some(Vec::new())), mappings: RwLock::new(RangeMap::new()), eager: AtomicBool::new(eager), - primary: role == MapperRole::Primary, + soft_large_pages: matches!( + role, + MapperRole::Primary { + soft_large_pages: true + } + ), req_send, }); diff --git a/openvmm/membacking/src/memory_manager/mod.rs b/openvmm/membacking/src/memory_manager/mod.rs index f2ef1f430b..f30256e841 100644 --- a/openvmm/membacking/src/memory_manager/mod.rs +++ b/openvmm/membacking/src/memory_manager/mod.rs @@ -570,10 +570,14 @@ impl GuestMemoryBuilder { // The primary mapper is created as part of `MappingManager::new`: it is // the loader's target and the partition's fault resolver. - let (mapping_manager, va_mapper) = - MappingManager::new(&spawner, max_addr, max_hugepage_size) - .await - .map_err(MemoryBuildError::VaMapper)?; + let (mapping_manager, va_mapper) = MappingManager::new( + &spawner, + max_addr, + max_hugepage_size, + self.supports_memory_fault_resolution, + ) + .await + .map_err(MemoryBuildError::VaMapper)?; let region_manager = RegionManager::new(&spawner, mapping_manager.client().clone()); diff --git a/vmm_core/virt_whp/src/lib.rs b/vmm_core/virt_whp/src/lib.rs index 17bb2173dc..daa07fd05a 100644 --- a/vmm_core/virt_whp/src/lib.rs +++ b/vmm_core/virt_whp/src/lib.rs @@ -967,15 +967,15 @@ impl ProtoPartition for WhpProtoPartition<'_> { fn supports_memory_fault_resolution(&self) -> bool { // WHP forwards guest memory-access faults back to the VMM, so the - // memory backing can resolve them on demand (soft large pages, lazy - // commit). + // memory backing can use the soft-large-page deferred-protect scheme: + // map guest RAM read-only and resolve the first write fault on demand. // // The aarch64 memory-intercept handler does not yet implement RAM // fault population (unlike x86), so report that faults cannot be - // resolved there. This keeps the memory backing eager and avoids - // intercepts on lazily-mapped RAM, which the aarch64 handler would - // otherwise treat as fatal (e.g. a `DC ZVA` to a not-yet-populated - // page). + // resolved there. The memory backing then maps RAM plain read-write + // instead of deferred-protect read-only, so first-write accesses never + // fault into the VMM \u2014 which the aarch64 handler would otherwise treat + // as fatal (e.g. a `DC ZVA` to a not-yet-populated page). cfg!(guest_arch = "x86_64") }