From a006099ebce9247198366bf32ae2caf60bdfc6ba Mon Sep 17 00:00:00 2001 From: John Starks Date: Wed, 22 Jul 2026 14:04:15 +0000 Subject: [PATCH 1/5] vfio_assigned_device: allow fixed BAR addresses The nvgrace-gpu VFIO variant driver synthesizes a coherent-memory BAR whose physical address is not reported through the PCI sysfs resource table. OpenVMM therefore cannot discover that BAR's host address, even though it must be placed at the same guest address for peer-to-peer DMA on platforms where transactions bypass IOMMU translation. Allow each VFIO BAR to use either the host-reported address or an explicit fixed address. Carry this policy through the CLI, management service, and VFIO resource boundary, and validate fixed addresses against the discovered BAR layout before programming them. This replaces the opaque `barN=pt` CLI syntax with `barN=host`; the old spelling is no longer accepted. The management schema gains equivalent structured support for host-derived and fixed BAR addresses. --- Guide/src/reference/openvmm/management/cli.md | 2 +- Guide/src/user_guide/openvmm/vfio.md | 21 ++- openvmm/openvmm_entry/src/cli_args.rs | 81 +++++---- openvmm/openvmm_entry/src/lib.rs | 4 +- openvmm/openvmm_entry/src/ttrpc/mod.rs | 79 ++++++++- .../src/vmservice.proto | 15 ++ .../pci/vfio_assigned_device/src/lib.rs | 156 ++++++++++++++---- .../pci/vfio_assigned_device/src/resolver.rs | 8 +- .../vfio_assigned_device_resources/src/lib.rs | 22 ++- 9 files changed, 304 insertions(+), 84 deletions(-) diff --git a/Guide/src/reference/openvmm/management/cli.md b/Guide/src/reference/openvmm/management/cli.md index dde6a32581..666ee6caee 100644 --- a/Guide/src/reference/openvmm/management/cli.md +++ b/Guide/src/reference/openvmm/management/cli.md @@ -407,7 +407,7 @@ For `--virtio-rng` and `--virtio-console`, use their separate PCIe port flags: --iommu id=iommu0 --vfio host=0000:01:00.0,port=rp0,iommu=iommu0 # Pin BAR0 to its physical address for P2P DMA: ---vfio host=0000:01:00.0,port=rp0,bar0=pt +--vfio host=0000:01:00.0,port=rp0,bar0=host ``` ### SMMU (aarch64 only) diff --git a/Guide/src/user_guide/openvmm/vfio.md b/Guide/src/user_guide/openvmm/vfio.md index abb5395ec5..26e5250d54 100644 --- a/Guide/src/user_guide/openvmm/vfio.md +++ b/Guide/src/user_guide/openvmm/vfio.md @@ -118,9 +118,12 @@ The `--vfio` value is a comma-separated list of `key=value` pairs: - `host=` (required) — the PCI BDF of the VFIO device on the host (e.g., `0000:01:00.0`) - `port=` (required) — the name of the PCIe root port to attach the device to (must match a `--pcie-root-port` name) - `iommu=` (optional) — reference to an `--iommu` context; see [Using iommufd (cdev path)](#using-iommufd-cdev-path) below -- `bar0=pt` through `bar5=pt` (optional) — pin the specified BAR to its +- `bar0=host` through `bar5=host` (optional) — pin the specified BAR to its physical host address (GPA = HPA); see [Peer-to-peer DMA](#peer-to-peer-dma) below +- `bar0=0x` through `bar5=0x` (optional) — pin the specified BAR + to an explicit host physical address; use this for BARs synthesized by a + VFIO variant driver whose address is not reported through sysfs ```admonish tip You can assign multiple devices by adding more root ports and `--vfio` flags: @@ -182,7 +185,7 @@ P2P DMA. This means the guest BAR addresses must be identity-mapped to the host BAR addresses (GPA = HPA), or P2P DMA will target the wrong location. -To enable this, pin the relevant BARs with `bar=pt` on each `--vfio` +To enable this, pin the relevant BARs with `bar=host` on each `--vfio` device and set `preserve_bars` on the root complex so the PCI resource allocator keeps pinned BARs at their physical addresses: @@ -192,11 +195,21 @@ sudo openvmm \ rc0,preserve_bars,low_mmio_base=0xc0000000,high_mmio_base=0x100000000 \ --pcie-root-port rc0:rp0 \ --pcie-root-port rc0:rp1 \ - --vfio host=0000:01:00.0,port=rp0,bar0=pt \ - --vfio host=0000:02:00.0,port=rp1,bar0=pt \ + --vfio host=0000:01:00.0,port=rp0,bar0=host \ + --vfio host=0000:02:00.0,port=rp1,bar0=host \ ... ``` +For a BAR synthesized by a VFIO variant driver, the host physical address may +not appear in the device's sysfs resource table. In that case, provide the +address directly with `bar=0x`. For example, the `nvgrace-gpu` driver +exposes CPU-coherent GPU memory as a synthetic 64-bit BAR4 whose physical base +comes from the `nvidia,gpu-mem-base-pa` ACPI `_DSD` property: + +```bash + --vfio host=0008:06:00.0,port=rp0,bar0=host,bar2=host,bar4=0x110000000000 +``` + The `low_mmio_base=` and `high_mmio_base=` options pin the MMIO apertures to fixed addresses so the allocator can place both pinned and dynamic BARs correctly. diff --git a/openvmm/openvmm_entry/src/cli_args.rs b/openvmm/openvmm_entry/src/cli_args.rs index 38ac1baae3..f34faaca06 100644 --- a/openvmm/openvmm_entry/src/cli_args.rs +++ b/openvmm/openvmm_entry/src/cli_args.rs @@ -3058,7 +3058,7 @@ pub struct PcieRemoteCli { /// CLI configuration for a VFIO-assigned PCI device. /// -/// Syntax: `host=,port=[,iommu=][,bar0=pt..bar5=pt]` +/// Syntax: `host=,port=[,iommu=][,barN=host|barN=0x]` #[cfg(target_os = "linux")] #[derive(Clone, Debug)] pub struct VfioDeviceCli { @@ -3069,36 +3069,43 @@ pub struct VfioDeviceCli { /// Optional iommufd context ID. When set, uses VFIO cdev + iommufd /// instead of the legacy group/container path. pub iommu: Option, - /// Per-BAR passthrough flags. When `bar_pt[i]` is true, the virtual - /// BAR is pre-programmed with the physical BAR address (GPA = HPA). - pub bar_pt: [bool; 6], + /// Per-BAR pre-programming configuration. + pub bar_addresses: [vfio_assigned_device_resources::BarAddressConfig; 6], } -/// Marker for a BAR passthrough value; only `pt` is accepted. +/// Per-BAR address configuration parsed from the CLI. #[cfg(target_os = "linux")] -struct Pt; +struct BarAddressCli(vfio_assigned_device_resources::BarAddressConfig); #[cfg(target_os = "linux")] -impl FromStr for Pt { +impl FromStr for BarAddressCli { type Err = anyhow::Error; fn from_str(s: &str) -> anyhow::Result { - anyhow::ensure!(s == "pt", "expected 'pt'"); - Ok(Pt) + let config = if s == "host" { + vfio_assigned_device_resources::BarAddressConfig::HostAssigned + } else if let Some(value) = s.strip_prefix("0x") { + let address = u64::from_str_radix(value, 16).context("invalid BAR address")?; + anyhow::ensure!(address != 0, "BAR address must be nonzero"); + vfio_assigned_device_resources::BarAddressConfig::Fixed(address) + } else { + anyhow::bail!("expected 'host' or a hexadecimal address starting with '0x'"); + }; + Ok(Self(config)) } } -/// Per-BAR passthrough flags (`bar0=pt` .. `bar5=pt`), flattened into +/// Per-BAR address configuration, flattened into /// [`VfioArgs`]. #[cfg(target_os = "linux")] #[derive(vmm_cli::KeyValueArgs)] struct BarFlags { - bar0: Option, - bar1: Option, - bar2: Option, - bar3: Option, - bar4: Option, - bar5: Option, + bar0: Option, + bar1: Option, + bar2: Option, + bar3: Option, + bar4: Option, + bar5: Option, } /// Raw `--vfio` options, resolved and validated into a [`VfioDeviceCli`]. @@ -3125,20 +3132,20 @@ impl FromStr for VfioDeviceCli { } let bars = args.bars; - let bar_pt = [ - bars.bar0.is_some(), - bars.bar1.is_some(), - bars.bar2.is_some(), - bars.bar3.is_some(), - bars.bar4.is_some(), - bars.bar5.is_some(), + let bar_addresses = [ + bars.bar0.map(|bar| bar.0).unwrap_or_default(), + bars.bar1.map(|bar| bar.0).unwrap_or_default(), + bars.bar2.map(|bar| bar.0).unwrap_or_default(), + bars.bar3.map(|bar| bar.0).unwrap_or_default(), + bars.bar4.map(|bar| bar.0).unwrap_or_default(), + bars.bar5.map(|bar| bar.0).unwrap_or_default(), ]; Ok(VfioDeviceCli { port_name: args.port, pci_id: args.host, iommu: args.iommu, - bar_pt, + bar_addresses, }) } } @@ -4805,6 +4812,8 @@ mod tests { #[cfg(target_os = "linux")] #[test] fn test_vfio_device_cli_parse() { + use vfio_assigned_device_resources::BarAddressConfig; + // Required keys only. let v = VfioDeviceCli::from_str("host=0000:01:00.0,port=rp0").unwrap(); assert_eq!(v.pci_id, "0000:01:00.0"); @@ -4817,13 +4826,14 @@ mod tests { assert_eq!(v.port_name, "rp1"); assert_eq!(v.iommu.as_deref(), Some("iommu0")); - // BAR passthrough flags set the corresponding indices. - let v = VfioDeviceCli::from_str("host=0000:01:00.0,port=rp0,bar0=pt,bar2=pt").unwrap(); - assert_eq!(v.bar_pt, [true, false, true, false, false, false]); - - // A BAR flag only accepts `pt`, and requires a value. - assert!(VfioDeviceCli::from_str("host=0000:01:00.0,port=rp0,bar0=x").is_err()); - assert!(VfioDeviceCli::from_str("host=0000:01:00.0,port=rp0,bar0").is_err()); + let v = VfioDeviceCli::from_str( + "host=0000:03:00.0,port=rp2,bar0=host,bar2=0x80000000,bar4=0x110000000000", + ) + .unwrap(); + assert_eq!(v.bar_addresses[0], BarAddressConfig::HostAssigned); + assert_eq!(v.bar_addresses[1], BarAddressConfig::GuestAssigned); + assert_eq!(v.bar_addresses[2], BarAddressConfig::Fixed(0x80000000)); + assert_eq!(v.bar_addresses[4], BarAddressConfig::Fixed(0x110000000000)); } #[cfg(target_os = "linux")] @@ -4853,6 +4863,15 @@ mod tests { // Path-traversal characters in the host BDF are rejected. assert!(VfioDeviceCli::from_str("host=../../etc/passwd,port=rp0").is_err()); assert!(VfioDeviceCli::from_str("host=foo/bar,port=rp0").is_err()); + + // Invalid and duplicate BAR configurations are rejected. + assert!(VfioDeviceCli::from_str("host=0000:01:00.0,port=rp0,bar0=0").is_err()); + assert!(VfioDeviceCli::from_str("host=0000:01:00.0,port=rp0,bar0=0x0").is_err()); + assert!(VfioDeviceCli::from_str("host=0000:01:00.0,port=rp0,bar0=0xnope").is_err()); + assert!(VfioDeviceCli::from_str("host=0000:01:00.0,port=rp0,bar0=pt").is_err()); + assert!( + VfioDeviceCli::from_str("host=0000:01:00.0,port=rp0,bar0=0x1000,bar0=host").is_err() + ); } #[cfg(target_os = "linux")] diff --git a/openvmm/openvmm_entry/src/lib.rs b/openvmm/openvmm_entry/src/lib.rs index 340eaab6ba..a40689242e 100644 --- a/openvmm/openvmm_entry/src/lib.rs +++ b/openvmm/openvmm_entry/src/lib.rs @@ -1058,7 +1058,7 @@ async fn vm_config_from_command_line( cdev, iommufd, iommu_id: iommu_id.clone(), - bar_pt: cli_cfg.bar_pt, + bar_addresses: cli_cfg.bar_addresses, } .into_resource(), }) @@ -1085,7 +1085,7 @@ async fn vm_config_from_command_line( resource: vfio_assigned_device_resources::VfioDeviceHandle { pci_id: cli_cfg.pci_id.clone(), group, - bar_pt: cli_cfg.bar_pt, + bar_addresses: cli_cfg.bar_addresses, } .into_resource(), }) diff --git a/openvmm/openvmm_entry/src/ttrpc/mod.rs b/openvmm/openvmm_entry/src/ttrpc/mod.rs index 1e299f0317..4bd2ce7ed4 100644 --- a/openvmm/openvmm_entry/src/ttrpc/mod.rs +++ b/openvmm/openvmm_entry/src/ttrpc/mod.rs @@ -1643,7 +1643,11 @@ async fn build_pcie_device( /// opened. The device must already be bound to `vfio-pci` on the host. #[cfg(target_os = "linux")] fn build_vfio_device(vfio: vmservice::VfioDevice) -> anyhow::Result> { - let vmservice::VfioDevice { host_pci_address } = vfio; + let vmservice::VfioDevice { + host_pci_address, + bar_addresses, + } = vfio; + let bar_addresses = parse_vfio_bar_addresses(bar_addresses)?; // The address is joined into a sysfs path below; reject path separators so // it cannot escape `/sys/bus/pci/devices` (an absolute path or `..` would // otherwise redirect the join). @@ -1669,11 +1673,40 @@ fn build_vfio_device(vfio: vmservice::VfioDevice) -> anyhow::Result, +) -> anyhow::Result<[vfio_assigned_device_resources::BarAddressConfig; 6]> { + use vfio_assigned_device_resources::BarAddressConfig; + use vmservice::vfio_bar_address::Source; + + let mut bar_addresses = [BarAddressConfig::GuestAssigned; 6]; + for entry in entries { + let index = + usize::try_from(entry.bar_index).context("VFIO BAR index does not fit usize")?; + let config = bar_addresses + .get_mut(index) + .with_context(|| format!("VFIO BAR index {} is out of range", entry.bar_index))?; + anyhow::ensure!( + *config == BarAddressConfig::GuestAssigned, + "duplicate VFIO BAR index {}", + entry.bar_index + ); + *config = match entry.source.context("missing VFIO BAR address source")? { + Source::Host(()) => BarAddressConfig::HostAssigned, + Source::Fixed(address) => { + anyhow::ensure!(address != 0, "VFIO BAR fixed address must be nonzero"); + BarAddressConfig::Fixed(address) + } + }; + } + Ok(bar_addresses) +} + #[cfg(not(target_os = "linux"))] fn build_vfio_device( _vfio: vmservice::VfioDevice, @@ -1915,3 +1948,45 @@ fn build_vhost_user_device( ) -> anyhow::Result> { anyhow::bail!("vhost-user is only supported on unix hosts") } + +#[cfg(test)] +mod tests { + use super::*; + use vfio_assigned_device_resources::BarAddressConfig; + use vmservice::vfio_bar_address::Source; + + fn vfio_bar_address(bar_index: u32, source: Option) -> vmservice::VfioBarAddress { + vmservice::VfioBarAddress { bar_index, source } + } + + #[test] + fn parse_vfio_bar_address_config() { + let bars = parse_vfio_bar_addresses(vec![ + vfio_bar_address(0, Some(Source::Host(()))), + vfio_bar_address(4, Some(Source::Fixed(0x11_0000_0000))), + ]) + .unwrap(); + + assert_eq!(bars[0], BarAddressConfig::HostAssigned); + assert_eq!(bars[1], BarAddressConfig::GuestAssigned); + assert_eq!(bars[4], BarAddressConfig::Fixed(0x11_0000_0000)); + } + + #[test] + fn reject_invalid_vfio_bar_address_config() { + assert!( + parse_vfio_bar_addresses(vec![vfio_bar_address(6, Some(Source::Host(())))]).is_err() + ); + assert!(parse_vfio_bar_addresses(vec![vfio_bar_address(0, None)]).is_err()); + assert!( + parse_vfio_bar_addresses(vec![vfio_bar_address(0, Some(Source::Fixed(0)))]).is_err() + ); + assert!( + parse_vfio_bar_addresses(vec![ + vfio_bar_address(0, Some(Source::Host(()))), + vfio_bar_address(0, Some(Source::Fixed(0x1000))), + ]) + .is_err() + ); + } +} diff --git a/openvmm/openvmm_ttrpc_vmservice/src/vmservice.proto b/openvmm/openvmm_ttrpc_vmservice/src/vmservice.proto index 32f55df524..a5f1f4a1dd 100644 --- a/openvmm/openvmm_ttrpc_vmservice/src/vmservice.proto +++ b/openvmm/openvmm_ttrpc_vmservice/src/vmservice.proto @@ -294,6 +294,21 @@ message NvmeNamespace { message VfioDevice { // Host PCI address (BDF), e.g. "0000:01:00.0". string host_pci_address = 1; + // BARs whose initial addresses are pinned. Omitted BARs are assigned by + // the guest normally. + repeated VfioBarAddress bar_addresses = 2; +} + +// Initial address configuration for one VFIO device BAR. +message VfioBarAddress { + // BAR index, from 0 through 5. + uint32 bar_index = 1; + oneof source { + // Use the physical BAR address reported by the host. + google.protobuf.Empty host = 2; + // Use an explicit host physical address. + uint64 fixed = 3; + } } // diff --git a/vm/devices/pci/vfio_assigned_device/src/lib.rs b/vm/devices/pci/vfio_assigned_device/src/lib.rs index 797cfedc11..46300268f1 100644 --- a/vm/devices/pci/vfio_assigned_device/src/lib.rs +++ b/vm/devices/pci/vfio_assigned_device/src/lib.rs @@ -41,6 +41,7 @@ use std::collections::BTreeMap; use std::ops::Range; use std::os::fd::AsFd; use std::os::unix::fs::FileExt; +use vfio_assigned_device_resources::BarAddressConfig; use vmcore::device_state::ChangeDeviceState; use vmcore::save_restore::RestoreError; use vmcore::save_restore::SaveError; @@ -282,7 +283,7 @@ impl VfioAssignedPciDevice { register_mmio: &mut (dyn chipset_device::mmio::RegisterMmioIntercept + Send), msi_target: &MsiTarget, memory_mapper: &dyn MemoryMapper, - bar_pt: [bool; 6], + bar_addresses: [BarAddressConfig; 6], ) -> anyhow::Result { let vfio_device = binding .group() @@ -296,7 +297,7 @@ impl VfioAssignedPciDevice { register_mmio, msi_target, memory_mapper, - bar_pt, + bar_addresses, ) .await } @@ -308,7 +309,7 @@ impl VfioAssignedPciDevice { register_mmio: &mut (dyn chipset_device::mmio::RegisterMmioIntercept + Send), msi_target: &MsiTarget, memory_mapper: &dyn MemoryMapper, - bar_pt: [bool; 6], + bar_addresses: [BarAddressConfig; 6], ) -> anyhow::Result { let (device, binding) = cdev_binding.into_parts(); Self::from_device( @@ -318,7 +319,7 @@ impl VfioAssignedPciDevice { register_mmio, msi_target, memory_mapper, - bar_pt, + bar_addresses, ) .await } @@ -330,7 +331,7 @@ impl VfioAssignedPciDevice { register_mmio: &mut (dyn chipset_device::mmio::RegisterMmioIntercept + Send), msi_target: &MsiTarget, memory_mapper: &dyn MemoryMapper, - bar_pt: [bool; 6], + bar_addresses: [BarAddressConfig; 6], ) -> anyhow::Result { let config_info = vfio_device .region_info(vfio_bindings::bindings::vfio::VFIO_PCI_CONFIG_REGION_INDEX) @@ -545,7 +546,7 @@ impl VfioAssignedPciDevice { // Build initial BAR values. Start from bar_flags (encoding bits // only — guaranteed clean). For passthrough BARs, overlay the // physical addresses from sysfs. - let bars = apply_bar_passthrough(&pci_id, &bar_flags, &bar_masks, &bar_pt)?; + let bars = apply_bar_addresses(&pci_id, &bar_flags, &bar_masks, &bar_addresses)?; let bar_reset_defaults = bars; Ok(Self { @@ -779,24 +780,26 @@ fn page_size() -> u64 { vfio_sys::host_page_size() } -/// Apply BAR passthrough: validate the `bar_pt` flags against the discovered -/// BAR layout and overlay physical addresses from sysfs. +/// Apply the configured initial addresses to the discovered BAR layout. /// /// Rejects requests for unimplemented BARs (zero mask) and for the upper half /// of a 64-bit BAR pair (the lower BAR implicitly covers both halves). -fn apply_bar_passthrough( +fn apply_bar_addresses( pci_id: &str, bar_flags: &[u32; 6], bar_masks: &[u32; 6], - bar_pt: &[bool; 6], + bar_addresses: &[BarAddressConfig; 6], ) -> anyhow::Result<[u32; 6]> { - if !bar_pt.iter().any(|&pt| pt) { + if bar_addresses + .iter() + .all(|bar| *bar == BarAddressConfig::GuestAssigned) + { return Ok(*bar_flags); } // Validate before reading sysfs. for i in 0..6 { - if !bar_pt[i] { + if bar_addresses[i] == BarAddressConfig::GuestAssigned { continue; } if bar_masks[i] == 0 { @@ -811,31 +814,46 @@ fn apply_bar_passthrough( } } - // VFIO config space returns cleared BARs after device reset, so sysfs - // is the only reliable source of physical addresses. - let phys = read_physical_bar_addresses(pci_id)?; + // VFIO config space returns cleared BARs after device reset, so sysfs is + // the reliable source for host-assigned addresses. Avoid requiring sysfs + // when every configured BAR has an explicit address. + let physical_addresses = bar_addresses + .contains(&BarAddressConfig::HostAssigned) + .then(|| read_physical_bar_addresses(pci_id)) + .transpose()?; let mut bars = *bar_flags; for i in 0..6 { - if bar_pt[i] { - let addr = phys[i]; - if addr == 0 { - anyhow::bail!("BAR {i} passthrough requested but sysfs address is 0"); - } - let is_64bit = cfg_space::BarEncodingBits::from(bar_flags[i]).type_64_bit(); - if !is_64bit && addr > u32::MAX as u64 { - anyhow::bail!("BAR {i} is 32-bit but sysfs address {addr:#x} exceeds 4 GB"); - } - bars[i] = (addr as u32 & !0xf) | bar_flags[i]; - if is_64bit && i + 1 < 6 { - bars[i + 1] = (addr >> 32) as u32; - } - tracing::info!( - pci_id, - bar_index = i, - addr = format_args!("{:#x}", addr), - "passthrough BAR" - ); + let address = match bar_addresses[i] { + BarAddressConfig::GuestAssigned => continue, + BarAddressConfig::HostAssigned => physical_addresses.as_ref().unwrap()[i], + BarAddressConfig::Fixed(address) => address, + }; + if address == 0 { + anyhow::bail!("BAR {i} address is 0"); + } + let is_64bit = cfg_space::BarEncodingBits::from(bar_flags[i]).type_64_bit(); + if !is_64bit && address > u32::MAX as u64 { + anyhow::bail!("BAR {i} is 32-bit but address {address:#x} exceeds 4 GB"); } + let address_mask = if is_64bit { + (bar_masks[i + 1] as u64) << 32 | (bar_masks[i] as u64 & !0xf) + } else { + (bar_masks[i] as i32 as i64 as u64) & !0xf + }; + let size = (!address_mask).wrapping_add(1); + if address & (size - 1) != 0 { + anyhow::bail!("BAR {i} address {address:#x} is not aligned to its size {size:#x}"); + } + bars[i] = (address as u32 & !0xf) | bar_flags[i]; + if is_64bit { + bars[i + 1] = (address >> 32) as u32; + } + tracing::info!( + pci_id, + bar_index = i, + address = format_args!("{address:#x}"), + "pre-programmed BAR address" + ); } Ok(bars) } @@ -1575,6 +1593,76 @@ mod tests { use pci_core::msi::MsiTarget; use test_with_tracing::test; + #[test] + fn apply_explicit_32_bit_bar_address() { + let bar_flags = [0; 6]; + let mut bar_masks = [0; 6]; + bar_masks[0] = 0xffff_f000; + let mut bar_addresses = [BarAddressConfig::GuestAssigned; 6]; + bar_addresses[0] = BarAddressConfig::Fixed(0x8000_0000); + + let bars = apply_bar_addresses("not-a-real-device", &bar_flags, &bar_masks, &bar_addresses) + .unwrap(); + + assert_eq!(bars[0], 0x8000_0000); + } + + #[test] + fn apply_explicit_64_bit_bar_address() { + let mut bar_flags = [0; 6]; + bar_flags[2] = 0x4; + let mut bar_masks = [0; 6]; + bar_masks[2] = 0xffff_f004; + bar_masks[3] = 0xffff_ffff; + let mut bar_addresses = [BarAddressConfig::GuestAssigned; 6]; + bar_addresses[2] = BarAddressConfig::Fixed(0x11_0000_0000); + + let bars = apply_bar_addresses("not-a-real-device", &bar_flags, &bar_masks, &bar_addresses) + .unwrap(); + + assert_eq!(bars[2], 0x4); + assert_eq!(bars[3], 0x11); + } + + #[test] + fn reject_invalid_explicit_bar_addresses() { + let bar_flags = [0; 6]; + let mut bar_masks = [0; 6]; + bar_masks[0] = 0xffff_f000; + + for address in [0, 0x8000_0001, 0x1_0000_0000] { + let mut bar_addresses = [BarAddressConfig::GuestAssigned; 6]; + bar_addresses[0] = BarAddressConfig::Fixed(address); + assert!( + apply_bar_addresses("not-a-real-device", &bar_flags, &bar_masks, &bar_addresses) + .is_err() + ); + } + } + + #[test] + fn reject_invalid_bar_indices() { + let mut bar_flags = [0; 6]; + bar_flags[0] = 0x4; + let mut bar_masks = [0; 6]; + bar_masks[0] = 0xffff_f004; + bar_masks[1] = 0xffff_ffff; + + let mut bar_addresses = [BarAddressConfig::GuestAssigned; 6]; + bar_addresses[1] = BarAddressConfig::Fixed(0x8000_0000); + assert!( + apply_bar_addresses("not-a-real-device", &bar_flags, &bar_masks, &bar_addresses) + .is_err() + ); + + bar_addresses = [BarAddressConfig::GuestAssigned; 6]; + bar_addresses[2] = BarAddressConfig::Fixed(0x8000_0000); + assert!( + apply_bar_addresses("not-a-real-device", &bar_flags, &bar_masks, &bar_addresses) + .is_err() + ); + } + /// In-memory config space backing store for unit tests. struct MockConfigSpace { data: Vec, diff --git a/vm/devices/pci/vfio_assigned_device/src/resolver.rs b/vm/devices/pci/vfio_assigned_device/src/resolver.rs index fdd2603532..8926464133 100644 --- a/vm/devices/pci/vfio_assigned_device/src/resolver.rs +++ b/vm/devices/pci/vfio_assigned_device/src/resolver.rs @@ -64,7 +64,7 @@ impl AsyncResolveResource for VfioDeviceR let VfioDeviceHandle { pci_id, group, - bar_pt, + bar_addresses, } = resource; // The legacy VFIO group/type1 path can only do identity DMA, so only a @@ -100,7 +100,7 @@ impl AsyncResolveResource for VfioDeviceR input.register_mmio, input.dma_target.msi_target(), memory_mapper, - bar_pt, + bar_addresses, ) .await?; @@ -156,7 +156,7 @@ impl AsyncResolveResource for VfioCde cdev, iommufd, iommu_id, - bar_pt, + bar_addresses, } = resource; // The cdev/iommufd path currently attaches devices to an identity @@ -197,7 +197,7 @@ impl AsyncResolveResource for VfioCde input.register_mmio, input.dma_target.msi_target(), memory_mapper, - bar_pt, + bar_addresses, ) .await?; diff --git a/vm/devices/pci/vfio_assigned_device_resources/src/lib.rs b/vm/devices/pci/vfio_assigned_device_resources/src/lib.rs index 846b0c5ffa..0f636bb3c2 100644 --- a/vm/devices/pci/vfio_assigned_device_resources/src/lib.rs +++ b/vm/devices/pci/vfio_assigned_device_resources/src/lib.rs @@ -10,6 +10,18 @@ use std::fs::File; use vm_resource::ResourceId; use vm_resource::kind::PciDeviceHandleKind; +/// How a virtual BAR should be pre-programmed before the guest configures it. +#[derive(Copy, Clone, Debug, Default, Eq, MeshPayload, PartialEq)] +pub enum BarAddressConfig { + /// Do not pre-program this BAR; let the guest assign it normally. + #[default] + GuestAssigned, + /// Use the physical BAR address reported by the host. + HostAssigned, + /// Use an explicitly supplied host physical address. + Fixed(u64), +} + /// A handle to a VFIO-assigned PCI device (legacy group path). /// /// The launcher opens the VFIO group file descriptor (e.g., `/dev/vfio/N`) @@ -21,9 +33,8 @@ pub struct VfioDeviceHandle { pub pci_id: String, /// Pre-opened VFIO group file descriptor (`/dev/vfio/`). pub group: File, - /// Per-BAR passthrough flags. When `bar_pt[i]` is true, the virtual - /// BAR is pre-programmed with the physical BAR address (GPA = HPA). - pub bar_pt: [bool; 6], + /// Per-BAR pre-programming configuration. + pub bar_addresses: [BarAddressConfig; 6], } impl ResourceId for VfioDeviceHandle { @@ -47,9 +58,8 @@ pub struct VfioCdevDeviceHandle { /// The `--iommu` context ID this device belongs to. All devices /// sharing the same ID share a single IOAS (one set of page tables). pub iommu_id: String, - /// Per-BAR passthrough flags. When `bar_pt[i]` is true, the virtual - /// BAR is pre-programmed with the physical BAR address (GPA = HPA). - pub bar_pt: [bool; 6], + /// Per-BAR pre-programming configuration. + pub bar_addresses: [BarAddressConfig; 6], } impl ResourceId for VfioCdevDeviceHandle { From eab9eb202b551070ce8f7d4c1a5f832633d0dc01 Mon Sep 17 00:00:00 2001 From: John Starks Date: Wed, 22 Jul 2026 14:39:58 +0000 Subject: [PATCH 2/5] fix --- vmm_tests/vmm_tests/tests/tests/aarch64_exclusive.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/vmm_tests/vmm_tests/tests/tests/aarch64_exclusive.rs b/vmm_tests/vmm_tests/tests/tests/aarch64_exclusive.rs index e3352b0a74..c65f5db8c5 100644 --- a/vmm_tests/vmm_tests/tests/tests/aarch64_exclusive.rs +++ b/vmm_tests/vmm_tests/tests/tests/aarch64_exclusive.rs @@ -11,6 +11,7 @@ use petri::PetriVmmBackend; use petri::openvmm::OpenVmmPetriBackend; use petri::pipette::cmd; use std::time::Duration; +use vfio_assigned_device_resources::BarAddressConfig; use vm_resource::IntoResource; use vmm_test_macros::vmm_test; use vmm_test_macros::vmm_test_with; @@ -148,7 +149,7 @@ async fn boot_no_vmbus_pcie_aarch64_tcg( cdev, iommufd, iommu_id: "iommu0".into(), - bar_pt: [false; 6], + bar_addresses: [BarAddressConfig::GuestAssigned; 6], } .into_resource(), }); @@ -309,7 +310,7 @@ async fn assigned_device_peer_to_peer_dma_aarch64_tcg( cdev: edu_cdev, iommufd, iommu_id: "iommu0".into(), - bar_pt: [false; 6], + bar_addresses: [BarAddressConfig::GuestAssigned; 6], } .into_resource(), }); @@ -320,7 +321,7 @@ async fn assigned_device_peer_to_peer_dma_aarch64_tcg( cdev: ivshmem_cdev, iommufd: iommufd2, iommu_id: "iommu0".into(), - bar_pt: [false; 6], + bar_addresses: [BarAddressConfig::GuestAssigned; 6], } .into_resource(), }); From 61e40a190b1a53759bcfa3f3132349fbaa0789eb Mon Sep 17 00:00:00 2001 From: John Starks Date: Wed, 22 Jul 2026 16:18:55 +0000 Subject: [PATCH 3/5] feedback --- .../pci/vfio_assigned_device/src/lib.rs | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/vm/devices/pci/vfio_assigned_device/src/lib.rs b/vm/devices/pci/vfio_assigned_device/src/lib.rs index 46300268f1..5d9527b852 100644 --- a/vm/devices/pci/vfio_assigned_device/src/lib.rs +++ b/vm/devices/pci/vfio_assigned_device/src/lib.rs @@ -825,12 +825,18 @@ fn apply_bar_addresses( for i in 0..6 { let address = match bar_addresses[i] { BarAddressConfig::GuestAssigned => continue, - BarAddressConfig::HostAssigned => physical_addresses.as_ref().unwrap()[i], + BarAddressConfig::HostAssigned => { + let address = physical_addresses.as_ref().unwrap()[i]; + if address == 0 { + anyhow::bail!( + "BAR {i} host address reported by sysfs is 0; use an explicit address for a VFIO variant-driver BAR" + ); + } + address + } + BarAddressConfig::Fixed(0) => anyhow::bail!("BAR {i} fixed address is 0"), BarAddressConfig::Fixed(address) => address, }; - if address == 0 { - anyhow::bail!("BAR {i} address is 0"); - } let is_64bit = cfg_space::BarEncodingBits::from(bar_flags[i]).type_64_bit(); if !is_64bit && address > u32::MAX as u64 { anyhow::bail!("BAR {i} is 32-bit but address {address:#x} exceeds 4 GB"); @@ -1630,7 +1636,18 @@ mod tests { let mut bar_masks = [0; 6]; bar_masks[0] = 0xffff_f000; - for address in [0, 0x8000_0001, 0x1_0000_0000] { + let mut bar_addresses = [BarAddressConfig::GuestAssigned; 6]; + bar_addresses[0] = BarAddressConfig::Fixed(0); + let error = apply_bar_addresses( + "not-a-real-device", + &bar_flags, + &bar_masks, + &bar_addresses, + ) + .unwrap_err(); + assert_eq!(error.to_string(), "BAR 0 fixed address is 0"); + + for address in [0x8000_0001, 0x1_0000_0000] { let mut bar_addresses = [BarAddressConfig::GuestAssigned; 6]; bar_addresses[0] = BarAddressConfig::Fixed(address); assert!( From 1c30135c57f53babb1d041dc71594a6d85d5e0c9 Mon Sep 17 00:00:00 2001 From: John Starks Date: Thu, 23 Jul 2026 05:15:06 +0000 Subject: [PATCH 4/5] feedback --- .../pci/vfio_assigned_device/src/lib.rs | 85 +++++++++++-------- 1 file changed, 49 insertions(+), 36 deletions(-) diff --git a/vm/devices/pci/vfio_assigned_device/src/lib.rs b/vm/devices/pci/vfio_assigned_device/src/lib.rs index 5d9527b852..7e7d5d49df 100644 --- a/vm/devices/pci/vfio_assigned_device/src/lib.rs +++ b/vm/devices/pci/vfio_assigned_device/src/lib.rs @@ -543,9 +543,8 @@ impl VfioAssignedPciDevice { "VFIO assigned PCI device initialized" ); - // Build initial BAR values. Start from bar_flags (encoding bits - // only — guaranteed clean). For passthrough BARs, overlay the - // physical addresses from sysfs. + // Build initial BAR values from clean encoding bits, applying any + // configured host-assigned or fixed physical addresses. let bars = apply_bar_addresses(&pci_id, &bar_flags, &bar_masks, &bar_addresses)?; let bar_reset_defaults = bars; @@ -805,6 +804,9 @@ fn apply_bar_addresses( if bar_masks[i] == 0 { anyhow::bail!("BAR {i} is not implemented by the device"); } + if i == 5 && cfg_space::BarEncodingBits::from(bar_flags[i]).type_64_bit() { + anyhow::bail!("64-bit BAR at index 5 is invalid"); + } // If the previous BAR is 64-bit, this index is its upper half. if i > 0 && cfg_space::BarEncodingBits::from(bar_flags[i - 1]).type_64_bit() @@ -822,9 +824,11 @@ fn apply_bar_addresses( .then(|| read_physical_bar_addresses(pci_id)) .transpose()?; let mut bars = *bar_flags; - for i in 0..6 { + let mut i = 0; + while i < 6 { + let is_64bit = cfg_space::BarEncodingBits::from(bar_flags[i]).type_64_bit(); let address = match bar_addresses[i] { - BarAddressConfig::GuestAssigned => continue, + BarAddressConfig::GuestAssigned => None, BarAddressConfig::HostAssigned => { let address = physical_addresses.as_ref().unwrap()[i]; if address == 0 { @@ -832,34 +836,36 @@ fn apply_bar_addresses( "BAR {i} host address reported by sysfs is 0; use an explicit address for a VFIO variant-driver BAR" ); } - address + Some(address) } BarAddressConfig::Fixed(0) => anyhow::bail!("BAR {i} fixed address is 0"), - BarAddressConfig::Fixed(address) => address, - }; - let is_64bit = cfg_space::BarEncodingBits::from(bar_flags[i]).type_64_bit(); - if !is_64bit && address > u32::MAX as u64 { - anyhow::bail!("BAR {i} is 32-bit but address {address:#x} exceeds 4 GB"); - } - let address_mask = if is_64bit { - (bar_masks[i + 1] as u64) << 32 | (bar_masks[i] as u64 & !0xf) - } else { - (bar_masks[i] as i32 as i64 as u64) & !0xf + BarAddressConfig::Fixed(address) => Some(address), }; - let size = (!address_mask).wrapping_add(1); - if address & (size - 1) != 0 { - anyhow::bail!("BAR {i} address {address:#x} is not aligned to its size {size:#x}"); - } - bars[i] = (address as u32 & !0xf) | bar_flags[i]; - if is_64bit { - bars[i + 1] = (address >> 32) as u32; + if let Some(address) = address { + if !is_64bit && address > u32::MAX as u64 { + anyhow::bail!("BAR {i} is 32-bit but address {address:#x} exceeds 4 GB"); + } + let address_mask = if is_64bit { + (bar_masks[i + 1] as u64) << 32 | (bar_masks[i] as u64 & !0xf) + } else { + (bar_masks[i] as i32 as i64 as u64) & !0xf + }; + let size = (!address_mask).wrapping_add(1); + if address & (size - 1) != 0 { + anyhow::bail!("BAR {i} address {address:#x} is not aligned to its size {size:#x}"); + } + bars[i] = (address as u32 & !0xf) | bar_flags[i]; + if is_64bit { + bars[i + 1] = (address >> 32) as u32; + } + tracing::info!( + pci_id, + bar_index = i, + address = format_args!("{address:#x}"), + "pre-programmed BAR address" + ); } - tracing::info!( - pci_id, - bar_index = i, - address = format_args!("{address:#x}"), - "pre-programmed BAR address" - ); + i += if is_64bit { 2 } else { 1 }; } Ok(bars) } @@ -1638,13 +1644,9 @@ mod tests { let mut bar_addresses = [BarAddressConfig::GuestAssigned; 6]; bar_addresses[0] = BarAddressConfig::Fixed(0); - let error = apply_bar_addresses( - "not-a-real-device", - &bar_flags, - &bar_masks, - &bar_addresses, - ) - .unwrap_err(); + let error = + apply_bar_addresses("not-a-real-device", &bar_flags, &bar_masks, &bar_addresses) + .unwrap_err(); assert_eq!(error.to_string(), "BAR 0 fixed address is 0"); for address in [0x8000_0001, 0x1_0000_0000] { @@ -1678,6 +1680,17 @@ mod tests { apply_bar_addresses("not-a-real-device", &bar_flags, &bar_masks, &bar_addresses) .is_err() ); + + bar_flags = [0; 6]; + bar_flags[5] = 0x4; + bar_masks = [0; 6]; + bar_masks[5] = 0xffff_f004; + bar_addresses = [BarAddressConfig::GuestAssigned; 6]; + bar_addresses[5] = BarAddressConfig::Fixed(0x8000_0000); + let error = + apply_bar_addresses("not-a-real-device", &bar_flags, &bar_masks, &bar_addresses) + .unwrap_err(); + assert_eq!(error.to_string(), "64-bit BAR at index 5 is invalid"); } /// In-memory config space backing store for unit tests. From 2f14f20b1c2e43a0ece3385441666967fc5faa05 Mon Sep 17 00:00:00 2001 From: John Starks Date: Thu, 23 Jul 2026 08:23:58 +0000 Subject: [PATCH 5/5] fix --- openvmm/openvmm_entry/src/ttrpc/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openvmm/openvmm_entry/src/ttrpc/mod.rs b/openvmm/openvmm_entry/src/ttrpc/mod.rs index 4bd2ce7ed4..ca2482e53e 100644 --- a/openvmm/openvmm_entry/src/ttrpc/mod.rs +++ b/openvmm/openvmm_entry/src/ttrpc/mod.rs @@ -1678,6 +1678,7 @@ fn build_vfio_device(vfio: vmservice::VfioDevice) -> anyhow::Result, ) -> anyhow::Result<[vfio_assigned_device_resources::BarAddressConfig; 6]> { @@ -1949,7 +1950,7 @@ fn build_vhost_user_device( anyhow::bail!("vhost-user is only supported on unix hosts") } -#[cfg(test)] +#[cfg(all(test, target_os = "linux"))] mod tests { use super::*; use vfio_assigned_device_resources::BarAddressConfig;