vfio_assigned_device: allow fixed BAR addresses

Guide/src/reference/openvmm/management/cli.md

@@ -407,7 +407,7 @@ For `--virtio-rng` and `--virtio-console`, use their separate PCIe port flags:
 --iommu id=iommu0 --vfio host=0000:01:00.0,port=rp0,iommu=iommu0
 
 # Pin BAR0 to its physical address for P2P DMA:
---vfio host=0000:01:00.0,port=rp0,bar0=pt
+--vfio host=0000:01:00.0,port=rp0,bar0=host
 ```
 
 ### SMMU (aarch64 only)

Guide/src/user_guide/openvmm/vfio.md

@@ -118,9 +118,12 @@ The `--vfio` value is a comma-separated list of `key=value` pairs:
 - `host=<pci_bdf>` (required) — the PCI BDF of the VFIO device on the host (e.g., `0000:01:00.0`)
 - `port=<name>` (required) — the name of the PCIe root port to attach the device to (must match a `--pcie-root-port` name)
 - `iommu=<id>` (optional) — reference to an `--iommu` context; see [Using iommufd (cdev path)](#using-iommufd-cdev-path) below
-- `bar0=pt` through `bar5=pt` (optional) — pin the specified BAR to its
+- `bar0=host` through `bar5=host` (optional) — pin the specified BAR to its
   physical host address (GPA = HPA); see [Peer-to-peer DMA](#peer-to-peer-dma)
   below
+- `bar0=0x<addr>` through `bar5=0x<addr>` (optional) — pin the specified BAR
+  to an explicit host physical address; use this for BARs synthesized by a
+  VFIO variant driver whose address is not reported through sysfs
 
 ```admonish tip
 You can assign multiple devices by adding more root ports and `--vfio` flags:
@@ -182,7 +185,7 @@ P2P DMA. This means the guest BAR addresses must be identity-mapped to
 the host BAR addresses (GPA = HPA), or P2P DMA will target the wrong
 location.
 
-To enable this, pin the relevant BARs with `bar<N>=pt` on each `--vfio`
+To enable this, pin the relevant BARs with `bar<N>=host` on each `--vfio`
 device and set `preserve_bars` on the root complex so the PCI resource
 allocator keeps pinned BARs at their physical addresses:
 
@@ -192,11 +195,21 @@ sudo openvmm \
     rc0,preserve_bars,low_mmio_base=0xc0000000,high_mmio_base=0x100000000 \
   --pcie-root-port rc0:rp0 \
   --pcie-root-port rc0:rp1 \
-  --vfio host=0000:01:00.0,port=rp0,bar0=pt \
-  --vfio host=0000:02:00.0,port=rp1,bar0=pt \
+  --vfio host=0000:01:00.0,port=rp0,bar0=host \
+  --vfio host=0000:02:00.0,port=rp1,bar0=host \
   ...
 ```
 
+For a BAR synthesized by a VFIO variant driver, the host physical address may
+not appear in the device's sysfs resource table. In that case, provide the
+address directly with `bar<N>=0x<addr>`. For example, the `nvgrace-gpu` driver
+exposes CPU-coherent GPU memory as a synthetic 64-bit BAR4 whose physical base
+comes from the `nvidia,gpu-mem-base-pa` ACPI `_DSD` property:
+
+```bash
+  --vfio host=0008:06:00.0,port=rp0,bar0=host,bar2=host,bar4=0x110000000000
+```
+
 The `low_mmio_base=` and `high_mmio_base=` options pin the MMIO apertures
 to fixed addresses so the allocator can place both pinned and dynamic BARs
 correctly.

openvmm/openvmm_entry/src/cli_args.rs

@@ -3058,7 +3058,7 @@ pub struct PcieRemoteCli {
 
 /// CLI configuration for a VFIO-assigned PCI device.
 ///
-/// Syntax: `host=<bdf>,port=<name>[,iommu=<id>][,bar0=pt..bar5=pt]`
+/// Syntax: `host=<bdf>,port=<name>[,iommu=<id>][,barN=host|barN=0x<addr>]`
 #[cfg(target_os = "linux")]
 #[derive(Clone, Debug)]
 pub struct VfioDeviceCli {
@@ -3069,36 +3069,43 @@ pub struct VfioDeviceCli {
     /// Optional iommufd context ID. When set, uses VFIO cdev + iommufd
     /// instead of the legacy group/container path.
     pub iommu: Option<String>,
-    /// Per-BAR passthrough flags. When `bar_pt[i]` is true, the virtual
-    /// BAR is pre-programmed with the physical BAR address (GPA = HPA).
-    pub bar_pt: [bool; 6],
+    /// Per-BAR pre-programming configuration.
+    pub bar_addresses: [vfio_assigned_device_resources::BarAddressConfig; 6],
 }
 
-/// Marker for a BAR passthrough value; only `pt` is accepted.
+/// Per-BAR address configuration parsed from the CLI.
 #[cfg(target_os = "linux")]
-struct Pt;
+struct BarAddressCli(vfio_assigned_device_resources::BarAddressConfig);
 
 #[cfg(target_os = "linux")]
-impl FromStr for Pt {
+impl FromStr for BarAddressCli {
     type Err = anyhow::Error;
 
     fn from_str(s: &str) -> anyhow::Result<Self> {
-        anyhow::ensure!(s == "pt", "expected 'pt'");
-        Ok(Pt)
+        let config = if s == "host" {
+            vfio_assigned_device_resources::BarAddressConfig::HostAssigned
+        } else if let Some(value) = s.strip_prefix("0x") {
+            let address = u64::from_str_radix(value, 16).context("invalid BAR address")?;
+            anyhow::ensure!(address != 0, "BAR address must be nonzero");
+            vfio_assigned_device_resources::BarAddressConfig::Fixed(address)
+        } else {
+            anyhow::bail!("expected 'host' or a hexadecimal address starting with '0x'");
+        };
+        Ok(Self(config))
     }
 }
 
-/// Per-BAR passthrough flags (`bar0=pt` .. `bar5=pt`), flattened into
+/// Per-BAR address configuration, flattened into
 /// [`VfioArgs`].
 #[cfg(target_os = "linux")]
 #[derive(vmm_cli::KeyValueArgs)]
 struct BarFlags {
-    bar0: Option<Pt>,
-    bar1: Option<Pt>,
-    bar2: Option<Pt>,
-    bar3: Option<Pt>,
-    bar4: Option<Pt>,
-    bar5: Option<Pt>,
+    bar0: Option<BarAddressCli>,
+    bar1: Option<BarAddressCli>,
+    bar2: Option<BarAddressCli>,
+    bar3: Option<BarAddressCli>,
+    bar4: Option<BarAddressCli>,
+    bar5: Option<BarAddressCli>,
 }
 
 /// Raw `--vfio` options, resolved and validated into a [`VfioDeviceCli`].
@@ -3125,20 +3132,20 @@ impl FromStr for VfioDeviceCli {
         }
 
         let bars = args.bars;
-        let bar_pt = [
-            bars.bar0.is_some(),
-            bars.bar1.is_some(),
-            bars.bar2.is_some(),
-            bars.bar3.is_some(),
-            bars.bar4.is_some(),
-            bars.bar5.is_some(),
+        let bar_addresses = [
+            bars.bar0.map(|bar| bar.0).unwrap_or_default(),
+            bars.bar1.map(|bar| bar.0).unwrap_or_default(),
+            bars.bar2.map(|bar| bar.0).unwrap_or_default(),
+            bars.bar3.map(|bar| bar.0).unwrap_or_default(),
+            bars.bar4.map(|bar| bar.0).unwrap_or_default(),
+            bars.bar5.map(|bar| bar.0).unwrap_or_default(),
         ];
 
         Ok(VfioDeviceCli {
             port_name: args.port,
             pci_id: args.host,
             iommu: args.iommu,
-            bar_pt,
+            bar_addresses,
         })
     }
 }
@@ -4805,6 +4812,8 @@ mod tests {
     #[cfg(target_os = "linux")]
     #[test]
     fn test_vfio_device_cli_parse() {
+        use vfio_assigned_device_resources::BarAddressConfig;
+
         // Required keys only.
         let v = VfioDeviceCli::from_str("host=0000:01:00.0,port=rp0").unwrap();
         assert_eq!(v.pci_id, "0000:01:00.0");
@@ -4817,13 +4826,14 @@ mod tests {
         assert_eq!(v.port_name, "rp1");
         assert_eq!(v.iommu.as_deref(), Some("iommu0"));
 
-        // BAR passthrough flags set the corresponding indices.
-        let v = VfioDeviceCli::from_str("host=0000:01:00.0,port=rp0,bar0=pt,bar2=pt").unwrap();
-        assert_eq!(v.bar_pt, [true, false, true, false, false, false]);
-
-        // A BAR flag only accepts `pt`, and requires a value.
-        assert!(VfioDeviceCli::from_str("host=0000:01:00.0,port=rp0,bar0=x").is_err());
-        assert!(VfioDeviceCli::from_str("host=0000:01:00.0,port=rp0,bar0").is_err());
+        let v = VfioDeviceCli::from_str(
+            "host=0000:03:00.0,port=rp2,bar0=host,bar2=0x80000000,bar4=0x110000000000",
+        )
+        .unwrap();
+        assert_eq!(v.bar_addresses[0], BarAddressConfig::HostAssigned);
+        assert_eq!(v.bar_addresses[1], BarAddressConfig::GuestAssigned);
+        assert_eq!(v.bar_addresses[2], BarAddressConfig::Fixed(0x80000000));
+        assert_eq!(v.bar_addresses[4], BarAddressConfig::Fixed(0x110000000000));
     }
 
     #[cfg(target_os = "linux")]
@@ -4853,6 +4863,15 @@ mod tests {
         // Path-traversal characters in the host BDF are rejected.
         assert!(VfioDeviceCli::from_str("host=../../etc/passwd,port=rp0").is_err());
         assert!(VfioDeviceCli::from_str("host=foo/bar,port=rp0").is_err());
+
+        // Invalid and duplicate BAR configurations are rejected.
+        assert!(VfioDeviceCli::from_str("host=0000:01:00.0,port=rp0,bar0=0").is_err());
+        assert!(VfioDeviceCli::from_str("host=0000:01:00.0,port=rp0,bar0=0x0").is_err());
+        assert!(VfioDeviceCli::from_str("host=0000:01:00.0,port=rp0,bar0=0xnope").is_err());
+        assert!(VfioDeviceCli::from_str("host=0000:01:00.0,port=rp0,bar0=pt").is_err());
+        assert!(
+            VfioDeviceCli::from_str("host=0000:01:00.0,port=rp0,bar0=0x1000,bar0=host").is_err()
+        );
     }
 
     #[cfg(target_os = "linux")]

openvmm/openvmm_entry/src/lib.rs

@@ -1058,7 +1058,7 @@ async fn vm_config_from_command_line(
                             cdev,
                             iommufd,
                             iommu_id: iommu_id.clone(),
-                            bar_pt: cli_cfg.bar_pt,
+                            bar_addresses: cli_cfg.bar_addresses,
                         }
                         .into_resource(),
                     })
@@ -1085,7 +1085,7 @@ async fn vm_config_from_command_line(
                         resource: vfio_assigned_device_resources::VfioDeviceHandle {
                             pci_id: cli_cfg.pci_id.clone(),
                             group,
-                            bar_pt: cli_cfg.bar_pt,
+                            bar_addresses: cli_cfg.bar_addresses,
                         }
                         .into_resource(),
                     })

openvmm/openvmm_entry/src/ttrpc/mod.rs

@@ -1643,7 +1643,11 @@ async fn build_pcie_device(
 /// opened. The device must already be bound to `vfio-pci` on the host.
 #[cfg(target_os = "linux")]
 fn build_vfio_device(vfio: vmservice::VfioDevice) -> anyhow::Result<Resource<PciDeviceHandleKind>> {
-    let vmservice::VfioDevice { host_pci_address } = vfio;
+    let vmservice::VfioDevice {
+        host_pci_address,
+        bar_addresses,
+    } = vfio;
+    let bar_addresses = parse_vfio_bar_addresses(bar_addresses)?;
     // The address is joined into a sysfs path below; reject path separators so
     // it cannot escape `/sys/bus/pci/devices` (an absolute path or `..` would
     // otherwise redirect the join).
@@ -1669,11 +1673,41 @@ fn build_vfio_device(vfio: vmservice::VfioDevice) -> anyhow::Result<Resource<Pci
     Ok(vfio_assigned_device_resources::VfioDeviceHandle {
         pci_id: host_pci_address,
         group,
-        bar_pt: [false; 6],
+        bar_addresses,
     }
     .into_resource())
 }
 
+#[cfg(target_os = "linux")]
+fn parse_vfio_bar_addresses(
+    entries: Vec<vmservice::VfioBarAddress>,
+) -> anyhow::Result<[vfio_assigned_device_resources::BarAddressConfig; 6]> {
+    use vfio_assigned_device_resources::BarAddressConfig;
+    use vmservice::vfio_bar_address::Source;
+
+    let mut bar_addresses = [BarAddressConfig::GuestAssigned; 6];
+    for entry in entries {
+        let index =
+            usize::try_from(entry.bar_index).context("VFIO BAR index does not fit usize")?;
+        let config = bar_addresses
+            .get_mut(index)
+            .with_context(|| format!("VFIO BAR index {} is out of range", entry.bar_index))?;
+        anyhow::ensure!(
+            *config == BarAddressConfig::GuestAssigned,
+            "duplicate VFIO BAR index {}",
+            entry.bar_index
+        );
+        *config = match entry.source.context("missing VFIO BAR address source")? {
+            Source::Host(()) => BarAddressConfig::HostAssigned,
+            Source::Fixed(address) => {
+                anyhow::ensure!(address != 0, "VFIO BAR fixed address must be nonzero");
+                BarAddressConfig::Fixed(address)
+            }
+        };
+    }
+    Ok(bar_addresses)
+}
+
 #[cfg(not(target_os = "linux"))]
 fn build_vfio_device(
     _vfio: vmservice::VfioDevice,
@@ -1915,3 +1949,45 @@ fn build_vhost_user_device(
 ) -> anyhow::Result<Resource<VirtioDeviceHandle>> {
     anyhow::bail!("vhost-user is only supported on unix hosts")
 }
+
+#[cfg(all(test, target_os = "linux"))]
+mod tests {
+    use super::*;
+    use vfio_assigned_device_resources::BarAddressConfig;
+    use vmservice::vfio_bar_address::Source;
+
+    fn vfio_bar_address(bar_index: u32, source: Option<Source>) -> vmservice::VfioBarAddress {
+        vmservice::VfioBarAddress { bar_index, source }
+    }
+
+    #[test]
+    fn parse_vfio_bar_address_config() {
+        let bars = parse_vfio_bar_addresses(vec![
+            vfio_bar_address(0, Some(Source::Host(()))),
+            vfio_bar_address(4, Some(Source::Fixed(0x11_0000_0000))),
+        ])
+        .unwrap();
+
+        assert_eq!(bars[0], BarAddressConfig::HostAssigned);
+        assert_eq!(bars[1], BarAddressConfig::GuestAssigned);
+        assert_eq!(bars[4], BarAddressConfig::Fixed(0x11_0000_0000));
+    }
+
+    #[test]
+    fn reject_invalid_vfio_bar_address_config() {
+        assert!(
+            parse_vfio_bar_addresses(vec![vfio_bar_address(6, Some(Source::Host(())))]).is_err()
+        );
+        assert!(parse_vfio_bar_addresses(vec![vfio_bar_address(0, None)]).is_err());
+        assert!(
+            parse_vfio_bar_addresses(vec![vfio_bar_address(0, Some(Source::Fixed(0)))]).is_err()
+        );
+        assert!(
+            parse_vfio_bar_addresses(vec![
+                vfio_bar_address(0, Some(Source::Host(()))),
+                vfio_bar_address(0, Some(Source::Fixed(0x1000))),
+            ])
+            .is_err()
+        );
+    }
+}

openvmm/openvmm_ttrpc_vmservice/src/vmservice.proto

@@ -294,6 +294,21 @@ message NvmeNamespace {
 message VfioDevice {
     // Host PCI address (BDF), e.g. "0000:01:00.0".
     string host_pci_address = 1;
+    // BARs whose initial addresses are pinned. Omitted BARs are assigned by
+    // the guest normally.
+    repeated VfioBarAddress bar_addresses = 2;
+}
+
+// Initial address configuration for one VFIO device BAR.
+message VfioBarAddress {
+    // BAR index, from 0 through 5.
+    uint32 bar_index = 1;
+    oneof source {
+        // Use the physical BAR address reported by the host.
+        google.protobuf.Empty host = 2;
+        // Use an explicit host physical address.
+        uint64 fixed = 3;
+    }
 }
 
 //