[opteadm] Add show-xde-underlay command.

This command only adds value for debugging and development
purposes. It is unlikely to be of much value in the field.
It is acutely useful for experimenting with different
combinations of underlying macs, such as this Chelsio and
Mellanox contraption:

$ opteadm show-xde-underlay
LINK    MTU   MAC
mlxcx0  9000  6C:B3:11:88:AC:84
cxgbe1  9000  00:07:43:2F:E6:C8

Signed-off-by: Daniel Levin <daniel.levin@oxidecomputer.com>

Author: daniel-levin

bin/opteadm/src/bin/opteadm.rs

@@ -17,6 +17,7 @@ use opte::api::Ipv6Addr;
 use opte::api::MAJOR_VERSION;
 use opte::api::MacAddr;
 use opte::api::MulticastUnderlay;
+use opte::api::ReadXdeUnderlayResp;
 use opte::api::Vni;
 use opte::print::print_layer;
 use opte::print::print_list_layers;
@@ -228,6 +229,9 @@ enum Command {
     /// Clear xde underlay devices
     ClearXdeUnderlay,
 
+    /// Describe the underlay devices, if they exist.
+    ShowXdeUnderlay,
+
     /// Set a virtual-to-physical mapping
     SetV2P { vpc_ip: IpAddr, vpc_mac: MacAddr, underlay_ip: Ipv6Addr, vni: Vni },
 
@@ -732,6 +736,18 @@ fn print_port(t: &mut impl Write, pi: PortInfo) -> std::io::Result<()> {
     Ok(())
 }
 
+fn print_xde_underlay(resp: ReadXdeUnderlayResp) -> std::io::Result<()> {
+    let mut t = TabWriter::new(std::io::stdout());
+    writeln!(t, "LINK\tMTU\tMAC")?;
+
+    for dev in resp.devices {
+        writeln!(t, "{}\t{}\t{}", dev.name, dev.mtu, dev.mac)?;
+    }
+
+    t.flush()?;
+    Ok(())
+}
+
 fn main() -> anyhow::Result<()> {
     let cmd = Command::parse();
     let hdl = OpteHdl::open()?;
@@ -895,6 +911,11 @@ fn main() -> anyhow::Result<()> {
             let _ = hdl.clear_xde_underlay()?;
         }
 
+        Command::ShowXdeUnderlay => {
+            let resp = hdl.read_xde_underlay()?;
+            print_xde_underlay(resp)?;
+        }
+
         Command::RmFwRule { port, direction, id } => {
             let request = RemFwRuleReq { port_name: port, dir: direction, id };
             hdl.remove_firewall_rule(&request)?;

crates/opte-api/src/cmd.rs

@@ -80,6 +80,11 @@ pub enum OpteCmd {
     /// Requires that no XDE ports exist.
     ClearXdeUnderlay = 73,
 
+    /// Describe the XDE underlay devices. Probably only useful for development
+    /// and debugging purposes, as the underlay devices will always be cxgbe0/1 on actual
+    /// sleds, at least until Metro arrives.
+    ReadXdeUnderlay = 74,
+
     /// Set all external IP config for a port.
     SetExternalIps = 80,
 
@@ -145,6 +150,7 @@ impl TryFrom<c_int> for OpteCmd {
             71 => Ok(Self::DeleteXde),
             72 => Ok(Self::SetXdeUnderlay),
             73 => Ok(Self::ClearXdeUnderlay),
+            74 => Ok(Self::ReadXdeUnderlay),
             80 => Ok(Self::SetExternalIps),
             90 => Ok(Self::AllowCidr),
             91 => Ok(Self::RemoveCidr),
@@ -488,3 +494,21 @@ pub struct RuleDump {
     pub data_predicates: Vec<String>,
     pub action: String,
 }
+
+/// Tack on 'Resp' to the name to avoid polluting
+/// the space of symbols in XDE-proper.
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct XdeUnderlayDeviceResp {
+    pub name: String,
+    pub mac: MacAddr,
+    pub mtu: u32,
+}
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct ReadXdeUnderlayResp {
+    /// Today, the number of expected underlay devices is zero or two.
+    /// It is convenient (and lazy) to simply report a possibly-empty list of them to userland.
+    pub devices: Vec<XdeUnderlayDeviceResp>,
+}
+
+impl CmdOk for ReadXdeUnderlayResp {}

lib/opte-ioctl/src/lib.rs

@@ -49,6 +49,7 @@ use oxide_vpc::api::ListPortsResp;
 use oxide_vpc::api::McastSubscribeReq;
 use oxide_vpc::api::McastUnsubscribeAllReq;
 use oxide_vpc::api::McastUnsubscribeReq;
+use oxide_vpc::api::ReadXdeUnderlayResp;
 use oxide_vpc::api::RemFwRuleReq;
 use oxide_vpc::api::RemoveCidrReq;
 use oxide_vpc::api::RemoveCidrResp;
@@ -326,6 +327,14 @@ impl OpteHdl {
         run_cmd_ioctl(self.device.as_raw_fd(), cmd, None::<&()>)
     }
 
+    pub fn read_xde_underlay(&self) -> Result<ReadXdeUnderlayResp, Error> {
+        run_cmd_ioctl(
+            self.device.as_raw_fd(),
+            OpteCmd::ReadXdeUnderlay,
+            None::<&()>,
+        )
+    }
+
     pub fn add_router_entry(
         &self,
         req: &AddRouterEntryReq,

xde/src/xde.rs

@@ -230,8 +230,10 @@ use opte::api::NoResp;
 use opte::api::OpteCmd;
 use opte::api::OpteCmdIoctl;
 use opte::api::OpteError;
+use opte::api::ReadXdeUnderlayResp;
 use opte::api::SetXdeUnderlayReq;
 use opte::api::XDE_IOC_OPTE_CMD;
+use opte::api::XdeUnderlayDeviceResp;
 use opte::d_error::LabelBlock;
 use opte::ddi::kstat::KStatNamed;
 use opte::ddi::kstat::KStatProvider;
@@ -913,6 +915,28 @@ fn clear_xde_underlay_hdlr() -> Result<NoResp, OpteError> {
     clear_xde_underlay()
 }
 
+#[unsafe(no_mangle)]
+fn read_xde_underlay_hdlr() -> Result<ReadXdeUnderlayResp, OpteError> {
+    let state = get_xde_state();
+    let xde_mgmt = state.management_lock.lock();
+
+    let mut resp = ReadXdeUnderlayResp::default();
+    let mut report = |&XdeUnderlayPort { ref name, mac, mtu, .. }| {
+        resp.devices.push(XdeUnderlayDeviceResp {
+            name: name.clone(),
+            mac: mac.into(),
+            mtu,
+        });
+    };
+
+    if let Some(underlay) = &xde_mgmt.underlay {
+        report(&underlay.u1);
+        report(&underlay.u2);
+    }
+
+    Ok(resp)
+}
+
 // This is the entry point for all OPTE commands. It verifies the API
 // version and then multiplexes the command to its appropriate handler.
 #[unsafe(no_mangle)]
@@ -973,6 +997,11 @@ unsafe extern "C" fn xde_ioc_opte_cmd(karg: *mut c_void, mode: c_int) -> c_int {
             hdlr_resp(&mut env, resp)
         }
 
+        OpteCmd::ReadXdeUnderlay => {
+            let resp = read_xde_underlay_hdlr();
+            hdlr_resp(&mut env, resp)
+        }
+
         OpteCmd::DumpLayer => {
             let resp = dump_layer_hdlr(&mut env);
             hdlr_resp(&mut env, resp)