Move underlay NICs back into H/W Classification

Cargo.toml

@@ -23,7 +23,6 @@ license = "MPL-2.0"
 repository = "https://github.com/oxidecomputer/opte"
 
 [workspace.dependencies]
-# Internal crates
 derror-macro = { path = "crates/derror-macro" }
 illumos-sys-hdrs = { path = "crates/illumos-sys-hdrs" }
 kstat-macro = { path = "crates/kstat-macro" }
@@ -46,6 +45,7 @@ cfg-if = "1"
 clap = { version = "4", features = ["derive", "string", "wrap_help"] }
 crc32fast = { version = "1", default-features = false }
 criterion = "0.5"
+ctf-bindgen = { git = "https://github.com/oxidecomputer/ctf-bindgen", rev = "96d2a532335d46bd42732161725188a4d67f6644", default-features = false }
 ctor = "0.2"
 darling = "0.20"
 dyn-clone = "1.0"

bin/opteadm/src/lib.rs

@@ -13,6 +13,7 @@ use opte::api::NoResp;
 use opte::api::OpteCmd;
 use opte::api::SetXdeUnderlayReq;
 use opte::engine::ioctl::{self as api};
+use opte_ioctl::fetch_fragile_types;
 use opte_ioctl::run_cmd_ioctl;
 use opte_ioctl::Error;
 use oxide_vpc::api::AddFwRuleReq;
@@ -99,7 +100,9 @@ impl OpteAdm {
         u1: &str,
         u2: &str,
     ) -> Result<NoResp, Error> {
-        let req = SetXdeUnderlayReq { u1: u1.into(), u2: u2.into() };
+        let illumos_state = fetch_fragile_types()?;
+        let req =
+            SetXdeUnderlayReq { u1: u1.into(), u2: u2.into(), illumos_state };
         let cmd = OpteCmd::SetXdeUnderlay;
         run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
     }

crates/opte-api/src/lib.rs

@@ -15,6 +15,7 @@ extern crate std;
 #[macro_use]
 extern crate alloc;
 
+use alloc::collections::BTreeMap;
 use alloc::string::String;
 use core::fmt;
 use core::fmt::Display;
@@ -47,7 +48,7 @@ pub use ulp::*;
 ///
 /// We rely on CI and the check-api-version.sh script to verify that
 /// this number is incremented anytime the oxide-api code changes.
-pub const API_VERSION: u64 = 33;
+pub const API_VERSION: u64 = 34;
 
 /// Major version of the OPTE package.
 pub const MAJOR_VERSION: u64 = 0;
@@ -86,6 +87,40 @@ impl Display for Direction {
 pub struct SetXdeUnderlayReq {
     pub u1: String,
     pub u2: String,
+
+    // There are limited/no CTF utilities available in the
+    // kernel, but there are non-committed (and private) interfaces
+    // we need to use in the short term. We need to be certain that
+    // the aspects we care about match the live system.
+    // Ideally we would build against a fixed proto area and limit
+    // ourselves to known compatible bits -- but that will require
+    // heavy build system rework.
+    pub illumos_state: FragileInternals,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct FragileInternals {
+    pub dld_str_s: StructDef,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct StructField {
+    pub bit_offset: u64,
+    pub ty: FieldType,
+    // TODO: size is not required for the one aspect we need this
+    //       for, which is reading one pointer offset in a struct defn.
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)]
+pub enum FieldType {
+    Pointer,
+    Other(Option<String>),
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct StructDef {
+    pub size: u64,
+    pub fields: BTreeMap<String, StructField>,
 }
 
 /// Clear the underlay devices used by the xde kernel module

lib/opte-ioctl/Cargo.toml

@@ -7,6 +7,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+ctf-bindgen.workspace = true
 opte = { workspace = true, default-features = false, features = ["api"] }
 oxide-vpc = { workspace = true, default-features = false, features = ["api"] }
 

lib/opte-ioctl/src/lib.rs

@@ -4,14 +4,21 @@
 
 // Copyright 2024 Oxide Computer Company
 
+use ctf_bindgen::Ctf;
+use ctf_bindgen::TypeInfo;
+use ctf_bindgen::TypeRepr;
 use opte::api::ClearXdeUnderlayReq;
 use opte::api::CmdOk;
 use opte::api::Direction;
+use opte::api::FieldType;
+use opte::api::FragileInternals;
 use opte::api::NoResp;
 use opte::api::OpteCmd;
 use opte::api::OpteCmdIoctl;
 pub use opte::api::OpteError;
 use opte::api::SetXdeUnderlayReq;
+use opte::api::StructDef;
+use opte::api::StructField;
 use opte::api::API_VERSION;
 use opte::api::XDE_IOC_OPTE_CMD;
 use oxide_vpc::api::AddRouterEntryReq;
@@ -36,6 +43,7 @@ use oxide_vpc::api::SetVirt2PhysReq;
 use oxide_vpc::api::VpcCfg;
 use serde::de::DeserializeOwned;
 use serde::Serialize;
+use std::collections::BTreeMap;
 use std::fs::File;
 use std::fs::OpenOptions;
 use std::os::unix::io::AsRawFd;
@@ -71,6 +79,9 @@ pub enum Error {
 
     #[error("command {0:?} failed: {1:?}")]
     CommandError(OpteCmd, OpteError),
+
+    #[error("failed to fetch local type information: {0:?}")]
+    CtfError(String),
 }
 
 impl From<std::io::Error> for Error {
@@ -185,7 +196,9 @@ impl OpteHdl {
         u1: &str,
         u2: &str,
     ) -> Result<NoResp, Error> {
-        let req = SetXdeUnderlayReq { u1: u1.into(), u2: u2.into() };
+        let illumos_state = fetch_fragile_types()?;
+        let req =
+            SetXdeUnderlayReq { u1: u1.into(), u2: u2.into(), illumos_state };
         let cmd = OpteCmd::SetXdeUnderlay;
         run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
     }
@@ -362,6 +375,56 @@ where
     Err(Error::MaxAttempts(cmd, MAX_ITERATIONS))
 }
 
+pub fn fetch_fragile_types() -> Result<FragileInternals, Error> {
+    let dld_ctf = Ctf::from_file("/system/object/dld/object")
+        .map_err(|e| Error::CtfError(e.to_string()))?;
+
+    let Some(dld) = dld_ctf.find_type_by_name("dld_str_s") else {
+        return Err(Error::CtfError(
+            "failed to find symbol 'dld_str_s'".into(),
+        ));
+    };
+
+    if let TypeRepr::Struct(ref fields) = dld.repr {
+        let mut processed_fields = BTreeMap::new();
+        for field in fields {
+            let field_name = dld_ctf.string_at(field.name_offset);
+            let ty = match dld_ctf.resolve_type(field.type_offset) {
+                Some(t) => match t.repr {
+                    TypeRepr::Othertype => FieldType::Pointer,
+                    _ => FieldType::Other(Some(
+                        dld_ctf.string_at(t.name_offset).into(),
+                    )),
+                },
+                None => FieldType::Other(None),
+            };
+
+            processed_fields.insert(
+                field_name.into(),
+                StructField { bit_offset: u64::from(field.offset), ty },
+            );
+        }
+
+        // invariant: struct should store info as size
+        let TypeInfo::Size(size) = dld.info else {
+            return Err(Error::CtfError(
+                "struct recorded info as 'type'\
+                rather than 'size'"
+                    .into(),
+            ));
+        };
+
+        Ok(FragileInternals {
+            dld_str_s: StructDef {
+                size: size.into(),
+                fields: processed_fields,
+            },
+        })
+    } else {
+        Err(Error::CtfError("symbol 'dld_str_s' is not of type struct".into()))
+    }
+}
+
 unsafe fn ioctl<T>(
     fd: libc::c_int,
     req: libc::c_int,

xde-tests/tests/loopback.rs

@@ -11,7 +11,7 @@ fn test_xde_loopback() -> Result<()> {
     let topol = xde_tests::two_node_topology()?;
 
     // Now we should be able to ping b from a on the overlay.
-    &topol.nodes[0]
+    _ = &topol.nodes[0]
         .zone
         .zone
         .zexec(&format!("ping {}", &topol.nodes[1].port.ip()))?;