Initial integration with Oxide Packet Transformation Engine

- Brings in OPTE via the `opte-ioctl` and `opte` crates.
- Modifies the instance-ensure request from Nexus to the sled agent, to
  carry the actual information required for setting up the guest OPTE
  port. This includes the actual IP subnet and MAC, rather than things
  like the VPC Subnet UUID.
- Adds a database query and method to extract the above information from
  both the network interface and VPC subnet tables.
- Adds OPTE port for the guests (and currently still a VNIC on top),
  with the right OPTE settings for traffic to flow between two guests in
  the same VPC subnet. That's the virtual-to-physical mapping and a
  router entry for the subnet.
- Adds the VNICs over each OPTE port to the running zone. Note that this
  removes the specification of guest NICs for the zone itself as VNICs.
  They are passed as OPTE ports, and the VNIC is pulled out internally,
  so hopefully little will need to change when the VNIC is removed
  entirely.
- Store the main underlay address for the sled agent, currently its
  dropshot server IP address, in the instance manager, and forward to
  each instance. It's then used as the underlay address when setting up
  the OPTE ports for the guest.

Addressing review comments

Add a unique VNI to each VPC

Updating OPTE dependency and package repos

Add dummy/mock module for OPTE on non-illumos systems

common/src/api/external/mod.rs

@@ -1170,6 +1170,15 @@ impl FromStr for IpNet {
     }
 }
 
+impl From<IpNet> for ipnetwork::IpNetwork {
+    fn from(net: IpNet) -> ipnetwork::IpNetwork {
+        match net {
+            IpNet::V4(net) => ipnetwork::IpNetwork::from(net.0),
+            IpNet::V6(net) => ipnetwork::IpNetwork::from(net.0),
+        }
+    }
+}
+
 /// A `RouteTarget` describes the possible locations that traffic matching a
 /// route destination can be sent.
 #[derive(
@@ -1668,6 +1677,62 @@ impl JsonSchema for MacAddr {
     }
 }
 
+/// A Geneve Virtual Network Identifier
+#[derive(
+    Debug,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    Hash,
+    PartialOrd,
+    Ord,
+    Deserialize,
+    Serialize,
+    JsonSchema,
+)]
+pub struct Vni(u32);
+
+impl Vni {
+    const MAX_VNI: u32 = 1 << 24;
+
+    /// Create a new random VNI.
+    pub fn random() -> Self {
+        use rand::Rng;
+        Self(rand::thread_rng().gen_range(0..=Self::MAX_VNI))
+    }
+}
+
+impl From<Vni> for u32 {
+    fn from(vni: Vni) -> u32 {
+        vni.0
+    }
+}
+
+impl TryFrom<u32> for Vni {
+    type Error = Error;
+
+    fn try_from(x: u32) -> Result<Self, Error> {
+        if x <= Self::MAX_VNI {
+            Ok(Self(x))
+        } else {
+            Err(Error::internal_error(
+                format!("Invalid Geneve VNI: {}", x).as_str(),
+            ))
+        }
+    }
+}
+
+impl TryFrom<i32> for Vni {
+    type Error = Error;
+
+    fn try_from(x: i32) -> Result<Self, Error> {
+        Self::try_from(u32::try_from(x).map_err(|_| {
+            Error::internal_error(format!("Invalid Geneve VNI: {}", x).as_str())
+        })?)
+    }
+}
+
 /// A `NetworkInterface` represents a virtual network interface device.
 #[derive(ObjectIdentity, Clone, Debug, Deserialize, JsonSchema, Serialize)]
 pub struct NetworkInterface {

common/src/sql/dbinit.sql

@@ -592,6 +592,13 @@ CREATE TABLE omicron.public.vpc (
     system_router_id UUID NOT NULL,
     dns_name STRING(63) NOT NULL,
 
+    /*
+     * The Geneve Virtual Network Identifier for this VPC. Note that this is a
+     * 24-bit unsigned value, properties which are checked in the application,
+     * not the database.
+     */
+    vni INT4 NOT NULL,
+
     /* The IPv6 prefix allocated to subnets. */
     ipv6_prefix INET NOT NULL,
 
@@ -606,6 +613,11 @@ CREATE UNIQUE INDEX ON omicron.public.vpc (
 ) WHERE
     time_deleted IS NULL;
 
+CREATE UNIQUE INDEX ON omicron.public.vpc (
+    vni
+) WHERE
+    time_deleted IS NULL;
+
 CREATE TABLE omicron.public.vpc_subnet (
     /* Identity metadata (resource) */
     id UUID PRIMARY KEY,

nexus/src/db/datastore.rs

@@ -65,6 +65,7 @@ use diesel::query_dsl::methods::LoadQuery;
 use diesel::upsert::excluded;
 use diesel::{ExpressionMethods, QueryDsl, SelectableHelper};
 use omicron_common::api;
+use omicron_common::api::external;
 use omicron_common::api::external::DataPageParams;
 use omicron_common::api::external::DeleteResult;
 use omicron_common::api::external::Error;
@@ -77,6 +78,7 @@ use omicron_common::api::external::{
     CreateResult, IdentityMetadataCreateParams,
 };
 use omicron_common::bail_unless;
+use sled_agent_client::types as sled_client_types;
 use std::convert::{TryFrom, TryInto};
 use std::net::Ipv6Addr;
 use std::sync::Arc;
@@ -1365,6 +1367,85 @@ impl DataStore {
         Ok(())
     }
 
+    /// Return the information about an instance's network interfaces required
+    /// for the sled agent to instantiate them via OPTE.
+    ///
+    /// OPTE requires information that's currently split across the network
+    /// interface and VPC subnet tables. This query just joins those for each
+    /// NIC in the given instance.
+    pub(crate) async fn derive_guest_network_interface_info(
+        &self,
+        opctx: &OpContext,
+        authz_instance: &authz::Instance,
+    ) -> ListResultVec<sled_client_types::NetworkInterface> {
+        opctx.authorize(authz::Action::ListChildren, authz_instance).await?;
+
+        use db::schema::network_interface;
+        use db::schema::vpc;
+        use db::schema::vpc_subnet;
+
+        // The record type for the results of the below JOIN query
+        #[derive(Debug, diesel::Queryable)]
+        struct NicInfo {
+            name: db::model::Name,
+            ip: ipnetwork::IpNetwork,
+            mac: db::model::MacAddr,
+            ipv4_block: db::model::Ipv4Net,
+            ipv6_block: db::model::Ipv6Net,
+            vni: db::model::Vni,
+            slot: i16,
+        }
+
+        impl From<NicInfo> for sled_client_types::NetworkInterface {
+            fn from(nic: NicInfo) -> sled_client_types::NetworkInterface {
+                let ip_subnet = if nic.ip.is_ipv4() {
+                    external::IpNet::V4(nic.ipv4_block.0)
+                } else {
+                    external::IpNet::V6(nic.ipv6_block.0)
+                };
+                sled_client_types::NetworkInterface {
+                    name: sled_client_types::Name::from(&nic.name.0),
+                    ip: nic.ip.ip().to_string(),
+                    mac: sled_client_types::MacAddr::from(nic.mac.0),
+                    subnet: sled_client_types::IpNet::from(ip_subnet),
+                    vni: sled_client_types::Vni::from(nic.vni.0),
+                    slot: u8::try_from(nic.slot).unwrap(),
+                }
+            }
+        }
+
+        let rows = network_interface::table
+            .filter(network_interface::instance_id.eq(authz_instance.id()))
+            .filter(network_interface::time_deleted.is_null())
+            .inner_join(
+                vpc_subnet::table
+                    .on(network_interface::subnet_id.eq(vpc_subnet::id)),
+            )
+            .inner_join(vpc::table.on(vpc_subnet::vpc_id.eq(vpc::id)))
+            .order_by(network_interface::slot)
+            // TODO-cleanup: Having to specify each column again is less than
+            // ideal, but we can't derive `Selectable` since this is the result
+            // of a JOIN and not from a single table. DRY this out if possible.
+            .select((
+                network_interface::name,
+                network_interface::ip,
+                network_interface::mac,
+                vpc_subnet::ipv4_block,
+                vpc_subnet::ipv6_block,
+                vpc::vni,
+                network_interface::slot,
+            ))
+            .get_results_async::<NicInfo>(self.pool_authorized(opctx).await?)
+            .await
+            .map_err(|e| {
+                public_error_from_diesel_pool(e, ErrorHandler::Server)
+            })?;
+        Ok(rows
+            .into_iter()
+            .map(sled_client_types::NetworkInterface::from)
+            .collect())
+    }
+
     /// List network interfaces associated with a given instance.
     pub async fn instance_list_network_interfaces(
         &self,

nexus/src/db/model/mod.rs

@@ -40,6 +40,7 @@ mod ssh_key;
 mod u16;
 mod update_artifact;
 mod user_builtin;
+mod vni;
 mod volume;
 mod vpc;
 mod vpc_firewall_rule;
@@ -84,6 +85,7 @@ pub use snapshot::*;
 pub use ssh_key::*;
 pub use update_artifact::*;
 pub use user_builtin::*;
+pub use vni::*;
 pub use volume::*;
 pub use vpc::*;
 pub use vpc_firewall_rule::*;

nexus/src/db/model/vni.rs

@@ -0,0 +1,43 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use omicron_common::api::external;
+use diesel::sql_types;
+use diesel::serialize;
+use diesel::deserialize;
+use diesel::serialize::ToSql;
+use diesel::deserialize::FromSql;
+use diesel::backend::RawValue;
+use diesel::backend::Backend;
+use diesel::query_builder::bind_collector::RawBytesBindCollector;
+
+#[derive(Clone, Debug, Copy, AsExpression, FromSqlRow)]
+#[diesel(sql_type = sql_types::Int4)]
+pub struct Vni(pub external::Vni);
+
+impl<DB> ToSql<sql_types::Int4, DB> for Vni
+where
+    DB: Backend<BindCollector = RawBytesBindCollector<DB>>,
+    i32: ToSql<sql_types::Int4, DB>,
+{
+    fn to_sql<'b>(
+        &'b self,
+        out: &mut serialize::Output<'b, '_, DB>,
+    ) -> serialize::Result {
+        // Reborrowing is necessary to ensure that the lifetime of the temporary
+        // i32 created here and `out` is the same, i.e., that `'b = '_`.
+        i32::try_from(u32::from(self.0)).unwrap().to_sql(&mut out.reborrow())
+    }
+}
+
+impl<DB> FromSql<sql_types::Int4, DB> for Vni
+where
+    DB: Backend,
+    i32: FromSql<sql_types::Int4, DB>,
+{
+    fn from_sql(bytes: RawValue<DB>) -> deserialize::Result<Self> {
+        Ok(Vni(external::Vni::try_from(i32::from_sql(bytes)?)?))
+    }
+}
+

nexus/src/db/model/vpc.rs

@@ -5,6 +5,7 @@
 use super::{Generation, Ipv6Net, Name, VpcFirewallRule};
 use crate::db::collection_insert::DatastoreCollection;
 use crate::db::schema::{vpc, vpc_firewall_rule};
+use crate::db::model::Vni;
 use crate::defaults;
 use crate::external_api::params;
 use chrono::{DateTime, Utc};
@@ -20,6 +21,7 @@ pub struct Vpc {
 
     pub project_id: Uuid,
     pub system_router_id: Uuid,
+    pub vni: Vni,
     pub ipv6_prefix: Ipv6Net,
     pub dns_name: Name,
 
@@ -54,6 +56,7 @@ impl Vpc {
             identity,
             project_id,
             system_router_id,
+            vni: Vni(external::Vni::random()),
             ipv6_prefix,
             dns_name: params.dns_name.into(),
             firewall_gen: Generation::new(),

nexus/src/db/schema.rs

@@ -333,6 +333,7 @@ table! {
         time_deleted -> Nullable<Timestamptz>,
         project_id -> Uuid,
         system_router_id -> Uuid,
+        vni -> Int4,
         ipv6_prefix -> Inet,
         dns_name -> Text,
         firewall_gen -> Int8,

nexus/src/nexus.rs

@@ -1847,22 +1847,10 @@ impl Nexus {
             });
         }
 
-        let nics: Vec<external::NetworkInterface> = self
+        let nics = self
             .db_datastore
-            .instance_list_network_interfaces(
-                &opctx,
-                &authz_instance,
-                &DataPageParams {
-                    marker: None,
-                    direction: dropshot::PaginationOrder::Ascending,
-                    limit: std::num::NonZeroU32::new(MAX_NICS_PER_INSTANCE)
-                        .unwrap(),
-                },
-            )
-            .await?
-            .iter()
-            .map(|x| x.clone().into())
-            .collect();
+            .derive_guest_network_interface_info(&opctx, &authz_instance)
+            .await?;
 
         // Ask the sled agent to begin the state change.  Then update the
         // database to reflect the new intermediate state.  If this update is
@@ -1873,7 +1861,7 @@ impl Nexus {
             runtime: sled_agent_client::types::InstanceRuntimeState::from(
                 db_instance.runtime().clone(),
             ),
-            nics: nics.iter().map(|nic| nic.clone().into()).collect(),
+            nics,
             disks: disk_reqs,
             cloud_init_bytes: Some(base64::encode(
                 db_instance.generate_cidata()?,