From 62565eaf4683edf7271cb5e1c81905f9c85701b4 Mon Sep 17 00:00:00 2001 From: bnaecker Date: Mon, 25 Jul 2022 10:26:46 -0700 Subject: [PATCH] Stop abusing OPTE's source NAT configuration for external IPs (#1479) - Update OPTE dep to include the API that allows setting an external IP address explicitly, rather than abusing the SNAT configuration for that - Carve up SNAT addresses into port ranges of 16K. We previously used the entire port range, since OPTE used the whole range anyway to allow inbound connections on any port. This allows 16K outbound connections from guests that don't need inbound, which is a reasonable starting point. - Add explicit SNAT and external IP addresses to the sled agent and client-library types, as well as the representation of instances and OPTE ports. --- Cargo.lock | 8 +- nexus/src/app/instance.rs | 18 ++--- nexus/src/app/sagas/instance_migrate.rs | 15 ++-- nexus/src/db/model/external_ip.rs | 2 +- nexus/src/db/queries/external_ip.rs | 6 +- openapi/sled-agent.json | 73 +++++++++++-------- sled-agent/Cargo.toml | 4 +- sled-agent/src/instance.rs | 25 ++++--- sled-agent/src/instance_manager.rs | 5 +- sled-agent/src/opte/illumos/port.rs | 25 +++++-- sled-agent/src/opte/illumos/port_manager.rs | 43 ++++++++--- sled-agent/src/opte/non_illumos/port.rs | 25 +++++-- .../src/opte/non_illumos/port_manager.rs | 8 +- sled-agent/src/params.rs | 10 ++- tools/install_opte.sh | 2 +- 15 files changed, 163 insertions(+), 106 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a20dd8acce..4c97381bca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2257,7 +2257,7 @@ dependencies = [ [[package]] name = "illumos-ddi-dki" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=eb9e0c687e3c072dbc7d4782475f4ba5a6f50258#eb9e0c687e3c072dbc7d4782475f4ba5a6f50258" +source = "git+https://github.com/oxidecomputer/opte?rev=23884d35aa7908e23accaa77f125a370ddf5c606#23884d35aa7908e23accaa77f125a370ddf5c606" dependencies = [ "illumos-sys-hdrs", ] @@ -2265,7 +2265,7 @@ dependencies = [ [[package]] name = "illumos-sys-hdrs" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=eb9e0c687e3c072dbc7d4782475f4ba5a6f50258#eb9e0c687e3c072dbc7d4782475f4ba5a6f50258" +source = "git+https://github.com/oxidecomputer/opte?rev=23884d35aa7908e23accaa77f125a370ddf5c606#23884d35aa7908e23accaa77f125a370ddf5c606" [[package]] name = "impl-trait-for-tuples" @@ -3360,7 +3360,7 @@ dependencies = [ [[package]] name = "opte" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=eb9e0c687e3c072dbc7d4782475f4ba5a6f50258#eb9e0c687e3c072dbc7d4782475f4ba5a6f50258" +source = "git+https://github.com/oxidecomputer/opte?rev=23884d35aa7908e23accaa77f125a370ddf5c606#23884d35aa7908e23accaa77f125a370ddf5c606" dependencies = [ "anymap", "cfg-if 0.1.10", @@ -3377,7 +3377,7 @@ dependencies = [ [[package]] name = "opte-ioctl" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=eb9e0c687e3c072dbc7d4782475f4ba5a6f50258#eb9e0c687e3c072dbc7d4782475f4ba5a6f50258" +source = "git+https://github.com/oxidecomputer/opte?rev=23884d35aa7908e23accaa77f125a370ddf5c606#23884d35aa7908e23accaa77f125a370ddf5c606" dependencies = [ "libc", "libnet", diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs index 0d67adefa0..2c3967c578 100644 --- a/nexus/src/app/instance.rs +++ b/nexus/src/app/instance.rs @@ -30,10 +30,10 @@ use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::UpdateResult; use omicron_common::api::internal::nexus; -use sled_agent_client::types::ExternalIp; use sled_agent_client::types::InstanceRuntimeStateMigrateParams; use sled_agent_client::types::InstanceRuntimeStateRequested; use sled_agent_client::types::InstanceStateRequested; +use sled_agent_client::types::SourceNatConfig; use sled_agent_client::Client as SledAgentClient; use std::sync::Arc; use uuid::Uuid; @@ -538,6 +538,8 @@ impl super::Nexus { .partition(|ip| ip.kind == IpKind::SNat); // Sanity checks on the number and kind of each IP address. + // TODO-correctness: Handle multiple IP addresses, see + // https://github.com/oxidecomputer/omicron/issues/1467 if external_ips.len() > MAX_EXTERNAL_IPS_PER_INSTANCE { return Err(Error::internal_error( format!( @@ -549,18 +551,15 @@ impl super::Nexus { .as_str(), )); } + let external_ips = + external_ips.into_iter().map(|model| model.ip.ip()).collect(); if snat_ip.len() != 1 { return Err(Error::internal_error( "Expected exactly one SNAT IP address for an instance", )); } - - // For now, we take the Ephemeral IP, if it exists, or the SNAT IP if not. - // TODO-correctness: Handle multiple IP addresses, see - // https://github.com/oxidecomputer/omicron/issues/1467 - let external_ip = ExternalIp::from( - external_ips.into_iter().chain(snat_ip).next().unwrap(), - ); + let source_nat = + SourceNatConfig::from(snat_ip.into_iter().next().unwrap()); // Gather the SSH public keys of the actor make the request so // that they may be injected into the new image via cloud-init. @@ -600,7 +599,8 @@ impl super::Nexus { db_instance.runtime().clone(), ), nics, - external_ip, + source_nat, + external_ips, disks: disk_reqs, cloud_init_bytes: Some(base64::encode( db_instance.generate_cidata(&public_keys)?, diff --git a/nexus/src/app/sagas/instance_migrate.rs b/nexus/src/app/sagas/instance_migrate.rs index a6a9f03f3b..96059202ba 100644 --- a/nexus/src/app/sagas/instance_migrate.rs +++ b/nexus/src/app/sagas/instance_migrate.rs @@ -15,13 +15,13 @@ use omicron_common::api::external::Error; use omicron_common::api::internal::nexus::InstanceRuntimeState; use serde::Deserialize; use serde::Serialize; -use sled_agent_client::types::ExternalIp; use sled_agent_client::types::InstanceEnsureBody; use sled_agent_client::types::InstanceHardware; use sled_agent_client::types::InstanceMigrateParams; use sled_agent_client::types::InstanceRuntimeStateMigrateParams; use sled_agent_client::types::InstanceRuntimeStateRequested; use sled_agent_client::types::InstanceStateRequested; +use sled_agent_client::types::SourceNatConfig; use std::net::Ipv6Addr; use std::sync::Arc; use steno::new_action_noop_undo; @@ -189,24 +189,21 @@ async fn sim_instance_migrate( .as_str(), ))); } + let external_ips = + external_ips.into_iter().map(|model| model.ip.ip()).collect(); if snat_ip.len() != 1 { return Err(ActionError::action_failed(Error::internal_error( "Expected exactly one SNAT IP address for an instance", ))); } - - // For now, we take the Ephemeral IP, if it exists, or the SNAT IP if not. - // TODO-correctness: Handle multiple IP addresses, see - // https://github.com/oxidecomputer/omicron/issues/1467 - let external_ip = ExternalIp::from( - external_ips.into_iter().chain(snat_ip).next().unwrap(), - ); + let source_nat = SourceNatConfig::from(snat_ip.into_iter().next().unwrap()); let instance_hardware = InstanceHardware { runtime: runtime.into(), // TODO: populate NICs nics: vec![], - external_ip, + source_nat, + external_ips, // TODO: populate disks disks: vec![], // TODO: populate cloud init bytes diff --git a/nexus/src/db/model/external_ip.rs b/nexus/src/db/model/external_ip.rs index 8e71a5937b..ec83574bcb 100644 --- a/nexus/src/db/model/external_ip.rs +++ b/nexus/src/db/model/external_ip.rs @@ -68,7 +68,7 @@ pub struct InstanceExternalIp { pub last_port: SqlU16, } -impl From for sled_agent_client::types::ExternalIp { +impl From for sled_agent_client::types::SourceNatConfig { fn from(eip: InstanceExternalIp) -> Self { Self { ip: eip.ip.ip(), diff --git a/nexus/src/db/queries/external_ip.rs b/nexus/src/db/queries/external_ip.rs index 8cd2f7111f..ba871f3d48 100644 --- a/nexus/src/db/queries/external_ip.rs +++ b/nexus/src/db/queries/external_ip.rs @@ -49,11 +49,7 @@ const INSTANCE_EXTERNAL_IP_FROM_CLAUSE: InstanceExternalIpFromClause = // instead to check if a candidate port range has any overlap with an existing // port range, which is more complicated. That's deferred until we actually have // that situation (which may be as soon as allocating ephemeral IPs). -// -// TODO-correctness: We're currently providing the entire port range, even for -// source NAT. We'd like to chunk this up in to something more like quadrants, -// e.g., ranges `[0, 16384)`, `[16384, 32768)`, etc. -const NUM_SOURCE_NAT_PORTS: usize = 1 << 16; +const NUM_SOURCE_NAT_PORTS: usize = 1 << 14; const MAX_PORT: i32 = u16::MAX as _; /// Select the next available IP address and port range for an instance's diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 50b3ab7ecc..ceb8c4ac34 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -709,34 +709,6 @@ "request_id" ] }, - "ExternalIp": { - "description": "An external IP address used for external connectivity for an instance.", - "type": "object", - "properties": { - "first_port": { - "description": "The first port used for instance NAT, inclusive.", - "type": "integer", - "format": "uint16", - "minimum": 0 - }, - "ip": { - "description": "The external address provided to the instance", - "type": "string", - "format": "ip" - }, - "last_port": { - "description": "The last port used for instance NAT, also inclusive.", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "first_port", - "ip", - "last_port" - ] - }, "Generation": { "description": "Generation numbers stored in the database, used for optimistic concurrency control", "type": "integer", @@ -798,8 +770,13 @@ "$ref": "#/components/schemas/DiskRequest" } }, - "external_ip": { - "$ref": "#/components/schemas/ExternalIp" + "external_ips": { + "description": "Zero or more external IP addresses (either floating or ephemeral), provided to an instance to allow inbound connectivity.", + "type": "array", + "items": { + "type": "string", + "format": "ip" + } }, "nics": { "type": "array", @@ -809,13 +786,17 @@ }, "runtime": { "$ref": "#/components/schemas/InstanceRuntimeState" + }, + "source_nat": { + "$ref": "#/components/schemas/SourceNatConfig" } }, "required": [ "disks", - "external_ip", + "external_ips", "nics", - "runtime" + "runtime", + "source_nat" ] }, "InstanceMigrateParams": { @@ -1236,6 +1217,34 @@ "format": "uint8", "minimum": 0 }, + "SourceNatConfig": { + "description": "An IP address and port range used for instance source NAT, i.e., making outbound network connections from guests.", + "type": "object", + "properties": { + "first_port": { + "description": "The first port used for instance NAT, inclusive.", + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "ip": { + "description": "The external address provided to the instance", + "type": "string", + "format": "ip" + }, + "last_port": { + "description": "The last port used for instance NAT, also inclusive.", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "first_port", + "ip", + "last_port" + ] + }, "UpdateArtifact": { "description": "Description of a single update artifact.", "type": "object", diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 9b2aa69e9e..f7e55e2b9b 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -55,8 +55,8 @@ vsss-rs = { version = "2.0.0-pre2", default-features = false, features = ["std"] zone = "0.1" [target.'cfg(target_os = "illumos")'.dependencies] -opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "eb9e0c687e3c072dbc7d4782475f4ba5a6f50258" } -opte = { git = "https://github.com/oxidecomputer/opte", rev = "eb9e0c687e3c072dbc7d4782475f4ba5a6f50258", features = [ "api", "std" ] } +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "23884d35aa7908e23accaa77f125a370ddf5c606" } +opte = { git = "https://github.com/oxidecomputer/opte", rev = "23884d35aa7908e23accaa77f125a370ddf5c606", features = [ "api", "std" ] } [dev-dependencies] expectorate = "1.0.5" diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index 708b18364a..15a8672666 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -16,8 +16,8 @@ use crate::instance_manager::InstanceTicket; use crate::nexus::LazyNexusClient; use crate::opte::PortManager; use crate::opte::PortTicket; -use crate::params::ExternalIp; use crate::params::NetworkInterface; +use crate::params::SourceNatConfig; use crate::params::{ InstanceHardware, InstanceMigrateParams, InstanceRuntimeStateRequested, InstanceSerialConsoleData, @@ -219,7 +219,8 @@ struct InstanceInner { // Guest NIC and OPTE port information requested_nics: Vec, - external_ip: ExternalIp, + source_nat: SourceNatConfig, + external_ips: Vec, // Disk related properties requested_disks: Vec, @@ -480,7 +481,8 @@ impl Instance { vnic_allocator, port_manager, requested_nics: initial.nics, - external_ip: initial.external_ip, + source_nat: initial.source_nat, + external_ips: initial.external_ips, requested_disks: initial.disks, cloud_init_bytes: initial.cloud_init_bytes, state: InstanceStates::new(initial.runtime), @@ -502,12 +504,16 @@ impl Instance { let mut opte_ports = Vec::with_capacity(inner.requested_nics.len()); let mut port_tickets = Vec::with_capacity(inner.requested_nics.len()); for nic in inner.requested_nics.iter() { - let external_ip = - if nic.primary { Some(inner.external_ip) } else { None }; + let (snat, external_ips) = if nic.primary { + (Some(inner.source_nat), Some(inner.external_ips.clone())) + } else { + (None, None) + }; let port = inner.port_manager.create_port( *inner.id(), nic, - external_ip, + snat, + external_ips, )?; port_tickets.push(port.ticket()); opte_ports.push(port); @@ -793,8 +799,8 @@ mod test { use crate::illumos::dladm::Etherstub; use crate::nexus::LazyNexusClient; use crate::opte::PortManager; - use crate::params::ExternalIp; use crate::params::InstanceStateRequested; + use crate::params::SourceNatConfig; use chrono::Utc; use macaddr::MacAddr6; use omicron_common::api::external::{ @@ -839,11 +845,12 @@ mod test { time_updated: Utc::now(), }, nics: vec![], - external_ip: ExternalIp { + source_nat: SourceNatConfig { ip: IpAddr::from(Ipv4Addr::new(10, 0, 0, 1)), first_port: 0, - last_port: 1 << 14 - 1, + last_port: 16_384, }, + external_ips: vec![], disks: vec![], cloud_init_bytes: None, } diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index 00c3e6349f..6284dc0acc 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -230,8 +230,8 @@ mod test { use crate::illumos::{dladm::MockDladm, zone::MockZones}; use crate::instance::MockInstance; use crate::nexus::LazyNexusClient; - use crate::params::ExternalIp; use crate::params::InstanceStateRequested; + use crate::params::SourceNatConfig; use chrono::Utc; use macaddr::MacAddr6; use omicron_common::api::external::{ @@ -271,11 +271,12 @@ mod test { time_updated: Utc::now(), }, nics: vec![], - external_ip: ExternalIp { + source_nat: SourceNatConfig { ip: IpAddr::from(Ipv4Addr::new(10, 0, 0, 1)), first_port: 0, last_port: 1 << 14 - 1, }, + external_ips: vec![], disks: vec![], cloud_init_bytes: None, } diff --git a/sled-agent/src/opte/illumos/port.rs b/sled-agent/src/opte/illumos/port.rs index 93fbe57373..efc15984bc 100644 --- a/sled-agent/src/opte/illumos/port.rs +++ b/sled-agent/src/opte/illumos/port.rs @@ -9,7 +9,7 @@ use crate::opte::BoundaryServices; use crate::opte::Gateway; use crate::opte::PortTicket; use crate::opte::Vni; -use crate::params::ExternalIp; +use crate::params::SourceNatConfig; use ipnetwork::IpNetwork; use macaddr::MacAddr6; use std::net::IpAddr; @@ -34,9 +34,12 @@ struct PortInner { _vni: Vni, // IP address of the hosting sled _underlay_ip: Ipv6Addr, - // The external IP information for this port, or None if it has no external - // connectivity. Only the primary interface has Some(_) here. - external_ip: Option, + // The external IP address and port range provided for this port, to allow + // outbound network connectivity. + source_nat: Option, + // The external IP addresses provided to this port, to allow _inbound_ + // network connectivity. + external_ips: Option>, // Information about the virtual gateway, aka OPTE _gateway: Gateway, // Information about Boundary Services, for forwarding traffic between sleds @@ -110,7 +113,8 @@ impl Port { slot: u8, vni: Vni, underlay_ip: Ipv6Addr, - external_ip: Option, + source_nat: Option, + external_ips: Option>, gateway: Gateway, boundary_services: BoundaryServices, vnic: String, @@ -125,7 +129,8 @@ impl Port { slot, _vni: vni, _underlay_ip: underlay_ip, - external_ip, + source_nat, + external_ips, _gateway: gateway, _boundary_services: boundary_services, vnic, @@ -133,8 +138,12 @@ impl Port { } } - pub fn external_ip(&self) -> &Option { - &self.inner.external_ip + pub fn source_nat(&self) -> &Option { + &self.inner.source_nat + } + + pub fn external_ips(&self) -> &Option> { + &self.inner.external_ips } pub fn mac(&self) -> &MacAddr6 { diff --git a/sled-agent/src/opte/illumos/port_manager.rs b/sled-agent/src/opte/illumos/port_manager.rs index cd6eb20593..23eae2cd96 100644 --- a/sled-agent/src/opte/illumos/port_manager.rs +++ b/sled-agent/src/opte/illumos/port_manager.rs @@ -12,8 +12,8 @@ use crate::opte::Error; use crate::opte::Gateway; use crate::opte::Port; use crate::opte::Vni; -use crate::params::ExternalIp; use crate::params::NetworkInterface; +use crate::params::SourceNatConfig; use ipnetwork::IpNetwork; use macaddr::MacAddr6; use opte::api::IpCidr; @@ -97,9 +97,9 @@ impl PortManagerInner { let secondary_macs = ports .values() .filter_map(|port| { - // Only advertise Ports with an external address (primary - // interface for an instance). - if port.external_ip().is_some() { + // Only advertise Ports with a publicly-visible external IP + // address, on the primary interface for this instance. + if port.external_ips().is_some() { Some(port.mac().to_string()) } else { None @@ -171,7 +171,8 @@ impl PortManager { &self, instance_id: Uuid, nic: &NetworkInterface, - external_ip: Option, + source_nat: Option, + external_ips: Option>, ) -> Result { // TODO-completess: Remove IPv4 restrictions once OPTE supports virtual // IPv6 networks. @@ -210,9 +211,9 @@ impl PortManager { let boundary_services = BoundaryServices::default(); // Describe the source NAT for this instance. - let snat = match external_ip { - Some(ip) => { - let public_ip = match ip.ip { + let snat = match source_nat { + Some(snat) => { + let public_ip = match snat.ip { IpAddr::V4(ip) => ip.into(), IpAddr::V6(_) => { return Err(opte_ioctl::Error::InvalidArgument( @@ -220,7 +221,7 @@ impl PortManager { ).into()); } }; - let ports = ip.first_port..=ip.last_port; + let ports = snat.first_port..=snat.last_port; Some(SNatCfg { public_ip, ports, @@ -232,6 +233,26 @@ impl PortManager { None => None, }; + // Describe the external IP addresses for this instance. + // + // Note that we're currently only taking the first address, which is all + // that OPTE supports. The array is guaranteed to be limited by Nexus. + // See https://github.com/oxidecomputer/omicron/issues/1467 + // See https://github.com/oxidecomputer/opte/issues/196 + let external_ip = match external_ips { + Some(ref ips) if !ips.is_empty() => { + match ips[0] { + IpAddr::V4(ipv4) => Some(ipv4.into()), + IpAddr::V6(_) => { + return Err(opte_ioctl::Error::InvalidArgument( + String::from("IPv6 is not yet supported for external addresses") + ).into()); + } + } + } + _ => None, + }; + // Create the xde device. // // The sequencing here is important. We'd like to make sure things are @@ -258,6 +279,7 @@ impl PortManager { vni, self.inner.underlay_ip, snat, + external_ip, /* passthru = */ false, )?; debug!( @@ -329,7 +351,8 @@ impl PortManager { nic.slot, vni, self.inner.underlay_ip, - external_ip, + source_nat, + external_ips, gateway, boundary_services, vnic, diff --git a/sled-agent/src/opte/non_illumos/port.rs b/sled-agent/src/opte/non_illumos/port.rs index 3d3745329a..556f8380a7 100644 --- a/sled-agent/src/opte/non_illumos/port.rs +++ b/sled-agent/src/opte/non_illumos/port.rs @@ -8,7 +8,7 @@ use crate::opte::BoundaryServices; use crate::opte::Gateway; use crate::opte::PortTicket; use crate::opte::Vni; -use crate::params::ExternalIp; +use crate::params::SourceNatConfig; use ipnetwork::IpNetwork; use macaddr::MacAddr6; use std::net::IpAddr; @@ -34,9 +34,12 @@ struct PortInner { _vni: Vni, // IP address of the hosting sled _underlay_ip: Ipv6Addr, - // The external IP information for this port, or None if it has no external - // connectivity. Only the primary interface has Some(_) here. - external_ip: Option, + // The external IP address and port range provided for this port, to allow + // outbound network connectivity. + source_nat: Option, + // The external IP addresses provided to this port, to allow _inbound_ + // network connectivity. + external_ips: Option>, // Information about the virtual gateway, aka OPTE _gateway: Gateway, // Information about Boundary Services, for forwarding traffic between sleds @@ -75,7 +78,8 @@ impl Port { slot: u8, vni: Vni, underlay_ip: Ipv6Addr, - external_ip: Option, + source_nat: Option, + external_ips: Option>, gateway: Gateway, boundary_services: BoundaryServices, vnic: String, @@ -90,7 +94,8 @@ impl Port { slot, _vni: vni, _underlay_ip: underlay_ip, - external_ip, + source_nat, + external_ips, _gateway: gateway, _boundary_services: boundary_services, vnic, @@ -98,8 +103,12 @@ impl Port { } } - pub fn external_ip(&self) -> &Option { - &self.inner.external_ip + pub fn source_nat(&self) -> &Option { + &self.inner.source_nat + } + + pub fn external_ips(&self) -> &Option> { + &self.inner.external_ips } pub fn mac(&self) -> &MacAddr6 { diff --git a/sled-agent/src/opte/non_illumos/port_manager.rs b/sled-agent/src/opte/non_illumos/port_manager.rs index f263b023b4..e2b1da0792 100644 --- a/sled-agent/src/opte/non_illumos/port_manager.rs +++ b/sled-agent/src/opte/non_illumos/port_manager.rs @@ -10,8 +10,8 @@ use crate::opte::Error; use crate::opte::Gateway; use crate::opte::Port; use crate::opte::Vni; -use crate::params::ExternalIp; use crate::params::NetworkInterface; +use crate::params::SourceNatConfig; use ipnetwork::IpNetwork; use macaddr::MacAddr6; use slog::debug; @@ -109,7 +109,8 @@ impl PortManager { &self, instance_id: Uuid, nic: &NetworkInterface, - external_ip: Option, + source_nat: Option, + external_ips: Option>, ) -> Result { // TODO-completess: Remove IPv4 restrictions once OPTE supports virtual // IPv6 networks. @@ -157,7 +158,8 @@ impl PortManager { nic.slot, vni, self.inner.underlay_ip, - external_ip, + source_nat, + external_ips, gateway, boundary_services, vnic, diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index 29edc5d07d..494a901be5 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -26,9 +26,10 @@ pub struct NetworkInterface { pub slot: u8, } -/// An external IP address used for external connectivity for an instance. +/// An IP address and port range used for instance source NAT, i.e., making +/// outbound network connections from guests. #[derive(Debug, Clone, Copy, Deserialize, Serialize, JsonSchema)] -pub struct ExternalIp { +pub struct SourceNatConfig { /// The external address provided to the instance pub ip: IpAddr, /// The first port used for instance NAT, inclusive. @@ -75,7 +76,10 @@ pub struct DiskEnsureBody { pub struct InstanceHardware { pub runtime: InstanceRuntimeState, pub nics: Vec, - pub external_ip: ExternalIp, + pub source_nat: SourceNatConfig, + /// Zero or more external IP addresses (either floating or ephemeral), + /// provided to an instance to allow inbound connectivity. + pub external_ips: Vec, pub disks: Vec, pub cloud_init_bytes: Option, } diff --git a/tools/install_opte.sh b/tools/install_opte.sh index 3f04f71947..82a3c057f5 100755 --- a/tools/install_opte.sh +++ b/tools/install_opte.sh @@ -122,7 +122,7 @@ function add_publisher { # `helios-netdev` provides the xde kernel driver and the `opteadm` userland tool # for interacting with it. HELIOS_NETDEV_BASE_URL="https://buildomat.eng.oxide.computer/public/file/oxidecomputer/opte/repo" -HELIOS_NETDEV_COMMIT="eb9e0c687e3c072dbc7d4782475f4ba5a6f50258" +HELIOS_NETDEV_COMMIT="23884d35aa7908e23accaa77f125a370ddf5c606" HELIOS_NETDEV_REPO_URL="$HELIOS_NETDEV_BASE_URL/$HELIOS_NETDEV_COMMIT/opte.p5p" HELIOS_NETDEV_REPO_SHA_URL="$HELIOS_NETDEV_BASE_URL/$HELIOS_NETDEV_COMMIT/opte.p5p.sha256" HELIOS_NETDEV_REPO_PATH="$XDE_DIR/$(basename "$HELIOS_NETDEV_REPO_URL")"