Implement ephemeral IPs (#1458)

* Implement ephemeral IPs

- Updates the current external IP allocation query to handle both
  floating and ephemeral IPs, by assuming that the whole port range is
  already reserved for any existing IP address.
- Add public datastore methods for creating SNAT and Ephemeral IPs,
  delegating to private method for the actual query running/handling
- Updates sagas to include UUID generation for external IPs as separate
  steps, for idempotency, and to create Ephemeral IPs if they're
  requested. Also rework instance creation/migration sagas to select the
  Ephemeral IP address, if one was requested, or the SNAT if not.
- Adds optional restriction of IP Pools to a project. This adds the
  project ID or name in a bunch of places, and updates the external IP
  allocation query to only consider pools which are unrestricted, or
  whose project ID matches the one of the instance we're allocating an
  IP for. This relies on a new index on the `instance_external_ip`
  table, which induces an undesirable sorting (by project, not IP), so
  we add a new sorting criterion to the query.
- Adds tests, especially for the external IP table's check constraints
  which verify integrity of the name / description / instance ID for
  different kinds of addresses, and for restriction of an IP pool to a
  project.
- Plumb the external IPs up to Nexus's public API, including instance
  creation and an endpoint for listing external IPs for an instance.
- Adds integration tests for assignment of Ephemeral IPs and authz tests
  for the endpoint(s)

* remove unused wrapper types around external IP model type

* Review feedback

- More comments and links to issues
- Better handling of external IP vs SNAT IPs during instance
  provision/migrate
- Revert bad MAC address

common/src/sql/dbinit.sql

@@ -958,6 +958,9 @@ CREATE TABLE omicron.public.ip_pool (
     time_modified TIMESTAMPTZ NOT NULL,
     time_deleted TIMESTAMPTZ,
 
+    /* Optional ID of the project for which this pool is reserved. */
+    project_id UUID,
+
     /* The collection's child-resource generation number */
     rcgen INT8 NOT NULL
 );
@@ -984,6 +987,15 @@ CREATE TABLE omicron.public.ip_pool_range (
     /* The range is inclusive of the last address. */
     last_address INET NOT NULL,
     ip_pool_id UUID NOT NULL,
+    /* Optional ID of the project for which this range is reserved.
+     *
+     * NOTE: This denormalizes the tables a bit, since the project_id is
+     * duplicated here and in the parent `ip_pool` table. We're allowing this
+     * for now, since it reduces the complexity of the already-bad IP allocation
+     * query, but we may want to revisit that, and JOIN with the parent table
+     * instead.
+     */
+    project_id UUID,
     /* Tracks child resources, IP addresses allocated out of this range. */
     rcgen INT8 NOT NULL
 );
@@ -1005,14 +1017,46 @@ STORING (first_address)
 WHERE time_deleted IS NULL;
 
 /*
- * External IP addresses used for instance source NAT.
- *
- * NOTE: This currently stores only address and port information for the
- * automatic source NAT supplied for all guest instances. It does not currently
- * store information about ephemeral or floating IPs.
+ * Index supporting allocation of IPs out of a Pool reserved for a project.
+ */
+CREATE INDEX ON omicron.public.ip_pool_range (
+    project_id
+) WHERE
+    time_deleted IS NULL;
+
+
+/* The kind of external IP address. */
+CREATE TYPE omicron.public.ip_kind AS ENUM (
+    /* Automatic source NAT provided to all guests by default */
+    'snat',
+
+    /*
+     * An ephemeral IP is a fixed, known address whose lifetime is the same as
+     * the instance to which it is attached.
+     */
+    'ephemeral',
+
+    /*
+     * A floating IP is an independent, named API resource. It is a fixed,
+     * known address that can be moved between instances. Its lifetime is not
+     * fixed to any instance.
+     */
+    'floating'
+);
+
+/*
+ * External IP addresses used for guest instances
  */
 CREATE TABLE omicron.public.instance_external_ip (
+    /* Identity metadata */
     id UUID PRIMARY KEY,
+
+    /* Name for floating IPs. See the constraints below. */
+    name STRING(128),
+
+    /* Description for floating IPs. See the constraints below. */
+    description STRING(512),
+
     time_created TIMESTAMPTZ NOT NULL,
     time_modified TIMESTAMPTZ NOT NULL,
     time_deleted TIMESTAMPTZ,
@@ -1023,8 +1067,14 @@ CREATE TABLE omicron.public.instance_external_ip (
     /* FK to the `ip_pool_range` table. */
     ip_pool_range_id UUID NOT NULL,
 
-    /* FK to the `instance` table. */
-    instance_id UUID NOT NULL,
+    /* FK to the `project` table. */
+    project_id UUID NOT NULL,
+
+    /* FK to the `instance` table. See the constraints below. */
+    instance_id UUID,
+
+    /* The kind of external address, e.g., ephemeral. */
+    kind omicron.public.ip_kind NOT NULL,
 
     /* The actual external IP address. */
     ip INET NOT NULL,
@@ -1033,7 +1083,28 @@ CREATE TABLE omicron.public.instance_external_ip (
     first_port INT4 NOT NULL,
 
     /* The last port in the allowed range, also inclusive. */
-    last_port INT4 NOT NULL
+    last_port INT4 NOT NULL,
+
+    /* The name must be non-NULL iff this is a floating IP. */
+    CONSTRAINT null_fip_name CHECK (
+        (kind != 'floating' AND name IS NULL) OR
+        (kind = 'floating' AND name IS NOT NULL)
+    ),
+
+    /* The description must be non-NULL iff this is a floating IP. */
+    CONSTRAINT null_fip_description CHECK (
+        (kind != 'floating' AND description IS NULL) OR
+        (kind = 'floating' AND description IS NOT NULL)
+    ),
+
+    /*
+     * Only nullable if this is a floating IP, which may exist not attached
+     * to any instance.
+     */
+    CONSTRAINT null_non_fip_instance_id CHECK (
+        (kind != 'floating' AND instance_id IS NOT NULL) OR
+        (kind = 'floating')
+    )
 );
 
 /*
@@ -1062,7 +1133,7 @@ CREATE UNIQUE INDEX ON omicron.public.instance_external_ip (
 CREATE INDEX ON omicron.public.instance_external_ip (
     instance_id
 )
-    WHERE time_deleted IS NULL;
+    WHERE instance_id IS NOT NULL AND time_deleted IS NULL;
 
 /*******************************************************************/
 

nexus/src/app/external_ip.rs

@@ -0,0 +1,43 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! External IP addresses for instances
+
+use crate::authz;
+use crate::context::OpContext;
+use crate::db::lookup::LookupPath;
+use crate::db::model::IpKind;
+use crate::db::model::Name;
+use crate::external_api::views::ExternalIp;
+use omicron_common::api::external::ListResultVec;
+
+impl super::Nexus {
+    pub async fn instance_list_external_ips(
+        &self,
+        opctx: &OpContext,
+        organization_name: &Name,
+        project_name: &Name,
+        instance_name: &Name,
+    ) -> ListResultVec<ExternalIp> {
+        let (.., authz_instance) = LookupPath::new(opctx, &self.db_datastore)
+            .organization_name(organization_name)
+            .project_name(project_name)
+            .instance_name(instance_name)
+            .lookup_for(authz::Action::Read)
+            .await?;
+        Ok(self
+            .db_datastore
+            .instance_lookup_external_ips(opctx, authz_instance.id())
+            .await?
+            .into_iter()
+            .filter_map(|ip| {
+                if ip.kind == IpKind::SNat {
+                    None
+                } else {
+                    Some(ip.try_into().unwrap())
+                }
+            })
+            .collect::<Vec<_>>())
+    }
+}

nexus/src/app/instance.rs

@@ -13,6 +13,7 @@ use crate::context::OpContext;
 use crate::db;
 use crate::db::identity::Resource;
 use crate::db::lookup::LookupPath;
+use crate::db::model::IpKind;
 use crate::db::model::Name;
 use crate::db::queries::network_interface;
 use crate::external_api::params;
@@ -216,12 +217,14 @@ impl super::Nexus {
         self.db_datastore
             .project_delete_instance(opctx, &authz_instance)
             .await?;
+        // Ignore the count of addresses deleted
         self.db_datastore
             .deallocate_instance_external_ip_by_instance_id(
                 opctx,
                 authz_instance.id(),
             )
-            .await
+            .await?;
+        Ok(())
     }
 
     pub async fn project_instance_migrate(
@@ -488,11 +491,40 @@ impl super::Nexus {
             .derive_guest_network_interface_info(&opctx, &authz_instance)
             .await?;
 
-        let external_ip = self
+        // Collect the external IPs for the instance.
+        // TODO-correctness: Handle Floating IPs, see
+        //  https://github.com/oxidecomputer/omicron/issues/1334
+        let (snat_ip, external_ips): (Vec<_>, Vec<_>) = self
             .db_datastore
-            .instance_lookup_external_ip(&opctx, authz_instance.id())
-            .await
-            .map(ExternalIp::from)?;
+            .instance_lookup_external_ips(&opctx, authz_instance.id())
+            .await?
+            .into_iter()
+            .partition(|ip| ip.kind == IpKind::SNat);
+
+        // Sanity checks on the number and kind of each IP address.
+        if external_ips.len() > crate::app::MAX_EPHEMERAL_IPS_PER_INSTANCE {
+            return Err(Error::internal_error(
+                format!(
+                    "Expected the number of external IPs to be limited to \
+                {}, but found {}",
+                    crate::app::MAX_EPHEMERAL_IPS_PER_INSTANCE,
+                    external_ips.len(),
+                )
+                .as_str(),
+            ));
+        }
+        if snat_ip.len() != 1 {
+            return Err(Error::internal_error(
+                "Expected exactly one SNAT IP address for an instance",
+            ));
+        }
+
+        // For now, we take the Ephemeral IP, if it exists, or the SNAT IP if not.
+        // TODO-correctness: Handle multiple IP addresses, see
+        //  https://github.com/oxidecomputer/omicron/issues/1467
+        let external_ip = ExternalIp::from(
+            external_ips.into_iter().chain(snat_ip).next().unwrap(),
+        );
 
         // Gather the SSH public keys of the actor make the request so
         // that they may be injected into the new image via cloud-init.

nexus/src/app/ip_pool.rs

@@ -106,11 +106,14 @@ impl super::Nexus {
         pool_name: &Name,
         range: &IpRange,
     ) -> UpdateResult<db::model::IpPoolRange> {
-        let (.., authz_pool) = LookupPath::new(opctx, &self.db_datastore)
-            .ip_pool_name(pool_name)
-            .lookup_for(authz::Action::Modify)
-            .await?;
-        self.db_datastore.ip_pool_add_range(opctx, &authz_pool, range).await
+        let (.., authz_pool, db_pool) =
+            LookupPath::new(opctx, &self.db_datastore)
+                .ip_pool_name(pool_name)
+                .fetch_for(authz::Action::Modify)
+                .await?;
+        self.db_datastore
+            .ip_pool_add_range(opctx, &authz_pool, &db_pool, range)
+            .await
     }
 
     pub async fn ip_pool_delete_range(

nexus/src/app/mod.rs

@@ -24,6 +24,7 @@ use uuid::Uuid;
 // by resource.
 mod device_auth;
 mod disk;
+mod external_ip;
 mod iam;
 mod image;
 mod instance;
@@ -53,6 +54,9 @@ pub(crate) const MAX_DISKS_PER_INSTANCE: u32 = 8;
 
 pub(crate) const MAX_NICS_PER_INSTANCE: u32 = 8;
 
+// TODO-completness: Support multiple Ephemeral IPs
+pub(crate) const MAX_EPHEMERAL_IPS_PER_INSTANCE: usize = 1;
+
 /// Manages an Oxide fleet -- the heart of the control plane
 pub struct Nexus {
     /// uuid for this nexus instance.