[sled-agent] Encrypt U.2 zpool top-level datasets

Encryption of U.2 devices relies on key generated by the `KeyManager`
introduced in #2990. The `KeyManager` is parameterized by a
`SecretRetriever`, which provides input key material (IKM) that the
`KeyManager` can use to derive keys. In this commit we introduce a
`LocalSecretRetriever` which provides secrets local to a sled. Right
now the secret is hardcoded, but we intend to either store random
secrets on the M.2s or retrieve them from VPD data in the short
term. Longer term, we will replace the `LocalSecretRetriever` with a
`TrustQuorumSecretRetriever` which will provide a rack secret recomputed
from trust quorum key shares.

The `KeyManger` has been  extended to run in an tokio task and process
requests for from different "requesters". Each requester only has the
ability to request keys that it requires to do its job. Currently there
is a single requester, `StorageKeyRequester`, that is owned by the
`StorageManager` which loans it out to `sled_hardware::Disk` to request
keys on demand.

`sled_hardware::Disk` requests keys for both creation of encrypted
datasets and mounting of existing encrypted datasets. In order to
actually allow ZFS to use the generated keys, it creates keyfiles inside
a RAMDisk. Attempts are made to limit the lifetime of these files as
well as accidental leakage of secrets.

Changes have also been made to `illumos_utils::Zfs` to support
encrytpion. To support future secret reconfiguration with trust quorum
we store an additional `epoch` property on each encrypted dataset, which
allows the `KeyManager` to retrieve the right vesion of the secret from
a `SecretRetriever`.

Cargo.toml

@@ -190,6 +190,7 @@ internal-dns = { path = "internal-dns" }
 ipcc-key-value = { path = "ipcc-key-value" }
 ipnetwork = "0.20"
 itertools = "0.10.5"
+key-manager = { path = "key-manager" }
 lazy_static = "1.4.0"
 libc = "0.2.143"
 linear-map = "1.2.0"

illumos-utils/src/zfs.rs

@@ -40,6 +40,9 @@ enum EnsureFilesystemErrorRaw {
 
     #[error("Unexpected output from ZFS commands: {0}")]
     Output(String),
+
+    #[error("Failed to mount encrypted filesystem")]
+    MountEncryptedFsFailed(crate::ExecutionError),
 }
 
 /// Error returned by [`Zfs::ensure_filesystem`].
@@ -104,6 +107,22 @@ impl fmt::Display for Mountpoint {
     }
 }
 
+/// This is the path for an encryption key used by ZFS
+#[derive(Debug, Clone)]
+pub struct Keypath(pub PathBuf);
+
+impl fmt::Display for Keypath {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.0.display())
+    }
+}
+
+#[derive(Debug)]
+pub struct EncryptionDetails {
+    pub keypath: Keypath,
+    pub epoch: u64,
+}
+
 #[cfg_attr(any(test, feature = "testing"), mockall::automock, allow(dead_code))]
 impl Zfs {
     /// Lists all datasets within a pool or existing dataset.
@@ -142,22 +161,16 @@ impl Zfs {
         mountpoint: Mountpoint,
         zoned: bool,
         do_format: bool,
+        encryption_details: Option<EncryptionDetails>,
     ) -> Result<(), EnsureFilesystemError> {
-        // If the dataset exists, we're done.
-        let mut command = std::process::Command::new(ZFS);
-        let cmd = command.args(&["list", "-Hpo", "name,type,mountpoint", name]);
-        // If the list command returns any valid output, validate it.
-        if let Ok(output) = execute(cmd) {
-            let stdout = String::from_utf8_lossy(&output.stdout);
-            let values: Vec<&str> = stdout.trim().split('\t').collect();
-            if values != &[name, "filesystem", &mountpoint.to_string()] {
-                return Err(EnsureFilesystemError {
-                    name: name.to_string(),
-                    mountpoint,
-                    err: EnsureFilesystemErrorRaw::Output(stdout.to_string()),
-                });
+        if Self::dataset_exists(name, &mountpoint)? {
+            if encryption_details.is_none() {
+                // If the dataset exists, we're done.
+                return Ok(());
+            } else {
+                // We need to load the encryption key and mount the filesystem
+                return Self::mount_encrypted_dataset(name, &mountpoint);
             }
-            return Ok(());
         }
 
         if !do_format {
@@ -174,6 +187,21 @@ impl Zfs {
         if zoned {
             cmd.args(&["-o", "zoned=on"]);
         }
+        if let Some(details) = encryption_details {
+            let keyloc =
+                format!("keylocation=file://{}", details.keypath.to_string());
+            let epoch = format!("oxide:epoch={}", details.epoch);
+            cmd.args(&[
+                "-o",
+                "encryption=aes-256-gcm",
+                "-o",
+                "keyformat=raw",
+                "-o",
+                &keyloc,
+                "-o",
+                &epoch,
+            ]);
+        }
         cmd.args(&["-o", &format!("mountpoint={}", mountpoint), name]);
         execute(cmd).map_err(|err| EnsureFilesystemError {
             name: name.to_string(),
@@ -183,6 +211,45 @@ impl Zfs {
         Ok(())
     }
 
+    fn mount_encrypted_dataset(
+        name: &str,
+        mountpoint: &Mountpoint,
+    ) -> Result<(), EnsureFilesystemError> {
+        let mut command = std::process::Command::new(PFEXEC);
+        let cmd = command.args(&[ZFS, "mount", "-l", name]);
+        execute(cmd).map_err(|err| EnsureFilesystemError {
+            name: name.to_string(),
+            mountpoint: mountpoint.clone(),
+            err: EnsureFilesystemErrorRaw::MountEncryptedFsFailed(err),
+        })?;
+        Ok(())
+    }
+
+    // Return true if the dataset exists, with an optional epoch if there is one.
+    // Epochs are only written to encrypted root datasets
+    fn dataset_exists(
+        name: &str,
+        mountpoint: &Mountpoint,
+    ) -> Result<bool, EnsureFilesystemError> {
+        let mut command = std::process::Command::new(ZFS);
+        let cmd = command.args(&["list", "-Hpo", "name,type,mountpoint", name]);
+        // If the list command returns any valid output, validate it.
+        if let Ok(output) = execute(cmd) {
+            let stdout = String::from_utf8_lossy(&output.stdout);
+            let values: Vec<&str> = stdout.trim().split('\t').collect();
+            if values != &[name, "filesystem", &mountpoint.to_string()] {
+                return Err(EnsureFilesystemError {
+                    name: name.to_string(),
+                    mountpoint: mountpoint.clone(),
+                    err: EnsureFilesystemErrorRaw::Output(stdout.to_string()),
+                });
+            }
+            Ok(true)
+        } else {
+            Ok(false)
+        }
+    }
+
     pub fn set_oxide_value(
         filesystem_name: &str,
         name: &str,

installinator/src/dispatch.rs

@@ -114,7 +114,7 @@ impl DebugHardwareScan {
         // Finding the write destination from the gimlet hardware logs details
         // about what it's doing sufficiently for this subcommand; just create a
         // write destination and then discard it.
-        _ = WriteDestination::from_hardware(log)?;
+        _ = WriteDestination::from_hardware(log).await?;
         Ok(())
     }
 }
@@ -243,11 +243,7 @@ impl InstallOpts {
                 |cx| async move {
                     let destination = if self.install_on_gimlet {
                         let log = log.clone();
-                        tokio::task::spawn_blocking(move || {
-                            WriteDestination::from_hardware(&log)
-                        })
-                        .await
-                        .unwrap()?
+                        WriteDestination::from_hardware(&log).await?
                     } else {
                         // clap ensures `self.destination` is not `None` if
                         // `install_on_gimlet` is false.

installinator/src/hardware.rs

@@ -22,7 +22,7 @@ pub struct Hardware {
 }
 
 impl Hardware {
-    pub fn scan(log: &Logger) -> Result<Self> {
+    pub async fn scan(log: &Logger) -> Result<Self> {
         let is_gimlet = sled_hardware::is_gimlet()
             .context("failed to detect whether host is a gimlet")?;
         ensure!(is_gimlet, "hardware scan only supported on gimlets");
@@ -64,13 +64,14 @@ impl Hardware {
                     }
                     DiskVariant::M2 => (),
                 }
-
-                Some(
-                    Disk::new(log, disk)
-                        .context("failed to instantiate Disk handle for M.2"),
-                )
-            })
-            .collect::<Result<Vec<_>>>()?;
+                DiskVariant::M2 => {
+                    let disk = Disk::new(log, disk, None)
+                        .await
+                        .context("failed to instantiate Disk handle for M.2")?;
+                    m2_disks.push(disk);
+                }
+            }
+        }
 
         Ok(Self { m2_disks })
     }

key-manager/Cargo.toml

@@ -9,7 +9,6 @@ async-trait.workspace = true
 hkdf = "0.12.3"
 secrecy.workspace = true
 sha3.workspace = true
-sled-hardware.workspace = true
 thiserror.workspace = true
 tokio.workspace = true
 zeroize.workspace = true