[wicket] Add wicket preflight uplink check (#3929)

The goal of this is to aid in initial installation: we expect to see
various issues with uplink connections and configuration (at multiple
layers: can we get an L1 optical link? once we have an L1 link, can we
talk to the external DNS/NTP servers?) that can impact rack setup in a
way that is difficult to diagnose and debug while rack setup is running.
Instead, _prior to running rack setup_, we can run this new `ssh -t
wicket@... preflight uplink` check: it will walk through setting up the
uplink and talking to the DNS and NTP servers more or less the same way
a person (or RSS) would.

There are (at least) two major shortcomings with the implementation of
this PR:

1. This is entirely an external / CLI-driven check; ideally we could
drive this from within the wicket TUI.
2. It only performs an uplink check for the _current switch_. To check
both switches, an operator must upload the RSS config to both wickets
and then ssh into each wicket to run this check. This isn't hard but is
annoying; ideally ssh'ing into one wicket would run checks on both
scrimlets.

But I think this is already useful as-is.

This covers a portion (but not the entirety of) #3198.

Cargo.toml

@@ -178,6 +178,7 @@ either = "1.9.0"
 expectorate = "1.0.7"
 fatfs = "0.3.6"
 flate2 = "1.0.27"
+flume = "0.11.0"
 fs-err = "2.9.0"
 futures = "0.3.28"
 gateway-client = { path = "gateway-client" }

openapi/wicketd.json

@@ -285,6 +285,56 @@
         }
       }
     },
+    "/preflight/uplink": {
+      "get": {
+        "summary": "An endpoint to get the report for the most recent (or still running)",
+        "description": "preflight uplink check.",
+        "operationId": "get_preflight_uplink_report",
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/EventReportForGenericSpec"
+                }
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      },
+      "post": {
+        "summary": "An endpoint to start a preflight check for uplink configuration.",
+        "operationId": "post_start_preflight_uplink_check",
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/PreflightUplinkCheckOptions"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "204": {
+            "description": "resource updated"
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
     "/rack-setup": {
       "get": {
         "summary": "Query current state of rack setup.",
@@ -1025,6 +1075,43 @@
           "request_id"
         ]
       },
+      "EventReportForGenericSpec": {
+        "description": "A report produced from an [`EventBuffer`](crate::EventBuffer).\n\nRemote reports can be passed into a [`StepContext`](crate::StepContext), in which case they show up as nested events.",
+        "type": "object",
+        "properties": {
+          "last_seen": {
+            "nullable": true,
+            "description": "The last event seen.\n\n`last_seen` can be used to retrieve deltas of events.",
+            "type": "integer",
+            "format": "uint",
+            "minimum": 0
+          },
+          "progress_events": {
+            "description": "A list of progress events, or whether we're currently waiting for a progress event.\n\nCurrently, this produces one progress event for each top-level and nested event in progress.",
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/ProgressEventForGenericSpec"
+            }
+          },
+          "root_execution_id": {
+            "nullable": true,
+            "description": "The root execution ID for this report.\n\nEach report has a root execution ID, which ties together all step and progress events. This is always filled out if the list of step events is non-empty.",
+            "type": "string",
+            "format": "uuid"
+          },
+          "step_events": {
+            "description": "A list of step events.\n\nStep events include success and failure events.",
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/StepEventForGenericSpec"
+            }
+          }
+        },
+        "required": [
+          "progress_events",
+          "step_events"
+        ]
+      },
       "EventReportForWicketdEngineSpec": {
         "description": "A report produced from an [`EventBuffer`](crate::EventBuffer).\n\nRemote reports can be passed into a [`StepContext`](crate::StepContext), in which case they show up as nested events.",
         "type": "object",
@@ -1312,6 +1399,17 @@
           "A2"
         ]
       },
+      "PreflightUplinkCheckOptions": {
+        "description": "Options provided to the preflight uplink check.",
+        "type": "object",
+        "properties": {
+          "dns_name_to_query": {
+            "nullable": true,
+            "description": "DNS name to query.",
+            "type": "string"
+          }
+        }
+      },
       "ProgressCounter": {
         "description": "Current progress.\n\nBoth `current` and `total` are abstract counters. These counters are often a number of bytes. There is no guarantee that the counter won't go back in subsequent events; that can happen e.g. if a fetch happens from multiple peers within a single attempt.",
         "type": "object",

update-engine/src/engine.rs

@@ -551,6 +551,17 @@ pub struct StepResult<T, S: StepSpec> {
     pub outcome: StepOutcome<S>,
 }
 
+impl<T, S: StepSpec> StepResult<T, S> {
+    /// Maps a `StepResult<T, S>` to `StepResult<U, S>` by applying a function
+    /// to the contained `output` value, leaving the `outcome` untouched.
+    pub fn map<U, F>(self, op: F) -> StepResult<U, S>
+    where
+        F: FnOnce(T) -> U,
+    {
+        StepResult { output: op(self.output), outcome: self.outcome }
+    }
+}
+
 /// A success result produced by a step.
 #[derive_where(Debug; T: std::fmt::Debug)]
 #[must_use = "StepSuccess must be used"]

update-engine/src/events.rs

@@ -972,6 +972,15 @@ impl<S: StepSpec> StepOutcome<S> {
             Self::Success { .. } | Self::Warning { .. } => false,
         }
     }
+
+    /// Returns the message associated with this outcome, if one exists.
+    pub fn message(&self) -> Option<&Cow<'static, str>> {
+        match self {
+            StepOutcome::Success { message, .. } => message.as_ref(),
+            StepOutcome::Warning { message, .. }
+            | StepOutcome::Skipped { message, .. } => Some(message),
+        }
+    }
 }
 
 #[derive(Deserialize, Serialize, JsonSchema)]

wicket/Cargo.toml

@@ -17,9 +17,11 @@ futures.workspace = true
 hex = { workspace = true, features = ["serde"] }
 humantime.workspace = true
 indexmap.workspace = true
+indicatif.workspace = true
 itertools.workspace = true
 omicron-common.workspace = true
 once_cell.workspace = true
+owo-colors.workspace = true
 reqwest.workspace = true
 rpassword.workspace = true
 semver.workspace = true

wicket/src/dispatch.rs

@@ -12,7 +12,9 @@ use clap::Parser;
 use omicron_common::{address::WICKETD_PORT, FileKv};
 use slog::Drain;
 
-use crate::{rack_setup::SetupArgs, upload::UploadArgs, Runner};
+use crate::{
+    preflight::PreflightArgs, rack_setup::SetupArgs, upload::UploadArgs, Runner,
+};
 
 pub fn exec() -> Result<()> {
     let wicketd_addr =
@@ -34,6 +36,7 @@ pub fn exec() -> Result<()> {
         match args {
             ShellCommand::UploadRepo(args) => args.exec(log, wicketd_addr),
             ShellCommand::Setup(args) => args.exec(log, wicketd_addr),
+            ShellCommand::Preflight(args) => args.exec(log, wicketd_addr),
         }
     } else {
         // Do not expose log messages via standard error since they'll show up
@@ -57,6 +60,9 @@ enum ShellCommand {
     /// Interact with rack setup configuration.
     #[command(subcommand)]
     Setup(SetupArgs),
+    /// Run checks prior to setting up the rack.
+    #[command(subcommand)]
+    Preflight(PreflightArgs),
 }
 
 fn setup_log(

wicket/src/lib.rs

@@ -10,6 +10,7 @@ use std::time::Duration;
 mod dispatch;
 mod events;
 mod keymap;
+mod preflight;
 mod rack_setup;
 mod runner;
 mod state;