add "live tests" (#6427)

.config/nextest.toml

@@ -10,7 +10,9 @@ experimental = ["setup-scripts"]
 [[profile.default.scripts]]
 # Exclude omicron-dev tests from crdb-seed as we explicitly want to simulate an
 # environment where the seed file doesn't exist.
-filter = 'rdeps(nexus-test-utils) - package(omicron-dev)'
+# Exclude omicron-live-tests because those don't need this and also don't have
+# it available in the environment in which they run.
+filter = 'rdeps(nexus-test-utils) - package(omicron-dev) - package(omicron-live-tests)'
 setup = 'crdb-seed'
 
 [profile.ci]
@@ -21,18 +23,26 @@ fail-fast = false
 # invocations of nextest happen.
 command = 'cargo run -p crdb-seed --profile test'
 
+[test-groups]
 # The ClickHouse cluster tests currently rely on a hard-coded set of ports for
 # the nodes in the cluster. We would like to relax this in the future, at which
 # point this test-group configuration can be removed or at least loosened to
 # support testing in parallel. For now, enforce strict serialization for all
 # tests with `replicated` in the name.
-[test-groups]
 clickhouse-cluster = { max-threads = 1 }
+# While most Omicron tests operate with their own simulated control plane, the
+# live-tests operate on a more realistic, shared control plane and test
+# behaviors that conflict with each other.  They need to be run serially.
+live-tests = { max-threads = 1 }
 
 [[profile.default.overrides]]
 filter = 'package(oximeter-db) and test(replicated)'
 test-group = 'clickhouse-cluster'
 
+[[profile.default.overrides]]
+filter = 'package(omicron-live-tests)'
+test-group = 'live-tests'
+
 [[profile.default.overrides]]
 # These tests can time out under heavy contention.
 filter = 'binary_id(omicron-nexus::test_all) and test(::schema::)'

.github/buildomat/build-and-test.sh

@@ -89,6 +89,12 @@ ptime -m timeout 2h cargo nextest run --profile ci --locked --verbose
 banner doctest
 ptime -m timeout 1h cargo test --doc --locked --verbose --no-fail-fast
 
+# Build the live-tests.  This is only supported on illumos.
+# We also can't actually run them here.  See the README for more details.
+if [[ $target_os == "illumos" ]]; then
+    ptime -m cargo xtask live-tests
+fi
+
 # We expect the seed CRDB to be placed here, so we explicitly remove it so the
 # rmdir check below doesn't get triggered. Nextest doesn't have support for
 # teardown scripts so this is the best we've got.

Cargo.toml

@@ -52,6 +52,8 @@ members = [
     "internal-dns",
     "ipcc",
     "key-manager",
+    "live-tests",
+    "live-tests/macros",
     "nexus",
     "nexus-config",
     "nexus-sled-agent-shared",
@@ -170,6 +172,9 @@ default-members = [
     "internal-dns",
     "ipcc",
     "key-manager",
+    # Do not include live-tests in the list of default members because its tests
+    # only work in a deployed system.  The macros can be here, though.
+    "live-tests/macros",
     "nexus",
     "nexus-config",
     "nexus-sled-agent-shared",
@@ -403,6 +408,7 @@ libc = "0.2.158"
 libfalcon = { git = "https://github.com/oxidecomputer/falcon", rev = "e69694a1f7cc9fe31fab27f321017280531fb5f7" }
 libnvme = { git = "https://github.com/oxidecomputer/libnvme", rev = "dd5bb221d327a1bc9287961718c3c10d6bd37da0" }
 linear-map = "1.2.0"
+live-tests-macros = { path = "live-tests/macros" }
 macaddr = { version = "1.0.1", features = ["serde_std"] }
 maplit = "1.0.2"
 mockall = "0.13"

README.adoc

@@ -62,6 +62,8 @@ Nextest https://github.com/nextest-rs/nextest/issues/16[does not support doctest
 
 Similarly, you can run tests inside a https://github.com/oxidecomputer/falcon[Falcon] based VM. This is described in the `test-utils` https://github.com/oxidecomputer/omicron/tree/main/test-utils[README].
 
+There's also a xref:./live-tests/README.adoc[`live-tests`] test suite that can be run by hand in a _deployed_ Omicron system.
+
 === rustfmt and clippy
 
 You can **format the code** using `cargo fmt`.  Make sure to run this before pushing changes.  The CI checks that the code is correctly formatted.

dev-tools/xtask/Cargo.toml

@@ -24,6 +24,7 @@ workspace = true
 # downstream binaries do depend on it.)
 anyhow.workspace = true
 camino.workspace = true
+camino-tempfile.workspace = true
 cargo_toml = "0.20"
 cargo_metadata.workspace = true
 clap.workspace = true
@@ -32,5 +33,6 @@ macaddr.workspace = true
 serde.workspace = true
 swrite.workspace = true
 tabled.workspace = true
+textwrap.workspace = true
 toml.workspace = true
 usdt.workspace = true

dev-tools/xtask/src/check_workspace_deps.rs

@@ -128,6 +128,7 @@ pub fn run_cmd() -> Result<()> {
                 // The tests here should not be run by default, as they require
                 // a running control plane.
                 "end-to-end-tests",
+                "omicron-live-tests",
             ]
             .contains(&package.name.as_str())
             .then_some(&package.id)

dev-tools/xtask/src/clippy.rs

@@ -4,7 +4,8 @@
 
 //! Subcommand: cargo xtask clippy
 
-use anyhow::{bail, Context, Result};
+use crate::common::run_subcmd;
+use anyhow::Result;
 use clap::Parser;
 use std::process::Command;
 
@@ -51,25 +52,5 @@ pub fn run_cmd(args: ClippyArgs) -> Result<()> {
         .arg("--deny")
         .arg("warnings");
 
-    eprintln!(
-        "running: {:?} {}",
-        &cargo,
-        command
-            .get_args()
-            .map(|arg| format!("{:?}", arg.to_str().unwrap()))
-            .collect::<Vec<_>>()
-            .join(" ")
-    );
-
-    let exit_status = command
-        .spawn()
-        .context("failed to spawn child process")?
-        .wait()
-        .context("failed to wait for child process")?;
-
-    if !exit_status.success() {
-        bail!("clippy failed: {}", exit_status);
-    }
-
-    Ok(())
+    run_subcmd(command)
 }

dev-tools/xtask/src/common.rs

@@ -0,0 +1,34 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Common xtask command helpers
+
+use anyhow::{bail, Context, Result};
+use std::process::Command;
+
+/// Runs the given command, printing some basic debug information around it, and
+/// failing with an error message if the command does not exit successfully
+pub fn run_subcmd(mut command: Command) -> Result<()> {
+    eprintln!(
+        "running: {} {}",
+        command.get_program().to_str().unwrap(),
+        command
+            .get_args()
+            .map(|arg| format!("{:?}", arg.to_str().unwrap()))
+            .collect::<Vec<_>>()
+            .join(" ")
+    );
+
+    let exit_status = command
+        .spawn()
+        .context("failed to spawn child process")?
+        .wait()
+        .context("failed to wait for child process")?;
+
+    if !exit_status.success() {
+        bail!("failed: {}", exit_status);
+    }
+
+    Ok(())
+}