From 7d0e33b4f5dcde7ab0fd5f7661c2e88f0c6c8c6b Mon Sep 17 00:00:00 2001
From: David Crespo <david.crespo@oxidecomputer.com>
Date: Fri, 18 Mar 2022 16:05:09 -0500
Subject: [PATCH 01/15] try cargo-nextest in CI, why not

---
 .github/workflows/rust.yml                 | 4 +++-
 nexus/src/external_api/http_entrypoints.rs | 3 ++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 091b5dbefd..880c7aa148 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -124,6 +124,8 @@ jobs:
     - name: Download CockroachDB binary
       if: steps.cache-cockroachdb.outputs.cache-hit != 'true'
       run: bash ./tools/ci_download_cockroachdb
+    - name: Install cargo-nextest
+      run: cargo install cargo-nextest
     - name: Create temporary directory for test outputs
       run: mkdir -p $OMICRON_TMP
     - name: Build
@@ -146,7 +148,7 @@ jobs:
       # rebuild here.
       # Put "./cockroachdb/bin" and "./clickhouse" on the PATH for the test
       # suite.
-      run: TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/cockroachdb/bin:$PWD/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo test --no-fail-fast --workspace --locked --verbose
+      run: TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/cockroachdb/bin:$PWD/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo nextest run --no-fail-fast --workspace --locked
     - name: Archive any failed test results
       if: ${{ failure() }}
       # actions/upload-artifact@v2.3.1
diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs
index 7585845622..4b72ca2914 100644
--- a/nexus/src/external_api/http_entrypoints.rs
+++ b/nexus/src/external_api/http_entrypoints.rs
@@ -2138,7 +2138,8 @@ struct RackPathParam {
     rack_id: Uuid,
 }
 
-/// Fetch information about a particular rack.
+// deliberate failure to test output
+/// Fetch information about a specific rack.
 #[endpoint {
     method = GET,
     path = "/hardware/racks/{rack_id}",

From 540abb2deefd8a37788b6e04ac8ce90704ad429b Mon Sep 17 00:00:00 2001
From: David Crespo <david.crespo@oxidecomputer.com>
Date: Fri, 25 Mar 2022 16:28:07 -0500
Subject: [PATCH 02/15] try --failure-output immediate-final

---
 .github/workflows/rust.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 880c7aa148..acd7fcd954 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -148,7 +148,7 @@ jobs:
       # rebuild here.
       # Put "./cockroachdb/bin" and "./clickhouse" on the PATH for the test
       # suite.
-      run: TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/cockroachdb/bin:$PWD/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo nextest run --no-fail-fast --workspace --locked
+      run: TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/cockroachdb/bin:$PWD/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo nextest run --no-fail-fast --workspace --locked --failure-output immediate-final
     - name: Archive any failed test results
       if: ${{ failure() }}
       # actions/upload-artifact@v2.3.1

From 15d49cb492d8c47682c42ffc2047fff285c2e93f Mon Sep 17 00:00:00 2001
From: David Crespo <david.crespo@oxidecomputer.com>
Date: Fri, 25 Mar 2022 16:30:38 -0500
Subject: [PATCH 03/15] also try nextest binary install action

---
 .github/workflows/rust.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index acd7fcd954..5e99fd2607 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -102,6 +102,7 @@ jobs:
     - uses: actions/checkout@28c7f3d2b5162b5ddd3dfd9a45aa55eaf396478b
     - uses: Swatinem/rust-cache@v1
       if: ${{ github.ref != 'refs/heads/main' }}
+    - uses: taiki-e/install-action@nextest
     - name: Report cargo version
       run: cargo --version
     - name: Configure GitHub cache for CockroachDB binaries
@@ -124,8 +125,6 @@ jobs:
     - name: Download CockroachDB binary
       if: steps.cache-cockroachdb.outputs.cache-hit != 'true'
       run: bash ./tools/ci_download_cockroachdb
-    - name: Install cargo-nextest
-      run: cargo install cargo-nextest
     - name: Create temporary directory for test outputs
       run: mkdir -p $OMICRON_TMP
     - name: Build

From c7128dc744fbef94062ddd2280ec07640c8d39d7 Mon Sep 17 00:00:00 2001
From: David Crespo <david.crespo@oxidecomputer.com>
Date: Fri, 25 Mar 2022 17:53:39 -0500
Subject: [PATCH 04/15] add some other random test failures

---
 nexus/src/db/datastore.rs              | 2 +-
 nexus/tests/integration_tests/disks.rs | 2 +-
 sled-agent/src/updates.rs              | 2 +-
 test-utils/tests/test_omicron_dev.rs   | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs
index 08050cab5c..f3069917a3 100644
--- a/nexus/src/db/datastore.rs
+++ b/nexus/src/db/datastore.rs
@@ -3520,7 +3520,7 @@ mod test {
 
         // time_last_used change persists in DB
         let fetched = datastore.session_fetch(token.clone()).await.unwrap();
-        assert!(fetched.time_last_used > session.time_last_used);
+        assert!(fetched.time_last_used < session.time_last_used);
 
         // delete it and fetch should come back with nothing
         let delete = datastore.session_hard_delete(token.clone()).await;
diff --git a/nexus/tests/integration_tests/disks.rs b/nexus/tests/integration_tests/disks.rs
index e2ab7a7978..f0dcae6bcc 100644
--- a/nexus/tests/integration_tests/disks.rs
+++ b/nexus/tests/integration_tests/disks.rs
@@ -151,7 +151,7 @@ async fn test_disk_create_attach_detach_delete(
     let url_instance_disks =
         get_instance_disks_url(instance.identity.name.as_str());
     let disks = disks_list(&client, &url_instance_disks).await;
-    assert_eq!(disks.len(), 0);
+    assert_eq!(disks.len(), 1);
 
     let url_instance_attach_disk =
         get_disk_attach_url(instance.identity.name.as_str());
diff --git a/sled-agent/src/updates.rs b/sled-agent/src/updates.rs
index 38c132a92a..28e7ac5a13 100644
--- a/sled-agent/src/updates.rs
+++ b/sled-agent/src/updates.rs
@@ -96,7 +96,7 @@ mod test {
         nexus_client.expect_cpapi_artifact_download().times(1).return_once(
             move |kind, name, version| {
                 assert_eq!(name, "test_artifact");
-                assert_eq!(version, 3);
+                assert_eq!(version, 4);
                 assert_eq!(kind.to_string(), "zone");
                 let response = Response::builder()
                     .status(StatusCode::OK)
diff --git a/test-utils/tests/test_omicron_dev.rs b/test-utils/tests/test_omicron_dev.rs
index 7e069244de..624e95cac1 100644
--- a/test-utils/tests/test_omicron_dev.rs
+++ b/test-utils/tests/test_omicron_dev.rs
@@ -257,7 +257,7 @@ async fn test_db_run() {
     // Send SIGINT to that process group.  This simulates an interactive session
     // where the user hits ^C.  Make sure everything is cleaned up gracefully.
     eprintln!("sending SIGINT to process group {}", pgid);
-    assert_eq!(0, unsafe { libc::kill(-pgid, libc::SIGINT) });
+    assert_eq!(1, unsafe { libc::kill(-pgid, libc::SIGINT) });
 
     let wait = verify_graceful_exit(dbrun);
     eprintln!("wait result: {:?}", wait);

From 64b361e6d24e96cd27d01f8ee763a0c95bb3b281 Mon Sep 17 00:00:00 2001
From: David Crespo <david.crespo@oxidecomputer.com>
Date: Fri, 25 Mar 2022 18:53:45 -0500
Subject: [PATCH 05/15] take out the failures

---
 nexus/src/db/datastore.rs                  | 2 +-
 nexus/src/external_api/http_entrypoints.rs | 3 +--
 nexus/tests/integration_tests/disks.rs     | 2 +-
 sled-agent/src/updates.rs                  | 2 +-
 test-utils/tests/test_omicron_dev.rs       | 2 +-
 5 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs
index f3069917a3..08050cab5c 100644
--- a/nexus/src/db/datastore.rs
+++ b/nexus/src/db/datastore.rs
@@ -3520,7 +3520,7 @@ mod test {
 
         // time_last_used change persists in DB
         let fetched = datastore.session_fetch(token.clone()).await.unwrap();
-        assert!(fetched.time_last_used < session.time_last_used);
+        assert!(fetched.time_last_used > session.time_last_used);
 
         // delete it and fetch should come back with nothing
         let delete = datastore.session_hard_delete(token.clone()).await;
diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs
index 4b72ca2914..7585845622 100644
--- a/nexus/src/external_api/http_entrypoints.rs
+++ b/nexus/src/external_api/http_entrypoints.rs
@@ -2138,8 +2138,7 @@ struct RackPathParam {
     rack_id: Uuid,
 }
 
-// deliberate failure to test output
-/// Fetch information about a specific rack.
+/// Fetch information about a particular rack.
 #[endpoint {
     method = GET,
     path = "/hardware/racks/{rack_id}",
diff --git a/nexus/tests/integration_tests/disks.rs b/nexus/tests/integration_tests/disks.rs
index f0dcae6bcc..e2ab7a7978 100644
--- a/nexus/tests/integration_tests/disks.rs
+++ b/nexus/tests/integration_tests/disks.rs
@@ -151,7 +151,7 @@ async fn test_disk_create_attach_detach_delete(
     let url_instance_disks =
         get_instance_disks_url(instance.identity.name.as_str());
     let disks = disks_list(&client, &url_instance_disks).await;
-    assert_eq!(disks.len(), 1);
+    assert_eq!(disks.len(), 0);
 
     let url_instance_attach_disk =
         get_disk_attach_url(instance.identity.name.as_str());
diff --git a/sled-agent/src/updates.rs b/sled-agent/src/updates.rs
index 28e7ac5a13..38c132a92a 100644
--- a/sled-agent/src/updates.rs
+++ b/sled-agent/src/updates.rs
@@ -96,7 +96,7 @@ mod test {
         nexus_client.expect_cpapi_artifact_download().times(1).return_once(
             move |kind, name, version| {
                 assert_eq!(name, "test_artifact");
-                assert_eq!(version, 4);
+                assert_eq!(version, 3);
                 assert_eq!(kind.to_string(), "zone");
                 let response = Response::builder()
                     .status(StatusCode::OK)
diff --git a/test-utils/tests/test_omicron_dev.rs b/test-utils/tests/test_omicron_dev.rs
index 624e95cac1..7e069244de 100644
--- a/test-utils/tests/test_omicron_dev.rs
+++ b/test-utils/tests/test_omicron_dev.rs
@@ -257,7 +257,7 @@ async fn test_db_run() {
     // Send SIGINT to that process group.  This simulates an interactive session
     // where the user hits ^C.  Make sure everything is cleaned up gracefully.
     eprintln!("sending SIGINT to process group {}", pgid);
-    assert_eq!(1, unsafe { libc::kill(-pgid, libc::SIGINT) });
+    assert_eq!(0, unsafe { libc::kill(-pgid, libc::SIGINT) });
 
     let wait = verify_graceful_exit(dbrun);
     eprintln!("wait result: {:?}", wait);

From d9411134bdf57f0f2866139e94e04478cc6ccdc3 Mon Sep 17 00:00:00 2001
From: David Crespo <david.crespo@oxidecomputer.com>
Date: Wed, 30 Mar 2022 21:16:31 -0500
Subject: [PATCH 06/15] nextest config file + turn on junit output

---
 .cargo/nextest.toml        | 6 ++++++
 .github/workflows/rust.yml | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)
 create mode 100644 .cargo/nextest.toml

diff --git a/.cargo/nextest.toml b/.cargo/nextest.toml
new file mode 100644
index 0000000000..47db2a1624
--- /dev/null
+++ b/.cargo/nextest.toml
@@ -0,0 +1,6 @@
+[profile.ci]
+fail-fast = false
+failure-output = "immediate-final"
+
+[profile.ci.junit]
+path = "junit.xml"
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 5e99fd2607..515ed7f8ff 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -147,7 +147,7 @@ jobs:
       # rebuild here.
       # Put "./cockroachdb/bin" and "./clickhouse" on the PATH for the test
       # suite.
-      run: TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/cockroachdb/bin:$PWD/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo nextest run --no-fail-fast --workspace --locked --failure-output immediate-final
+      run: TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/cockroachdb/bin:$PWD/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo nextest run --workspace --locked --config-file .cargo/nextest.toml
     - name: Archive any failed test results
       if: ${{ failure() }}
       # actions/upload-artifact@v2.3.1

From 720c55c9366b58f36215dd9299cefc89dadd16b2 Mon Sep 17 00:00:00 2001
From: David Crespo <david.crespo@oxidecomputer.com>
Date: Thu, 14 Apr 2022 15:40:04 -0500
Subject: [PATCH 07/15] error so we can see that beautiful output

---
 nexus/src/external_api/http_entrypoints.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs
index ad24c4b748..5784d526dd 100644
--- a/nexus/src/external_api/http_entrypoints.rs
+++ b/nexus/src/external_api/http_entrypoints.rs
@@ -2002,7 +2002,7 @@ async fn vpc_subnets_post(
     apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await
 }
 
-/// Delete a subnet from a VPC.
+/// Delete a subnet from a VPC. Fail!
 #[endpoint {
     method = DELETE,
     path = "/organizations/{organization_name}/projects/{project_name}/vpcs/{vpc_name}/subnets/{subnet_name}",

From d71595ea2c963422de92816ac7ebcbe6a2aee4e8 Mon Sep 17 00:00:00 2001
From: David Crespo <david.crespo@oxidecomputer.com>
Date: Thu, 14 Apr 2022 18:03:42 -0500
Subject: [PATCH 08/15] Revert "error so we can see that beautiful output"

This reverts commit 720c55c9366b58f36215dd9299cefc89dadd16b2.
---
 nexus/src/external_api/http_entrypoints.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs
index 5784d526dd..ad24c4b748 100644
--- a/nexus/src/external_api/http_entrypoints.rs
+++ b/nexus/src/external_api/http_entrypoints.rs
@@ -2002,7 +2002,7 @@ async fn vpc_subnets_post(
     apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await
 }
 
-/// Delete a subnet from a VPC. Fail!
+/// Delete a subnet from a VPC.
 #[endpoint {
     method = DELETE,
     path = "/organizations/{organization_name}/projects/{project_name}/vpcs/{vpc_name}/subnets/{subnet_name}",

From 2134e59d870f0650ea136cba8dd70b65f387fe5f Mon Sep 17 00:00:00 2001
From: David Crespo <david.crespo@oxidecomputer.com>
Date: Thu, 14 Apr 2022 20:40:19 -0500
Subject: [PATCH 09/15] archive JUnit output, also change a test to try to
 force tests to run

---
 .github/workflows/rust.yml                   | 8 ++++++++
 nexus/tests/integration_tests/console_api.rs | 6 +++---
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 93e922e88f..bd958c44e0 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -168,6 +168,14 @@ jobs:
       # Put "./cockroachdb/bin" and "./clickhouse" on the PATH for the test
       # suite.
       run: TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo nextest run --workspace --locked --config-file .cargo/nextest.toml
+    - name: Archive test runner JUnit output
+      # actions/upload-artifact@v2.3.1
+      uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2
+      with:
+        name: junit_${{ runner.os }}
+        retention-days: 90
+        path: |
+          junit.xml
     - name: Archive any failed test results
       if: ${{ failure() }}
       # actions/upload-artifact@v2.3.1
diff --git a/nexus/tests/integration_tests/console_api.rs b/nexus/tests/integration_tests/console_api.rs
index b1d2c5b423..a8767eb51c 100644
--- a/nexus/tests/integration_tests/console_api.rs
+++ b/nexus/tests/integration_tests/console_api.rs
@@ -54,7 +54,7 @@ async fn test_sessions(cptestctx: &ControlPlaneTestContext) {
         .expect("failed to 401 on unauthed API request");
 
     // console pages don't 401, they 302
-    RequestBuilder::new(&testctx, Method::GET, "/orgs/whatever")
+    RequestBuilder::new(&testctx, Method::GET, "/orgs/nonexistent")
         .expect_status(Some(StatusCode::FOUND))
         .execute()
         .await
@@ -70,7 +70,7 @@ async fn test_sessions(cptestctx: &ControlPlaneTestContext) {
         .await
         .expect("failed to create org with session cookie");
 
-    RequestBuilder::new(&testctx, Method::GET, "/orgs/whatever")
+    RequestBuilder::new(&testctx, Method::GET, "/orgs/nonexistent")
         .header(header::COOKIE, &session_token)
         .expect_status(Some(StatusCode::OK))
         .execute()
@@ -100,7 +100,7 @@ async fn test_sessions(cptestctx: &ControlPlaneTestContext) {
         .await
         .expect("failed to get 401 for unauthed API request");
 
-    RequestBuilder::new(&testctx, Method::GET, "/orgs/whatever")
+    RequestBuilder::new(&testctx, Method::GET, "/orgs/nonexistent")
         .header(header::COOKIE, &session_token)
         .expect_status(Some(StatusCode::FOUND))
         .execute()

From 96c6fbda52c71b19a132b0c963e4eaed9b1a9668 Mon Sep 17 00:00:00 2001
From: David Crespo <david.crespo@oxidecomputer.com>
Date: Thu, 14 Apr 2022 21:05:32 -0500
Subject: [PATCH 10/15] do junit upload more goodly

---
 .github/workflows/rust.yml                   | 4 ++--
 nexus/tests/integration_tests/console_api.rs | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index bd958c44e0..99b8676026 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -167,7 +167,7 @@ jobs:
       # rebuild here.
       # Put "./cockroachdb/bin" and "./clickhouse" on the PATH for the test
       # suite.
-      run: TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo nextest run --workspace --locked --config-file .cargo/nextest.toml
+      run: TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo nextest run --profile ci --workspace --locked --config-file .cargo/nextest.toml
     - name: Archive test runner JUnit output
       # actions/upload-artifact@v2.3.1
       uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2
@@ -175,7 +175,7 @@ jobs:
         name: junit_${{ runner.os }}
         retention-days: 90
         path: |
-          junit.xml
+          target/nextest/ci/junit.xml
     - name: Archive any failed test results
       if: ${{ failure() }}
       # actions/upload-artifact@v2.3.1
diff --git a/nexus/tests/integration_tests/console_api.rs b/nexus/tests/integration_tests/console_api.rs
index a8767eb51c..b991a99e8d 100644
--- a/nexus/tests/integration_tests/console_api.rs
+++ b/nexus/tests/integration_tests/console_api.rs
@@ -54,7 +54,7 @@ async fn test_sessions(cptestctx: &ControlPlaneTestContext) {
         .expect("failed to 401 on unauthed API request");
 
     // console pages don't 401, they 302
-    RequestBuilder::new(&testctx, Method::GET, "/orgs/nonexistent")
+    RequestBuilder::new(&testctx, Method::GET, "/orgs/blah")
         .expect_status(Some(StatusCode::FOUND))
         .execute()
         .await
@@ -70,7 +70,7 @@ async fn test_sessions(cptestctx: &ControlPlaneTestContext) {
         .await
         .expect("failed to create org with session cookie");
 
-    RequestBuilder::new(&testctx, Method::GET, "/orgs/nonexistent")
+    RequestBuilder::new(&testctx, Method::GET, "/orgs/blah")
         .header(header::COOKIE, &session_token)
         .expect_status(Some(StatusCode::OK))
         .execute()
@@ -100,7 +100,7 @@ async fn test_sessions(cptestctx: &ControlPlaneTestContext) {
         .await
         .expect("failed to get 401 for unauthed API request");
 
-    RequestBuilder::new(&testctx, Method::GET, "/orgs/nonexistent")
+    RequestBuilder::new(&testctx, Method::GET, "/orgs/blah")
         .header(header::COOKIE, &session_token)
         .expect_status(Some(StatusCode::FOUND))
         .execute()

From c5ebb72a10eed80bc0238383de15372bcc93ea24 Mon Sep 17 00:00:00 2001
From: David Crespo <david.crespo@oxidecomputer.com>
Date: Mon, 18 Apr 2022 14:36:24 -0500
Subject: [PATCH 11/15] as the wachowskis said after revolutions, forgive me
 for this horrible matrix

---
 .github/workflows/rust.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 99b8676026..595649452e 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -97,6 +97,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-18.04, macos-11 ]
+        test-cmd: [ cargo-test, nextest ]
     steps:
     # actions/checkout@v2
     - uses: actions/checkout@28c7f3d2b5162b5ddd3dfd9a45aa55eaf396478b
@@ -167,7 +168,12 @@ jobs:
       # rebuild here.
       # Put "./cockroachdb/bin" and "./clickhouse" on the PATH for the test
       # suite.
-      run: TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo nextest run --profile ci --workspace --locked --config-file .cargo/nextest.toml
+      run: |
+        TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" \
+          ${{ matrix.test-cmd == 'cargo-test' 
+            && 'cargo test --no-fail-fast --workspace --locked --verbose'
+            || 'cargo nextest run --profile ci --workspace --locked --config-file .cargo/nextest.toml'
+          }}
     - name: Archive test runner JUnit output
       # actions/upload-artifact@v2.3.1
       uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2

From cc46662165d9b37891ad75e334ad02a7ee1fda0d Mon Sep 17 00:00:00 2001
From: David Crespo <david.crespo@oxidecomputer.com>
Date: Mon, 18 Apr 2022 14:46:18 -0500
Subject: [PATCH 12/15] only attempt to upload nextest junit.xml when we're
 using nextest

---
 .github/workflows/rust.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 595649452e..4150b0a260 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -97,7 +97,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-18.04, macos-11 ]
-        test-cmd: [ cargo-test, nextest ]
+        test-cmd: [ cargo-test, cargo-nextest ]
     steps:
     # actions/checkout@v2
     - uses: actions/checkout@28c7f3d2b5162b5ddd3dfd9a45aa55eaf396478b
@@ -174,14 +174,14 @@ jobs:
             && 'cargo test --no-fail-fast --workspace --locked --verbose'
             || 'cargo nextest run --profile ci --workspace --locked --config-file .cargo/nextest.toml'
           }}
-    - name: Archive test runner JUnit output
+    - name: Archive nextest JUnit output
       # actions/upload-artifact@v2.3.1
       uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2
+      if: ${{ matrix.test-cmd == 'cargo-nextest' }}
       with:
         name: junit_${{ runner.os }}
         retention-days: 90
-        path: |
-          target/nextest/ci/junit.xml
+        path: target/nextest/ci/junit.xml
     - name: Archive any failed test results
       if: ${{ failure() }}
       # actions/upload-artifact@v2.3.1

From 5f04247177d05f94f9a8badb61d53dc173f0c941 Mon Sep 17 00:00:00 2001
From: David Crespo <david.crespo@oxidecomputer.com>
Date: Mon, 18 Apr 2022 14:53:36 -0500
Subject: [PATCH 13/15] be less appalling

---
 .github/workflows/rust.yml | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 4150b0a260..55e081455c 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -162,18 +162,24 @@ jobs:
       # - TMPDIR=$OMICRON_TMP: we specify a specific temporary directory so that
       #   failed test outputs will be in a known place that we can grab at the
       #   end without also grabbing random other temporary files.
-      run: TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo build --locked --all-targets --verbose
-    - name: Run tests
-      # Use the same RUSTFLAGS and RUSTDOCFLAGS as above to avoid having to
-      # rebuild here.
-      # Put "./cockroachdb/bin" and "./clickhouse" on the PATH for the test
-      # suite.
       run: |
         TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" \
-          ${{ matrix.test-cmd == 'cargo-test' 
-            && 'cargo test --no-fail-fast --workspace --locked --verbose'
-            || 'cargo nextest run --profile ci --workspace --locked --config-file .cargo/nextest.toml'
-          }}
+          cargo build --locked --all-targets --verbose
+
+    # Only one of the following steps runs in a given job.
+    # Use the same RUSTFLAGS and RUSTDOCFLAGS as above to avoid a rebuild.
+    # Put "./cockroachdb/bin" and "./clickhouse" on the PATH.
+    - name: Run tests with cargo test
+      if: ${{ matrix.test-cmd == 'cargo-test' }}
+      run: |
+        TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" \
+          cargo test --no-fail-fast --workspace --locked --verbose
+    - name: Run tests with cargo nextest
+      if: ${{ matrix.test-cmd == 'cargo-nextest' }}
+      run: |
+        TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" \
+          cargo nextest run --profile ci --workspace --locked --config-file .cargo/nextest.toml
+
     - name: Archive nextest JUnit output
       # actions/upload-artifact@v2.3.1
       uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2

From 6b0c970b24bb263106cab77fecb94c0f0c16c9d3 Mon Sep 17 00:00:00 2001
From: David Crespo <david.crespo@oxidecomputer.com>
Date: Mon, 18 Apr 2022 17:06:23 -0500
Subject: [PATCH 14/15] do nextest in its own job instead of adding to
 build-and-test

---
 .github/workflows/rust.yml | 113 +++++++++++++++++++++++++------------
 tools/ci_clear_space       |  20 +++++++
 2 files changed, 96 insertions(+), 37 deletions(-)
 create mode 100755 tools/ci_clear_space

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 55e081455c..77133b70bb 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -97,35 +97,15 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-18.04, macos-11 ]
-        test-cmd: [ cargo-test, cargo-nextest ]
     steps:
     # actions/checkout@v2
     - uses: actions/checkout@28c7f3d2b5162b5ddd3dfd9a45aa55eaf396478b
     - uses: Swatinem/rust-cache@v1
       if: ${{ github.ref != 'refs/heads/main' }}
-    - uses: taiki-e/install-action@nextest
     - name: Report cargo version
       run: cargo --version
     - name: Remove unnecessary software
-      run: |
-        echo "Disk space:"
-        df -h
-
-        if [ -d "/usr/share/dotnet" ]; then
-          echo "Removing dotnet"
-          sudo rm -rf /usr/share/dotnet
-        fi
-        if [ -d "/usr/local/lib/android" ]; then
-          echo "Removing android"
-          sudo rm -rf /usr/local/lib/android
-        fi
-        if [ -d "/opt/ghc" ]; then
-          echo "Removing haskell"
-          sudo rm -rf /opt/ghc
-        fi
-
-        echo "Disk space:"
-        df -h
+      run: ./tools/ci_clear_space
     - name: Configure GitHub cache for CockroachDB binaries
       id: cache-cockroachdb
       # actions/cache@v2.1.4
@@ -166,28 +146,12 @@ jobs:
         TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" \
           cargo build --locked --all-targets --verbose
 
-    # Only one of the following steps runs in a given job.
     # Use the same RUSTFLAGS and RUSTDOCFLAGS as above to avoid a rebuild.
     # Put "./cockroachdb/bin" and "./clickhouse" on the PATH.
     - name: Run tests with cargo test
-      if: ${{ matrix.test-cmd == 'cargo-test' }}
       run: |
         TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" \
           cargo test --no-fail-fast --workspace --locked --verbose
-    - name: Run tests with cargo nextest
-      if: ${{ matrix.test-cmd == 'cargo-nextest' }}
-      run: |
-        TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" \
-          cargo nextest run --profile ci --workspace --locked --config-file .cargo/nextest.toml
-
-    - name: Archive nextest JUnit output
-      # actions/upload-artifact@v2.3.1
-      uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2
-      if: ${{ matrix.test-cmd == 'cargo-nextest' }}
-      with:
-        name: junit_${{ runner.os }}
-        retention-days: 90
-        path: target/nextest/ci/junit.xml
     - name: Archive any failed test results
       if: ${{ failure() }}
       # actions/upload-artifact@v2.3.1
@@ -199,3 +163,78 @@ jobs:
           ${{ env.OMICRON_TMP }}
           !${{ env.OMICRON_TMP }}/crdb-base
           !${{ env.OMICRON_TMP }}/rustc*
+
+  # Copy of build-and-test. Only difference is it uses cargo-nextest instead of
+  # cargo test and archives JUnit output instead of all of OMICRON_TMP
+  build-and-nextest:
+    env:
+      OMICRON_TMP: /tmp/omicron_tmp
+    needs: skip_duplicate_jobs
+    if: ${{ needs.skip_duplicate_jobs.outputs.should_skip != 'true' }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-18.04, macos-11 ]
+    steps:
+    # actions/checkout@v2
+    - uses: actions/checkout@28c7f3d2b5162b5ddd3dfd9a45aa55eaf396478b
+    - uses: Swatinem/rust-cache@v1
+      if: ${{ github.ref != 'refs/heads/main' }}
+    - uses: taiki-e/install-action@nextest
+    - name: Report cargo version
+      run: cargo --version
+    - name: Remove unnecessary software
+      run: ./tools/ci_clear_space
+    - name: Configure GitHub cache for CockroachDB binaries
+      id: cache-cockroachdb
+      # actions/cache@v2.1.4
+      uses: actions/cache@26968a09c0ea4f3e233fdddbafd1166051a095f6
+      with:
+        key: ${{ runner.os }}-cockroach-binary-${{ hashFiles('tools/cockroachdb_checksums') }}
+        path: "./out/cockroachdb"
+    - name: Configure GitHub cache for ClickHouse binaries
+      id: cache-clickhouse
+      # actions/cache@v2.1.4
+      uses: actions/cache@26968a09c0ea4f3e233fdddbafd1166051a095f6
+      with:
+        key: ${{ runner.os }}-clickhouse-binary-${{ hashFiles('tools/clickhouse_checksums') }}
+        path: "./out/clickhouse"
+    - name: Download ClickHouse
+      if: steps.cache-clickhouse.outputs.cache-hit != 'true'
+      run: ./tools/ci_download_clickhouse
+    - name: Download CockroachDB binary
+      if: steps.cache-cockroachdb.outputs.cache-hit != 'true'
+      run: bash ./tools/ci_download_cockroachdb
+    - name: Create temporary directory for test outputs
+      run: mkdir -p $OMICRON_TMP
+    - name: Build
+      # We build with:
+      # - RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings": disallow warnings
+      #   in CI builds.  This can result in breakage when the toolchain is
+      #   updated, but that should only happen with a change to the repo, which
+      #   gives us an opportunity to find and fix any newly-introduced warnings.
+      # - `--locked`: do not update Cargo.lock when building.  Checking in
+      #   Cargo.lock ensures that everyone is using the same dependencies and
+      #   also gives us a record of which dependencies were used for each CI
+      #   run.  Building with `--locked` ensures that the checked-in Cargo.lock
+      #   is up to date.
+      # - TMPDIR=$OMICRON_TMP: we specify a specific temporary directory so that
+      #   failed test outputs will be in a known place that we can grab at the
+      #   end without also grabbing random other temporary files.
+      run: |
+        TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" \
+          cargo build --locked --all-targets --verbose
+    # Use the same RUSTFLAGS and RUSTDOCFLAGS as above to avoid a rebuild.
+    # Put "./cockroachdb/bin" and "./clickhouse" on the PATH.
+    - name: Run tests with cargo nextest
+      run: |
+        TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" \
+          cargo nextest run --profile ci --workspace --locked --config-file .cargo/nextest.toml
+    - name: Archive nextest JUnit output
+      # actions/upload-artifact@v2.3.1
+      uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2
+      with:
+        name: junit_${{ runner.os }}
+        retention-days: 90
+        path: target/nextest/ci/junit.xml
diff --git a/tools/ci_clear_space b/tools/ci_clear_space
new file mode 100755
index 0000000000..8527cf08e1
--- /dev/null
+++ b/tools/ci_clear_space
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+echo "Disk space:"
+df -h
+
+if [ -d "/usr/share/dotnet" ]; then
+  echo "Removing dotnet"
+  sudo rm -rf /usr/share/dotnet
+fi
+if [ -d "/usr/local/lib/android" ]; then
+  echo "Removing android"
+  sudo rm -rf /usr/local/lib/android
+fi
+if [ -d "/opt/ghc" ]; then
+  echo "Removing haskell"
+  sudo rm -rf /opt/ghc
+fi
+
+echo "Disk space:"
+df -h
\ No newline at end of file

From 937a4753de4ad6abf41c1bc66ce17c2f29074a82 Mon Sep 17 00:00:00 2001
From: David Crespo <david.crespo@oxidecomputer.com>
Date: Mon, 18 Apr 2022 22:17:03 -0500
Subject: [PATCH 15/15] attempt to add buildomat nextest run too

---
 .github/buildomat/jobs/build-and-nextest.sh | 61 +++++++++++++++++++++
 .github/buildomat/jobs/build-and-test.sh    |  2 +-
 2 files changed, 62 insertions(+), 1 deletion(-)
 create mode 100644 .github/buildomat/jobs/build-and-nextest.sh

diff --git a/.github/buildomat/jobs/build-and-nextest.sh b/.github/buildomat/jobs/build-and-nextest.sh
new file mode 100644
index 0000000000..b66124ee7b
--- /dev/null
+++ b/.github/buildomat/jobs/build-and-nextest.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+#:
+#: name = "helios / build-and-test"
+#: variety = "basic"
+#: target = "helios"
+#: rust_toolchain = "nightly-2021-11-24"
+#: output_rules = []
+#:
+
+set -o errexit
+set -o pipefail
+set -o xtrace
+
+cargo --version
+rustc --version
+
+banner clickhouse
+ptime -m ./tools/ci_download_clickhouse
+
+banner cockroach
+ptime -m bash ./tools/ci_download_cockroachdb
+
+#
+# Put "./cockroachdb/bin" and "./clickhouse" on the PATH for the test
+# suite.
+#
+export PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse"
+
+ptime -m cargo install cargo-nextest
+
+#
+# We build with:
+#
+# - RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings": disallow warnings
+#   in CI builds.  This can result in breakage when the toolchain is
+#   updated, but that should only happen with a change to the repo, which
+#   gives us an opportunity to find and fix any newly-introduced warnings.
+#
+# - `--locked`: do not update Cargo.lock when building.  Checking in
+#   Cargo.lock ensures that everyone is using the same dependencies and
+#   also gives us a record of which dependencies were used for each CI
+#   run.  Building with `--locked` ensures that the checked-in Cargo.lock
+#   is up to date.
+#
+banner build
+export RUSTFLAGS="-D warnings"
+export RUSTDOCFLAGS="-D warnings"
+ptime -m cargo +'nightly-2021-11-24' build --locked --all-targets --verbose
+
+#
+# Check that building individual packages as when deploying Omicron succeeds
+#
+banner deploy-check
+ptime -m cargo run --bin omicron-package -- check
+
+#
+# NOTE: We're using using the same RUSTFLAGS and RUSTDOCFLAGS as above to avoid
+# having to rebuild here.
+#
+banner test
+ptime -m cargo +'nightly-2021-11-24' nextest run --profile ci --workspace --locked --config-file .cargo/nextest.toml
diff --git a/.github/buildomat/jobs/build-and-test.sh b/.github/buildomat/jobs/build-and-test.sh
index 2d204872a9..8d001ca164 100644
--- a/.github/buildomat/jobs/build-and-test.sh
+++ b/.github/buildomat/jobs/build-and-test.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 #:
-#: name = "helios / build-and-test"
+#: name = "helios / build-and-nextest"
 #: variety = "basic"
 #: target = "helios"
 #: rust_toolchain = "nightly-2021-11-24"