Run cargo nextest in addition to cargo test in CI

.cargo/nextest.toml

@@ -0,0 +1,6 @@
+[profile.ci]
+fail-fast = false
+failure-output = "immediate-final"
+
+[profile.ci.junit]
+path = "junit.xml"

.github/buildomat/jobs/build-and-nextest.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+#:
+#: name = "helios / build-and-test"
+#: variety = "basic"
+#: target = "helios"
+#: rust_toolchain = "nightly-2021-11-24"
+#: output_rules = []
+#:
+
+set -o errexit
+set -o pipefail
+set -o xtrace
+
+cargo --version
+rustc --version
+
+banner clickhouse
+ptime -m ./tools/ci_download_clickhouse
+
+banner cockroach
+ptime -m bash ./tools/ci_download_cockroachdb
+
+#
+# Put "./cockroachdb/bin" and "./clickhouse" on the PATH for the test
+# suite.
+#
+export PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse"
+
+ptime -m cargo install cargo-nextest
+
+#
+# We build with:
+#
+# - RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings": disallow warnings
+#   in CI builds.  This can result in breakage when the toolchain is
+#   updated, but that should only happen with a change to the repo, which
+#   gives us an opportunity to find and fix any newly-introduced warnings.
+#
+# - `--locked`: do not update Cargo.lock when building.  Checking in
+#   Cargo.lock ensures that everyone is using the same dependencies and
+#   also gives us a record of which dependencies were used for each CI
+#   run.  Building with `--locked` ensures that the checked-in Cargo.lock
+#   is up to date.
+#
+banner build
+export RUSTFLAGS="-D warnings"
+export RUSTDOCFLAGS="-D warnings"
+ptime -m cargo +'nightly-2021-11-24' build --locked --all-targets --verbose
+
+#
+# Check that building individual packages as when deploying Omicron succeeds
+#
+banner deploy-check
+ptime -m cargo run --bin omicron-package -- check
+
+#
+# NOTE: We're using using the same RUSTFLAGS and RUSTDOCFLAGS as above to avoid
+# having to rebuild here.
+#
+banner test
+ptime -m cargo +'nightly-2021-11-24' nextest run --profile ci --workspace --locked --config-file .cargo/nextest.toml

.github/buildomat/jobs/build-and-test.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #:
-#: name = "helios / build-and-test"
+#: name = "helios / build-and-nextest"
 #: variety = "basic"
 #: target = "helios"
 #: rust_toolchain = "nightly-2021-11-24"

.github/workflows/rust.yml

@@ -105,25 +105,7 @@ jobs:
     - name: Report cargo version
       run: cargo --version
     - name: Remove unnecessary software
-      run: |
-        echo "Disk space:"
-        df -h
-
-        if [ -d "/usr/share/dotnet" ]; then
-          echo "Removing dotnet"
-          sudo rm -rf /usr/share/dotnet
-        fi
-        if [ -d "/usr/local/lib/android" ]; then
-          echo "Removing android"
-          sudo rm -rf /usr/local/lib/android
-        fi
-        if [ -d "/opt/ghc" ]; then
-          echo "Removing haskell"
-          sudo rm -rf /opt/ghc
-        fi
-
-        echo "Disk space:"
-        df -h
+      run: ./tools/ci_clear_space
     - name: Configure GitHub cache for CockroachDB binaries
       id: cache-cockroachdb
       # actions/[email protected]
@@ -160,13 +142,16 @@ jobs:
       # - TMPDIR=$OMICRON_TMP: we specify a specific temporary directory so that
       #   failed test outputs will be in a known place that we can grab at the
       #   end without also grabbing random other temporary files.
-      run: TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo build --locked --all-targets --verbose
-    - name: Run tests
-      # Use the same RUSTFLAGS and RUSTDOCFLAGS as above to avoid having to
-      # rebuild here.
-      # Put "./cockroachdb/bin" and "./clickhouse" on the PATH for the test
-      # suite.
-      run: TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo test --no-fail-fast --workspace --locked --verbose
+      run: |
+        TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" \
+          cargo build --locked --all-targets --verbose
+
+    # Use the same RUSTFLAGS and RUSTDOCFLAGS as above to avoid a rebuild.
+    # Put "./cockroachdb/bin" and "./clickhouse" on the PATH.
+    - name: Run tests with cargo test
+      run: |
+        TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" \
+          cargo test --no-fail-fast --workspace --locked --verbose
     - name: Archive any failed test results
       if: ${{ failure() }}
       # actions/[email protected]
@@ -178,3 +163,78 @@ jobs:
           ${{ env.OMICRON_TMP }}
           !${{ env.OMICRON_TMP }}/crdb-base
           !${{ env.OMICRON_TMP }}/rustc*
+
+  # Copy of build-and-test. Only difference is it uses cargo-nextest instead of
+  # cargo test and archives JUnit output instead of all of OMICRON_TMP
+  build-and-nextest:
+    env:
+      OMICRON_TMP: /tmp/omicron_tmp
+    needs: skip_duplicate_jobs
+    if: ${{ needs.skip_duplicate_jobs.outputs.should_skip != 'true' }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-18.04, macos-11 ]
+    steps:
+    # actions/checkout@v2
+    - uses: actions/checkout@28c7f3d2b5162b5ddd3dfd9a45aa55eaf396478b
+    - uses: Swatinem/rust-cache@v1
+      if: ${{ github.ref != 'refs/heads/main' }}
+    - uses: taiki-e/install-action@nextest
+    - name: Report cargo version
+      run: cargo --version
+    - name: Remove unnecessary software
+      run: ./tools/ci_clear_space
+    - name: Configure GitHub cache for CockroachDB binaries
+      id: cache-cockroachdb
+      # actions/[email protected]
+      uses: actions/cache@26968a09c0ea4f3e233fdddbafd1166051a095f6
+      with:
+        key: ${{ runner.os }}-cockroach-binary-${{ hashFiles('tools/cockroachdb_checksums') }}
+        path: "./out/cockroachdb"
+    - name: Configure GitHub cache for ClickHouse binaries
+      id: cache-clickhouse
+      # actions/[email protected]
+      uses: actions/cache@26968a09c0ea4f3e233fdddbafd1166051a095f6
+      with:
+        key: ${{ runner.os }}-clickhouse-binary-${{ hashFiles('tools/clickhouse_checksums') }}
+        path: "./out/clickhouse"
+    - name: Download ClickHouse
+      if: steps.cache-clickhouse.outputs.cache-hit != 'true'
+      run: ./tools/ci_download_clickhouse
+    - name: Download CockroachDB binary
+      if: steps.cache-cockroachdb.outputs.cache-hit != 'true'
+      run: bash ./tools/ci_download_cockroachdb
+    - name: Create temporary directory for test outputs
+      run: mkdir -p $OMICRON_TMP
+    - name: Build
+      # We build with:
+      # - RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings": disallow warnings
+      #   in CI builds.  This can result in breakage when the toolchain is
+      #   updated, but that should only happen with a change to the repo, which
+      #   gives us an opportunity to find and fix any newly-introduced warnings.
+      # - `--locked`: do not update Cargo.lock when building.  Checking in
+      #   Cargo.lock ensures that everyone is using the same dependencies and
+      #   also gives us a record of which dependencies were used for each CI
+      #   run.  Building with `--locked` ensures that the checked-in Cargo.lock
+      #   is up to date.
+      # - TMPDIR=$OMICRON_TMP: we specify a specific temporary directory so that
+      #   failed test outputs will be in a known place that we can grab at the
+      #   end without also grabbing random other temporary files.
+      run: |
+        TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" \
+          cargo build --locked --all-targets --verbose
+    # Use the same RUSTFLAGS and RUSTDOCFLAGS as above to avoid a rebuild.
+    # Put "./cockroachdb/bin" and "./clickhouse" on the PATH.
+    - name: Run tests with cargo nextest
+      run: |
+        TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" \
+          cargo nextest run --profile ci --workspace --locked --config-file .cargo/nextest.toml
+    - name: Archive nextest JUnit output
+      # actions/[email protected]
+      uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2
+      with:
+        name: junit_${{ runner.os }}
+        retention-days: 90
+        path: target/nextest/ci/junit.xml

nexus/tests/integration_tests/console_api.rs

@@ -54,7 +54,7 @@ async fn test_sessions(cptestctx: &ControlPlaneTestContext) {
         .expect("failed to 401 on unauthed API request");
 
     // console pages don't 401, they 302
-    RequestBuilder::new(&testctx, Method::GET, "/orgs/whatever")
+    RequestBuilder::new(&testctx, Method::GET, "/orgs/blah")
         .expect_status(Some(StatusCode::FOUND))
         .execute()
         .await
@@ -70,7 +70,7 @@ async fn test_sessions(cptestctx: &ControlPlaneTestContext) {
         .await
         .expect("failed to create org with session cookie");
 
-    RequestBuilder::new(&testctx, Method::GET, "/orgs/whatever")
+    RequestBuilder::new(&testctx, Method::GET, "/orgs/blah")
         .header(header::COOKIE, &session_token)
         .expect_status(Some(StatusCode::OK))
         .execute()
@@ -100,7 +100,7 @@ async fn test_sessions(cptestctx: &ControlPlaneTestContext) {
         .await
         .expect("failed to get 401 for unauthed API request");
 
-    RequestBuilder::new(&testctx, Method::GET, "/orgs/whatever")
+    RequestBuilder::new(&testctx, Method::GET, "/orgs/blah")
         .header(header::COOKIE, &session_token)
         .expect_status(Some(StatusCode::FOUND))
         .execute()

tools/ci_clear_space

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+echo "Disk space:"
+df -h
+
+if [ -d "/usr/share/dotnet" ]; then
+  echo "Removing dotnet"
+  sudo rm -rf /usr/share/dotnet
+fi
+if [ -d "/usr/local/lib/android" ]; then
+  echo "Removing android"
+  sudo rm -rf /usr/local/lib/android
+fi
+if [ -d "/opt/ghc" ]; then
+  echo "Removing haskell"
+  sudo rm -rf /opt/ghc
+fi
+
+echo "Disk space:"
+df -h