From 05e9afcd75c536aa5e42003b78a9ee347ff9c082 Mon Sep 17 00:00:00 2001
From: Kor Nielsen <kor@google.com>
Date: Wed, 22 Nov 2023 12:52:37 -0800
Subject: [PATCH] release: Run tests under sw-emulator before releasing.

This will run all combinations of itrng/etrng and logging/no-logging.
---
 .github/workflows/fw-test-emu.yml     | 91 +++++++++++++++++++++++++++
 .github/workflows/nightly-release.yml | 50 ++++++++++++++-
 2 files changed, 140 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/fw-test-emu.yml

diff --git a/.github/workflows/fw-test-emu.yml b/.github/workflows/fw-test-emu.yml
new file mode 100644
index 0000000000..7723a59140
--- /dev/null
+++ b/.github/workflows/fw-test-emu.yml
@@ -0,0 +1,91 @@
+
+name: Build and Test Firmware (in emulator)
+
+on:
+  workflow_call:
+    inputs:
+      artifact-suffix:
+        type: string
+        required: false
+      extra-features:
+        default:
+        type: string
+      rom-logging:
+        default: true
+        type: boolean
+
+jobs:
+  build_and_test:
+    runs-on: ubuntu-22.04
+
+    env:
+      NEXTEST_VERSION: 0.9.63
+      CACHE_BUSTER: f7c64774f17c
+
+    steps:
+      - name: Restore cargo-nextest binary
+        uses: actions/cache/restore@v3
+        id: nextest_bin_restore
+        with:
+          path: ~/.cargo/bin/cargo-nextest
+          key: nextest-bin-${{ env.NEXTEST_VERSION}}-${{ env.CACHE_BUSTER }}
+
+      - name: Install cargo-nextest
+        if: steps.nextest_bin_restore.outputs.cache-hit != 'true'
+        run: |
+          cargo install cargo-nextest --version ${NEXTEST_VERSION} --locked --no-default-features --features=default-no-update
+
+      - name: Save cargo-nextest binary
+        uses: actions/cache/save@v3
+        if: steps.nextest_bin_restore.outputs.cache-hit != 'true'
+        with:
+          path: ~/.cargo/bin/cargo-nextest
+          key: ${{ steps.nextest_bin_restore.outputs.cache-primary-key }}
+
+      - name: Checkout repo
+        uses: actions/checkout@v3
+
+      - name: Pull dpe submodule
+        run: |
+          git submodule update --init dpe
+
+      - name: Build firmware
+        run: |
+          mkdir /tmp/caliptra-test-firmware
+          cargo run -p caliptra-builder -- --all_elfs /tmp/caliptra-test-firmware
+
+      - name: Run tests
+        run: |
+          export CALIPTRA_PREBUILT_FW_DIR=/tmp/caliptra-test-firmware
+
+          if [ "${{ inputs.rom-logging }}" == "true" ] || [ -z "${{ inputs.rom-logging }}" ]; then
+            export CPTRA_ROM_TYPE=ROM_WITH_UART
+          elif [ "${{ inputs.rom-logging }}" == false ]; then
+            export CPTRA_ROM_TYPE=ROM_WITHOUT_UART
+          else
+            echo "Unexpected inputs.rom-logging: ${{ inputs.rom-logging }}"
+            exit 1
+          fi
+
+          # Workaround https://github.com/nextest-rs/nextest/issues/267
+          export LD_LIBRARY_PATH=$(rustc --print sysroot)/lib
+
+          cargo-nextest nextest list \
+            --features="${{ inputs.extra-features }}" \
+            --message-format json \
+            > /tmp/nextest-list.json
+
+          cargo-nextest nextest run \
+            --features="${{ inputs.extra-features }}" \
+            --no-fail-fast \
+            --profile=nightly
+
+      - name: 'Upload test results'
+        uses: actions/upload-artifact@v3
+        if: success() || failure()
+        with:
+          name: caliptra-test-results${{ inputs.artifact-suffix }}
+          path: |
+            /tmp/junit.xml
+            /tmp/nextest-list.json
+
diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml
index acebc1c47f..ac135113bf 100644
--- a/.github/workflows/nightly-release.yml
+++ b/.github/workflows/nightly-release.yml
@@ -80,9 +80,57 @@ jobs:
       extra-features: slow_tests
       rom-logging: false
 
+  sw-emulator-full-suite-etrng-log:
+    name: sw-emulator Suite (etrng, log)
+    needs: find-latest-release
+    if: needs.find-latest-release.outputs.create_release
+    uses: ./.github/workflows/fw-test-emu.yml
+    with:
+      artifact-suffix: -sw-emulator-realtime-etrng-log
+      extra-features: slow_tests
+      rom-logging: true
+
+  sw-emulator-full-suite-etrng-nolog:
+    name: sw-emulator Suite (etrng, nolog)
+    needs: find-latest-release
+    if: needs.find-latest-release.outputs.create_release
+    uses: ./.github/workflows/fw-test-emu.yml
+    with:
+      artifact-suffix: -sw-emulator-realtime-etrng-nolog
+      extra-features: slow_tests
+      rom-logging: false
+
+  sw-emulator-full-suite-itrng-log:
+    name: sw-emulator Suite (itrng, log)
+    needs: find-latest-release
+    if: needs.find-latest-release.outputs.create_release
+    uses: ./.github/workflows/fw-test-emu.yml
+    with:
+      artifact-suffix: -sw-emulator-realtime-itrng-log
+      extra-features: slow_tests,itrng
+      rom-logging: true
+
+  sw-emulator-full-suite-itrng-nolog:
+    name: sw-emulator Suite (itrng, nolog)
+    needs: find-latest-release
+    if: needs.find-latest-release.outputs.create_release
+    uses: ./.github/workflows/fw-test-emu.yml
+    with:
+      artifact-suffix: -sw-emulator-realtime-itrng-nolog
+      extra-features: slow_tests,itrng
+      rom-logging: false
+
   create-release:
     name: Create New Release
-    needs: [find-latest-release, fpga-full-suite-etrng-log, fpga-full-suite-etrng-nolog]
+    needs:
+      - find-latest-release
+      - fpga-full-suite-etrng-log
+      - fpga-full-suite-etrng-nolog
+      - sw-emulator-full-suite-etrng-log
+      - sw-emulator-full-suite-etrng-nolog
+      - sw-emulator-full-suite-itrng-log
+      - sw-emulator-full-suite-itrng-nolog
+
     runs-on: ubuntu-22.04
 
     steps: