feat(test): Improve kurtosis workflow

.github/assets/kurtosis_op_network_params_remote.yaml

@@ -9,13 +9,13 @@ optimism_package:
       - el_type: op-geth
         el_image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-geth:latest
         cl_type: op-node
-        cl_image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:develop
+        cl_image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:latest
         cl_extra_params:
           - "--l1.trustrpc=true"
       - el_type: op-reth
         el_image: "ghcr.io/paradigmxyz/op-reth:kurtosis-ci"
         cl_type: op-node
-        cl_image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:develop
+        cl_image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:latest
         cl_extra_params:
           - "--l1.trustrpc=true"
       batcher_params:

.github/assets/kurtosis_op_network_params_remote_no_op_geth.yaml

@@ -8,7 +8,7 @@ optimism_package:
       - el_type: op-reth
         el_image: "ghcr.io/paradigmxyz/op-reth:kurtosis-ci"
         cl_type: op-node
-        cl_image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:develop
+        cl_image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:latest
         cl_extra_params:
           - "--l1.trustrpc=true"
       batcher_params:

.github/assets/kurtosis_op_network_params_remote_no_op_reth.yaml

@@ -8,7 +8,7 @@ optimism_package:
       - el_type: op-geth
         el_image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-geth:latest
         cl_type: op-node
-        cl_image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:develop
+        cl_image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:latest
         cl_extra_params:
           - "--l1.trustrpc=true"
       batcher_params:

.github/assets/kurtosis_op_network_params_remote_op_reth_sequencer.yaml

@@ -7,15 +7,15 @@ optimism_package:
     - participants:
       # first participant is sequencer
       - el_type: op-reth
-        el_image: "op-reth:latest"
+        el_image: "ghcr.io/paradigmxyz/op-reth:kurtosis-ci"
         cl_type: op-node
-        cl_image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:develop
+        cl_image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:latest
         cl_extra_params:
           - "--l1.trustrpc=true"
       - el_type: op-geth
         el_image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-geth:latest
         cl_type: op-node
-        cl_image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:develop
+        cl_image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:latest
         cl_extra_params:
           - "--l1.trustrpc=true"
       batcher_params:

.github/scripts/kurtosis-check-enclave-block-creation.sh

@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+
+# 
+# 
+# This is a test script used mostly in the CI pipeline
+# 
+# It will search for all EL clients in a specified kurtosis enclave
+# and will check whether their blocks advance in a reasonable amount of time
+# 
+# Usage:
+# 
+# ./kurtosis-check-enclave-block-creation.sh <ENCLAVE_NAME> [TARGET_BLOCK_NUMBER] [NUMBER_OF_NAPS]
+# 
+# 
+
+# Just so that we don't repeat ourselves
+ENCLAVES_API_URL=http://127.0.0.1:9779/api/enclaves
+
+# We pass the enclave name and the target block as script arguments
+ENCLAVE_NAME=$1
+if [ -z "$ENCLAVE_NAME" ]; then
+    echo "Please specify the enclave name as the first parameter"
+    exit 1
+fi
+
+# We default the target block to a good round 100
+TARGET_BLOCK=$2
+if [ -z "$TARGET_BLOCK" ]; then
+    TARGET_BLOCK=100
+fi
+
+# The number of sleeps the script will take
+NUM_NAPS=$3
+if [ -z "$NUM_NAPS" ]; then
+    NUM_NAPS=100
+fi
+
+echo "Checking whether EL clients in enclave $ENCLAVE_NAME reach target block $TARGET_BLOCK in reasonable time ($NUM_NAPS of 5s)"
+
+# Get the enclave UUID from the API
+# 
+# The API reponse is a JSON object with UUIDs as keys. To get the UUID we need to find a value
+# with a matching "name" property.
+# 
+# The "| values" at the end of the jq transformation will convert null to an empty string
+ENCLAVE_ID=$(curl -s $ENCLAVES_API_URL | jq -r 'to_entries | map(select(.value.name == "'$ENCLAVE_NAME'")) | .[0].key | values')
+
+# Make sure we got something
+if [ -z "$ENCLAVE_ID" ]; then
+    echo "No enclave found for enclave $ENCLAVE_NAME"
+    exit 1
+fi
+
+echo "Got enclave UUID: $ENCLAVE_ID"
+
+# Now get all the EL client services
+# 
+# We assume the convention is to name them op-el-xxx
+ENCLAVE_SERVICES=$(curl -s $ENCLAVES_API_URL/$ENCLAVE_ID/services)
+ENCLAVE_EL_SERVICES=$(echo $ENCLAVE_SERVICES | jq -r 'to_entries | map(select(.key | startswith("op-el-"))) | map(.value)')
+
+# Now get the EL client names and RPC ports and arrange them into single-line space-delimited strings
+# 
+# This is useful for bash iteration below
+ENCLAVE_EL_SERVICE_NAMES=$(echo $ENCLAVE_EL_SERVICES | jq -r 'map(.name) | join(" ")')
+ENCLAVE_EL_SERVICE_RPC_PORTS=$(echo $ENCLAVE_EL_SERVICES | jq -r 'map(.public_ports.rpc.number) | join(" ")')
+
+# Make sure we got something
+if [ -z "$ENCLAVE_EL_SERVICE_NAMES" ]; then
+    echo "No EL clients found for enclave $ENCLAVE_NAME"
+    exit 1
+fi
+
+echo "Got enclave EL services: $ENCLAVE_EL_SERVICE_NAMES"
+echo "Got enclave EL RPC ports: $ENCLAVE_EL_SERVICE_RPC_PORTS"
+
+# Convert the lists into bash arrays
+ENCLAVE_EL_SERVICE_NAMES_ARRAY=($ENCLAVE_EL_SERVICE_NAMES)
+ENCLAVE_EL_SERVICE_RPC_PORTS_ARRAY=($ENCLAVE_EL_SERVICE_RPC_PORTS)
+
+# Now check that the clients advance
+for STEP in $(seq 1 $NUM_NAPS); do
+    echo "Check $STEP/$NUM_NAPS"
+
+    # Iterate over the names/RPC ports arrays
+    for I in "${!ENCLAVE_EL_SERVICE_NAMES_ARRAY[@]}"; do
+        SERVICE_NAME=${ENCLAVE_EL_SERVICE_NAMES_ARRAY[$I]}
+        SERVICE_RPC_PORT=${ENCLAVE_EL_SERVICE_RPC_PORTS_ARRAY[$I]}
+        SERVICE_RPC_URL="http://127.0.0.1:$SERVICE_RPC_PORT"
+
+        BLOCK_NUMBER=$(cast bn --rpc-url $SERVICE_RPC_URL)
+        echo "  Got block for $SERVICE_NAME: $BLOCK_NUMBER"
+
+        # Check whether we reached the target block
+        if [ "$BLOCK_NUMBER" -gt "$TARGET_BLOCK" ]; then
+            echo "  Target block $TARGET_BLOCK reached for $SERVICE_NAME"
+
+            # If so, we remove the service from the array
+            unset ENCLAVE_EL_SERVICE_NAMES_ARRAY[$I]
+            unset ENCLAVE_EL_SERVICE_RPC_PORTS_ARRAY[$I]
+        fi
+    done
+
+    # Since we used unset, we need to reindex the arrays, we need to remove any gaps left behind
+    ENCLAVE_EL_SERVICE_NAMES_ARRAY=("${ENCLAVE_EL_SERVICE_NAMES_ARRAY[@]}")
+    ENCLAVE_EL_SERVICE_RPC_PORTS_ARRAY=("${ENCLAVE_EL_SERVICE_RPC_PORTS_ARRAY[@]}")
+
+    # Now we check whether the arrays are empty
+    # 
+    # This means all target blocks have been reached and we can exit fine
+    if [ ${#ENCLAVE_EL_SERVICE_NAMES_ARRAY[@]} -eq 0 ]; then
+        echo "All target blocks have been reached. Exiting."
+        exit 0
+    fi
+
+    sleep 5
+done
+
+echo "Target blocks have not been reached for the following services:"
+
+for I in "${!ENCLAVE_EL_SERVICE_NAMES_ARRAY[@]}"; do
+    SERVICE_NAME=${ENCLAVE_EL_SERVICE_NAMES_ARRAY[$I]}
+
+    echo "  $SERVICE_NAME"
+done
+
+exit 1

.github/scripts/kurtosis-dump-logs.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+# 
+# 
+# This is a log script used mostly in the CI pipeline
+# 
+# It will search for all OP clients in a specified kurtosis enclave
+# and will dump their logs in a CI-friendly manner
+# 
+# Usage:
+# 
+# ./kurtosis-dump-logs.sh <ENCLAVE_NAME>
+# 
+# 
+
+# Just so that we don't repeat ourselves
+ENCLAVES_API_URL=http://127.0.0.1:9779/api/enclaves
+
+# We pass the enclave name and the target block as script arguments
+ENCLAVE_NAME=$1
+if [ -z "$ENCLAVE_NAME" ]; then
+    echo "Please specify the enclave name as the first parameter"
+    exit 1
+fi
+
+echo "Dumping logs of the OP services for enclave $ENCLAVE_NAME"
+
+# Get the enclave UUID from the API
+# 
+# The | values at the end of the j1 transformation will convert null to an empty string
+ENCLAVE_ID=$(curl -s $ENCLAVES_API_URL | jq -r 'to_entries | map(select(.value.name == "'$ENCLAVE_NAME'")) | .[0].key | values')
+
+# Make sure we got something
+if [ -z "$ENCLAVE_ID" ]; then
+    echo "No enclave found for enclave $ENCLAVE_NAME"
+    exit 1
+fi
+
+echo "Got enclave UUID: $ENCLAVE_ID"
+
+# Now get all the OP client services
+# 
+# We assume the convention is to name them op-xxx
+ENCLAVE_SERVICES=$(curl -s $ENCLAVES_API_URL/$ENCLAVE_ID/services)
+ENCLAVE_OP_SERVICES=$(echo $ENCLAVE_SERVICES | jq -r 'to_entries | map(select(.key | startswith("op-"))) | map(.value)')
+
+# Now get the OP client names and arrange them into a single-line space-delimited string
+# 
+# This is useful for bash iteration below
+ENCLAVE_OP_SERVICE_NAMES=$(echo $ENCLAVE_OP_SERVICES | jq -r 'map(.name) | join(" ")')
+
+# Make sure we got something
+if [ -z "$ENCLAVE_OP_SERVICE_NAMES" ]; then
+    echo "No OP clients found for enclave $ENCLAVE_NAME"
+    exit 1
+fi
+
+echo "Got enclave OP services: $ENCLAVE_OP_SERVICE_NAMES"
+
+# Convert the list into a bash array
+ENCLAVE_OP_SERVICE_NAMES_ARRAY=($ENCLAVE_OP_SERVICE_NAMES)
+
+# Iterate over the names/RPC ports arrays
+for SERVICE_NAME in "${ENCLAVE_OP_SERVICE_NAMES_ARRAY[@]}"; do
+    # We use the github actions grouping to get a nicer output
+    echo "::group::$SERVICE_NAME"
+    kurtosis service logs -a $ENCLAVE_NAME $SERVICE_NAME
+    echo "::endgroup::"
+done

.github/workflows/kurtosis-op-local.yml

@@ -1,10 +1,26 @@
 # Runs simple OP stack setup in Kurtosis
 
-name: Run kurtosis (local images)
+name: Run kurtosis
+
+run-name: Run kurtosis (${{ inputs.args }})
 
 on:
   workflow_dispatch:
     inputs:
+      args:
+        type: choice
+        required: true
+        description: "Kurtosis args file to use"
+        options:
+          - kurtosis_op_network_params_local_no_op_geth
+          - kurtosis_op_network_params_local_no_op_reth
+          - kurtosis_op_network_params_local_op_reth_sequencer
+          - kurtosis_op_network_params_local
+          - kurtosis_op_network_params_remote_no_op_geth
+          - kurtosis_op_network_params_remote_no_op_reth
+          - kurtosis_op_network_params_remote_op_reth_sequencer
+          - kurtosis_op_network_params_remote
+        default: kurtosis_op_network_params_local
       op-node-repo:
         type: string
         required: true
@@ -66,16 +82,18 @@ jobs:
       cargo_features: optimism,asm-keccak
       cargo_package: crates/optimism/bin/Cargo.toml
 
-  test-op-geth:
+  test:
     timeout-minutes: 60
     strategy:
       fail-fast: false
-    name: Kurtosis with op-geth
+    name: Kurtosis
     runs-on: ubuntu-latest
     needs:
       - prepare-op-reth
       - prepare-op-geth
       - prepare-op-node
+    env:
+      ENCLAVE_NAME: op-devnet
     steps:
       - uses: actions/checkout@v4
         with:
@@ -102,91 +120,11 @@ jobs:
           docker image ls -a
 
       - name: Run kurtosis
-        run: |
-          kurtosis run --enclave op-devnet github.com/ethpandaops/optimism-package --args-file .github/assets/kurtosis_op_network_params_local.yaml
-          ENCLAVE_ID=$(curl http://127.0.0.1:9779/api/enclaves | jq --raw-output 'keys[0]')
-          GETH_PORT=$(curl "http://127.0.0.1:9779/api/enclaves/$ENCLAVE_ID/services" | jq '."op-el-1-op-geth-op-node-op-kurtosis".public_ports.rpc.number')
-          RETH_PORT=$(curl "http://127.0.0.1:9779/api/enclaves/$ENCLAVE_ID/services" | jq '."op-el-2-op-reth-op-node-op-kurtosis".public_ports.rpc.number')
-          echo "GETH_RPC=http://127.0.0.1:$GETH_PORT" >> $GITHUB_ENV
-          echo "RETH_RPC=http://127.0.0.1:$RETH_PORT" >> $GITHUB_ENV
+        run: kurtosis run --enclave $ENCLAVE_NAME github.com/ethpandaops/optimism-package --args-file .github/assets/${{ inputs.args }}.yaml
 
       - name: Assert that clients advance
-        run: |
-          for i in {1..100}; do
-            sleep 5
-            BLOCK_GETH=$(cast bn --rpc-url $GETH_RPC)
-            BLOCK_RETH=$(cast bn --rpc-url $RETH_RPC)
-
-            if [ $BLOCK_GETH -ge 100 ] && [ $BLOCK_RETH -ge 100 ] ; then exit 0; fi
-            echo "Waiting for clients to advance..., Reth: $BLOCK_RETH Geth: $BLOCK_GETH"
-          done
-          kurtosis service logs -a op-devnet op-el-2-op-reth-op-node-op-kurtosis
-          kurtosis service logs -a op-devnet op-cl-2-op-node-op-reth-op-kurtosis
-          exit 1
-
-  test-no-op-geth:
-    timeout-minutes: 60
-    strategy:
-      fail-fast: false
-    name: Kurtosis without op-geth
-    runs-on: ubuntu-latest
-    needs:
-      - prepare-op-reth
-      - prepare-op-node
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Setup environment
-        uses: ./.github/actions/setup-environment
+        run: ./.github/scripts/kurtosis-check-enclave-block-creation.sh $ENCLAVE_NAME
 
-      - name: Download docker image artifacts
-        uses: actions/download-artifact@v4
-        with:
-          pattern: artifacts--*
-          merge-multiple: true
-          path: /tmp
-
-      - name: Load Docker images
-        run: |
-          # Load all images from artifacts
-          docker load -i /tmp/op_reth_image.tar
-          docker load -i /tmp/op_node_image.tar
-
-          # List available images
-          docker image ls -a
-
-      - name: Run kurtosis
-        run: |
-          kurtosis run --enclave op-devnet github.com/ethpandaops/optimism-package --args-file .github/assets/kurtosis_op_network_params_local_no_op_geth.yaml
-          ENCLAVE_ID=$(curl http://127.0.0.1:9779/api/enclaves | jq --raw-output 'keys[0]')
-          RETH_PORT=$(curl "http://127.0.0.1:9779/api/enclaves/$ENCLAVE_ID/services" | jq '."op-el-1-op-reth-op-node-op-kurtosis".public_ports.rpc.number')
-          echo "RETH_RPC=http://127.0.0.1:$RETH_PORT" >> $GITHUB_ENV
-
-      - name: Assert that clients advance
-        run: |
-          for i in {1..100}; do
-            sleep 5
-            BLOCK_RETH=$(cast bn --rpc-url $RETH_RPC)
-
-            if [ $BLOCK_RETH -ge 100 ] ; then exit 0; fi
-            echo "Waiting for clients to advance..., Reth: $BLOCK_RETH"
-          done
-          kurtosis service logs -a op-devnet op-el-1-op-reth-op-node-op-kurtosis
-          kurtosis service logs -a op-devnet op-cl-1-op-node-op-reth-op-kurtosis
-          exit 1
-
-  notify-on-error:
-    needs: 
-      - test-op-geth
-      - test-no-op-geth
-    if: failure()
-    runs-on: ubuntu-latest
-    steps:
-      - name: Slack Webhook Action
-        uses: rtCamp/action-slack-notify@v2
-        env:
-          SLACK_COLOR: ${{ job.status }}
-          SLACK_MESSAGE: "Failed run: https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}"
-          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+      - name: Dump kurtosis logs on failure
+        if: ${{ failure() }}
+        run: ./.github/scripts/kurtosis-dump-logs.sh $ENCLAVE_NAME