ci: re-enable hive tests

.github/assets/hive/expected_failures.yaml

@@ -0,0 +1,128 @@
+# https://github.com/paradigmxyz/reth/issues/7015
+# https://github.com/paradigmxyz/reth/issues/6332
+rpc-compat:
+  - debug_getRawBlock/get-invalid-number (reth)
+  - debug_getRawHeader/get-invalid-number (reth)
+  - debug_getRawReceipts/get-invalid-number (reth)
+  - debug_getRawTransaction/get-invalid-hash (reth)
+
+  - eth_call/call-callenv (reth)
+  - eth_createAccessList/create-al-contract-eip1559 (reth)
+  - eth_createAccessList/create-al-contract (reth)
+  - eth_feeHistory/fee-history (reth)
+  - eth_getStorageAt/get-storage-invalid-key-too-large (reth)
+  - eth_getStorageAt/get-storage-invalid-key (reth)
+  - eth_getTransactionReceipt/get-access-list (reth)
+  - eth_getTransactionReceipt/get-blob-tx (reth)
+  - eth_getTransactionReceipt/get-dynamic-fee (reth)
+
+# https://github.com/paradigmxyz/reth/issues/8732
+engine-withdrawals:
+  - Withdrawals Fork On Genesis (Paris) (reth)
+  - Withdrawals Fork on Block 1 (Paris) (reth)
+  - Withdrawals Fork on Block 2 (Paris) (reth)
+  - Withdrawals Fork on Block 3 (Paris) (reth)
+  - Withdraw to a single account (Paris) (reth)
+  - Withdraw to two accounts (Paris) (reth)
+  - Withdraw many accounts (Paris) (reth)
+  - Withdraw zero amount (Paris) (reth)
+  - Empty Withdrawals (Paris) (reth)
+  - Corrupted Block Hash Payload (INVALID) (Paris) (reth)
+  - Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload (Paris) (reth)
+  - Withdrawals Fork on Block 1 - 8 Block Re-Org, Sync (Paris) (reth)
+  - Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload (Paris) (reth)
+  - Withdrawals Fork on Block 8 - 10 Block Re-Org Sync (Paris) (reth)
+  - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth)
+  - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org Sync (Paris) (reth)
+  - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth)
+  - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth)
+
+  # https://github.com/paradigmxyz/reth/issues/8304#issuecomment-2208515839
+  - Sync after 2 blocks - Withdrawals on Block 1 - Single Withdrawal Account - No Transactions (Paris) (reth)
+  - Sync after 2 blocks - Withdrawals on Block 1 - Single Withdrawal Account (Paris) (reth)
+  - Sync after 2 blocks - Withdrawals on Genesis - Single Withdrawal Account (Paris) (reth)
+  - Sync after 2 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts - No Transactions (Paris) (reth)
+  - Sync after 2 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts (Paris) (reth)
+  - Sync after 128 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts (Paris) (reth)
+
+# https://github.com/paradigmxyz/reth/issues/8305
+# https://github.com/paradigmxyz/reth/issues/6217
+engine-api:
+  - Inconsistent Head in ForkchoiceState (Paris) (reth)
+  - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Paris) (reth)
+  - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)
+  - Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)
+
+  # Hive issue
+  # https://github.com/ethereum/hive/issues/1135
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+
+# https://github.com/paradigmxyz/reth/issues/8305
+# https://github.com/paradigmxyz/reth/issues/6217
+# https://github.com/paradigmxyz/reth/issues/8306
+# https://github.com/paradigmxyz/reth/issues/7144
+engine-cancun:
+  - Blob Transaction Ordering, Multiple Clients (Cancun) (reth)
+  - Inconsistent Head in ForkchoiceState (Cancun) (reth)
+  - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Cancun) (reth)
+  - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+  - Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)
+  - Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth)
+  - Invalid NewPayload, ParentBeaconBlockRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+  - Invalid NewPayload, BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+  - Invalid NewPayload, Blob Count on BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+  - Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+
+  # Hive issue
+  # https://github.com/ethereum/hive/issues/1135
+  - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+
+# https://github.com/paradigmxyz/reth/issues/8579
+sync:
+  - sync reth -> reth

.github/assets/hive/parse.py

@@ -0,0 +1,43 @@
+import json
+import yaml
+import sys
+import argparse
+
+# Argument parser setup
+parser = argparse.ArgumentParser(description="Check for unexpected test results based on an exclusion list.")
+parser.add_argument("report_json", help="Path to the hive report JSON file.")
+parser.add_argument("--exclusion", required=True, help="Path to the exclusion YAML file.")
+args = parser.parse_args()
+
+# Load hive JSON
+with open(args.report_json, 'r') as file:
+    report = json.load(file)
+
+# Load exclusion YAML
+with open(args.exclusion, 'r') as file:
+    exclusion_data = yaml.safe_load(file)
+    exclusions = exclusion_data.get(report['name'], [])
+
+# Collect unexpected failures and passes
+unexpected_failures = []
+unexpected_passes = []
+
+for test in report['testCases'].values():
+    test_name = test['name']
+    test_pass = test['summaryResult']['pass']
+    if test_name in exclusions:
+        if test_pass:
+            unexpected_passes.append(test_name)
+    else:
+        if not test_pass:
+            unexpected_failures.append(test_name)
+
+# Check if there are any unexpected failures or passes and exit with error
+if unexpected_failures or unexpected_passes:
+    if unexpected_failures:
+        print("Unexpected Failures:", unexpected_failures)
+    if unexpected_passes:
+        print("Unexpected Passes:", unexpected_passes)
+    sys.exit(1)
+
+print("Success.")

.github/workflows/hive.yml

@@ -72,12 +72,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # TODO: enable etherem/sync once resolved:
-        # https://github.com/paradigmxyz/reth/issues/8579
         # ethereum/rpc to be deprecated:
         # https://github.com/ethereum/hive/pull/1117
-        # sim: [smoke/genesis, smoke/network, ethereum/sync]
-        sim: [smoke/genesis, smoke/network]
+        sim: [smoke/genesis, smoke/network, ethereum/sync]
         include:
           - sim: devp2p
             limit: discv4
@@ -104,26 +101,16 @@ jobs:
               - TestBlobViolations
           - sim: ethereum/engine
             limit: engine-exchange-capabilities
-          # TODO: enable engine-withdrawals once resolved:
-          # https://github.com/paradigmxyz/reth/issues/8732
-          # - sim: ethereum/engine
-          #   limit: engine-withdrawals
+          - sim: ethereum/engine
+            limit: engine-withdrawals
           - sim: ethereum/engine
             limit: engine-auth
           - sim: ethereum/engine
             limit: engine-transition
-          # TODO: enable engine-api once resolved:
-          # https://github.com/paradigmxyz/reth/issues/8305
-          # https://github.com/paradigmxyz/reth/issues/6217
-          # - sim: ethereum/engine
-          #   limit: engine-api
-          # TODO: enable cancun once resolved:
-          # https://github.com/paradigmxyz/reth/issues/8305
-          # https://github.com/paradigmxyz/reth/issues/6217
-          # https://github.com/paradigmxyz/reth/issues/8306
-          # https://github.com/paradigmxyz/reth/issues/7144
-          # - sim: ethereum/engine
-          #   limit: cancun
+          - sim: ethereum/engine
+            limit: engine-api
+          - sim: ethereum/engine
+            limit: cancun
           # eth_ rpc methods
           - sim: ethereum/rpc-compat
             include:
@@ -144,12 +131,9 @@ jobs:
               - eth_getTransactionReceipt
               - eth_sendRawTransaction
               - eth_syncing
-          # TODO: enable debug_ rpc-compat once resolved:
-          # https://github.com/paradigmxyz/reth/issues/7015
-          # https://github.com/paradigmxyz/reth/issues/6332
           # debug_ rpc methods
-          # - sim: ethereum/rpc-compat
-          # include: [debug_]
+          - sim: ethereum/rpc-compat
+            include: [debug_]
           # Pyspec cancun jobs
           - sim: pyspec
             include: [cancun/eip4844]
@@ -190,6 +174,10 @@ jobs:
     permissions:
       issues: write
     steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
       - name: Download artifacts
         uses: actions/download-artifact@v4
         with:
@@ -215,7 +203,11 @@ jobs:
       - name: Run ${{ matrix.sim }} simulator
         run: |
           cd hivetests
-          hive --sim "${{ matrix.sim }}$" --sim.limit "${{matrix.limit}}/${{join(matrix.include, '|')}}" --client reth
+          hive --sim "${{ matrix.sim }}$" --sim.limit "${{matrix.limit}}/${{join(matrix.include, '|')}}" --sim.parallelism 2 --client reth || true
+
+      - name: Parse hive output
+        run: |
+          find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.yaml
 
       - name: Create github issue if sim failed
         env:
@@ -239,6 +231,7 @@ jobs:
             -f title='Hive Test Failure: ${{ matrix.sim }}' \
             -f body="!!!!!!! This is an automated issue created by the hive test failure !!!!!!!<br /><br />The hive test for ${{ matrix.sim }} failed. Please investigate and fix the issue.<br /><br />[Link to the failed run](https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }})" \
             -f "labels[]=C-hivetest"
+
       - name: Print simulator output
         if: ${{ failure() }}
         run: |

.github/workflows/integration.yml

@@ -31,7 +31,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - name: Install Geth
-        run: .github/scripts/install_geth.sh
+        run: .github/assets/install_geth.sh
       - uses: taiki-e/install-action@nextest
       - uses: Swatinem/rust-cache@v2
         with:

.github/workflows/label-pr.yml

@@ -19,5 +19,5 @@ jobs:
         uses: actions/github-script@v7
         with:
           script: |
-            const label_pr = require('./.github/scripts/label_pr.js')
+            const label_pr = require('./.github/assets/label_pr.js')
             await label_pr({github, context})

.github/workflows/lint.yml

@@ -58,7 +58,7 @@ jobs:
         with:
           cache-on-failure: true
       - name: Run no_std checks
-        run: .github/scripts/check_no_std.sh
+        run: .github/assets/check_no_std.sh
 
   crate-checks:
     runs-on: ubuntu-latest