From 2b77109ddb12a5ef4eb6cd1e7ae466a7ae271f09 Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Tue, 2 Jul 2024 14:19:58 +0200
Subject: [PATCH 01/13] wip
---
.github/{scripts => assets}/check_no_std.sh | 0
.github/assets/hive_exclusion.json | 70 +++++++++++++++++++++
.github/{scripts => assets}/install_geth.sh | 0
.github/{scripts => assets}/label_pr.js | 0
.github/assets/parse.py | 42 +++++++++++++
.github/workflows/hive.yml | 69 ++++++++++----------
.github/workflows/integration.yml | 2 +-
.github/workflows/label-pr.yml | 2 +-
.github/workflows/lint.yml | 2 +-
9 files changed, 152 insertions(+), 35 deletions(-)
rename .github/{scripts => assets}/check_no_std.sh (100%)
create mode 100644 .github/assets/hive_exclusion.json
rename .github/{scripts => assets}/install_geth.sh (100%)
rename .github/{scripts => assets}/label_pr.js (100%)
create mode 100644 .github/assets/parse.py
diff --git a/.github/scripts/check_no_std.sh b/.github/assets/check_no_std.sh
similarity index 100%
rename from .github/scripts/check_no_std.sh
rename to .github/assets/check_no_std.sh
diff --git a/.github/assets/hive_exclusion.json b/.github/assets/hive_exclusion.json
new file mode 100644
index 000000000000..991297ae03e7
--- /dev/null
+++ b/.github/assets/hive_exclusion.json
@@ -0,0 +1,70 @@
+{
+ "rpc-compat": [
+ "debug_getRawBlock/get-invalid-number (reth)",
+ "debug_getRawHeader/get-invalid-number (reth)",
+ "debug_getRawReceipts/get-invalid-number (reth)",
+ "debug_getRawTransaction/get-invalid-hash (reth)",
+
+ "eth_call/call-callenv (reth)",
+ "eth_createAccessList/create-al-contract-eip1559 (reth)",
+ "eth_createAccessList/create-al-contract (reth)",
+ "eth_feeHistory/fee-history (reth)",
+ "eth_getStorageAt/get-storage-invalid-key-too-large (reth)",
+ "eth_getStorageAt/get-storage-invalid-key (reth)",
+ "eth_getTransactionReceipt/get-access-list (reth)",
+ "eth_getTransactionReceipt/get-blob-tx (reth)",
+ "eth_getTransactionReceipt/get-dynamic-fee (reth)"
+ ],
+ "engine": [
+ "Blob Transaction Ordering, Multiple Clients (Cancun) (reth)",
+ "Inconsistent Head in ForkchoiceState (Cancun) (reth)",
+ "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Cancun) (reth)",
+ "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
+ "Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)",
+ "Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth)",
+ "Invalid NewPayload, ParentBeaconBlockRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
+ "Invalid NewPayload, BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
+ "Invalid NewPayload, Blob Count on BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
+ "Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
+
+ "Inconsistent Head in ForkchoiceState (Paris) (reth)",
+ "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Paris) (reth)",
+ "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)",
+ "Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Paris) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Paris) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)",
+
+ "Withdrawals Fork On Genesis (Paris) (reth)",
+ "Withdrawals Fork on Block 1 (Paris) (reth)",
+ "Withdrawals Fork on Block 2 (Paris) (reth)",
+ "Withdrawals Fork on Block 3 (Paris) (reth)",
+ "Withdraw to a single account (Paris) (reth)",
+ "Withdraw to two accounts (Paris) (reth)",
+ "Withdraw many accounts (Paris) (reth)",
+ "Withdraw zero amount (Paris) (reth)",
+ "Empty Withdrawals (Paris) (reth)",
+ "Corrupted Block Hash Payload (INVALID) (Paris) (reth)",
+ "Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload (Paris) (reth)",
+ "Withdrawals Fork on Block 1 - 8 Block Re-Org, Sync (Paris) (reth)",
+ "Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload (Paris) (reth)",
+ "Withdrawals Fork on Block 8 - 10 Block Re-Org Sync (Paris) (reth)",
+ "Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth)",
+ "Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org Sync (Paris) (reth)",
+ "Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth)",
+ "Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth)"
+
+ ],
+ "sync": [
+ "sync reth -> reth"
+ ]
+}
\ No newline at end of file
diff --git a/.github/scripts/install_geth.sh b/.github/assets/install_geth.sh
similarity index 100%
rename from .github/scripts/install_geth.sh
rename to .github/assets/install_geth.sh
diff --git a/.github/scripts/label_pr.js b/.github/assets/label_pr.js
similarity index 100%
rename from .github/scripts/label_pr.js
rename to .github/assets/label_pr.js
diff --git a/.github/assets/parse.py b/.github/assets/parse.py
new file mode 100644
index 000000000000..eece2dd5da48
--- /dev/null
+++ b/.github/assets/parse.py
@@ -0,0 +1,42 @@
+import json
+import sys
+import argparse
+
+# Argument parser setup
+parser = argparse.ArgumentParser(description="Check for unexpected test results based on an exclusion list.")
+parser.add_argument("main_json", help="Path to the main JSON file.")
+parser.add_argument("--exclusion", required=True, help="Path to the exclusion JSON file.")
+args = parser.parse_args()
+
+# Load main JSON
+with open(args.main_json, 'r') as file:
+ main_data = json.load(file)
+
+# Load exclusion JSON
+with open(args.exclusion, 'r') as file:
+ exclusion_data = json.load(file)
+ exclusions = exclusion_data.get(main_data['name'], [])
+
+# Collect unexpected failures and passes
+unexpected_failures = []
+unexpected_passes = []
+
+for test in main_data['testCases'].values():
+ test_name = test['name']
+ test_pass = test['summaryResult']['pass']
+ if test_name in exclusions:
+ if test_pass:
+ unexpected_passes.append(test_name)
+ else:
+ if not test_pass:
+ unexpected_failures.append(test_name)
+
+# Check if there are any unexpected failures or passes and exit with error
+if unexpected_failures or unexpected_passes:
+ if unexpected_failures:
+ print("Unexpected Failures:", unexpected_failures)
+ if unexpected_passes:
+ print("Unexpected Passes:", unexpected_passes)
+ sys.exit(1)
+
+print("All tests passed or expected failures/passes as per the exclusion list.")
\ No newline at end of file
diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml
index 3340393d0873..c7ac25cc536a 100644
--- a/.github/workflows/hive.yml
+++ b/.github/workflows/hive.yml
@@ -70,8 +70,8 @@ jobs:
# https://github.com/paradigmxyz/reth/issues/8579
# ethereum/rpc to be deprecated:
# https://github.com/ethereum/hive/pull/1117
- # sim: [smoke/genesis, smoke/network, ethereum/sync]
- sim: [smoke/genesis, smoke/network]
+ sim: [smoke/genesis, smoke/network, ethereum/sync]
+ # sim: [smoke/genesis, smoke/network]
include:
- sim: devp2p
limit: discv4
@@ -100,8 +100,8 @@ jobs:
limit: engine-exchange-capabilities
# TODO: enable engine-withdrawals once resolved:
# https://github.com/paradigmxyz/reth/issues/8732
- # - sim: ethereum/engine
- # limit: engine-withdrawals
+ - sim: ethereum/engine
+ limit: engine-withdrawals
- sim: ethereum/engine
limit: engine-auth
- sim: ethereum/engine
@@ -109,15 +109,15 @@ jobs:
# TODO: enable engine-api once resolved:
# https://github.com/paradigmxyz/reth/issues/8305
# https://github.com/paradigmxyz/reth/issues/6217
- # - sim: ethereum/engine
- # limit: engine-api
+ - sim: ethereum/engine
+ limit: engine-api
# TODO: enable cancun once resolved:
# https://github.com/paradigmxyz/reth/issues/8305
# https://github.com/paradigmxyz/reth/issues/6217
# https://github.com/paradigmxyz/reth/issues/8306
# https://github.com/paradigmxyz/reth/issues/7144
- # - sim: ethereum/engine
- # limit: cancun
+ - sim: ethereum/engine
+ limit: cancun
# eth_ rpc methods
- sim: ethereum/rpc-compat
include:
@@ -142,8 +142,8 @@ jobs:
# https://github.com/paradigmxyz/reth/issues/7015
# https://github.com/paradigmxyz/reth/issues/6332
# debug_ rpc methods
- # - sim: ethereum/rpc-compat
- # include: [debug_]
+ - sim: ethereum/rpc-compat
+ include: [debug_]
# Pyspec cancun jobs
- sim: pyspec
include: [cancun/eip4844]
@@ -209,30 +209,35 @@ jobs:
- name: Run ${{ matrix.sim }} simulator
run: |
cd hivetests
- hive --sim "${{ matrix.sim }}$" --sim.limit "${{matrix.limit}}/${{join(matrix.include, '|')}}" --client reth
+ hive --sim "${{ matrix.sim }}$" --sim.limit "${{matrix.limit}}/${{join(matrix.include, '|')}}" --client reth || true
- - name: Create github issue if sim failed
- env:
- GH_TOKEN: ${{ github.token }}
- if: ${{ failure() }}
+ # - name: Create github issue if sim failed
+ # env:
+ # GH_TOKEN: ${{ github.token }}
+ # if: ${{ failure() }}
+ # run: |
+ # echo "Simulator failed, creating issue"
+ # # Check if issue already exists
+ # # get all issues with the label C-hivetest, loop over each page and check if the issue already exists
+
+ # existing_issues=$(gh api /repos/paradigmxyz/reth/issues -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -F "labels=C-hivetest" --method GET | jq '.[].title')
+ # if [[ $existing_issues == *"Hive Test Failure: ${{ matrix.sim }}"* ]]; then
+ # echo "Issue already exists"
+ # exit 0
+ # fi
+ # gh api \
+ # --method POST \
+ # -H "Accept: application/vnd.github+json" \
+ # -H "X-GitHub-Api-Version: 2022-11-28" \
+ # /repos/${{ github.repository }}/issues \
+ # -f title='Hive Test Failure: ${{ matrix.sim }}' \
+ # -f body="!!!!!!! This is an automated issue created by the hive test failure !!!!!!!
The hive test for ${{ matrix.sim }} failed. Please investigate and fix the issue.
[Link to the failed run](https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }})" \
+ # -f "labels[]=C-hivetest"
+
+ - name: Parse hive output
run: |
- echo "Simulator failed, creating issue"
- # Check if issue already exists
- # get all issues with the label C-hivetest, loop over each page and check if the issue already exists
-
- existing_issues=$(gh api /repos/paradigmxyz/reth/issues -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -F "labels=C-hivetest" --method GET | jq '.[].title')
- if [[ $existing_issues == *"Hive Test Failure: ${{ matrix.sim }}"* ]]; then
- echo "Issue already exists"
- exit 0
- fi
- gh api \
- --method POST \
- -H "Accept: application/vnd.github+json" \
- -H "X-GitHub-Api-Version: 2022-11-28" \
- /repos/${{ github.repository }}/issues \
- -f title='Hive Test Failure: ${{ matrix.sim }}' \
- -f body="!!!!!!! This is an automated issue created by the hive test failure !!!!!!!
The hive test for ${{ matrix.sim }} failed. Please investigate and fix the issue.
[Link to the failed run](https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }})" \
- -f "labels[]=C-hivetest"
+ find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" -exec python .github/assets/parse.py {} --exclusion .github/assets/hive_exclusion.json +
+
- name: Print simulator output
if: ${{ failure() }}
run: |
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index b4b494a90256..103a87706bca 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -31,7 +31,7 @@ jobs:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- name: Install Geth
- run: .github/scripts/install_geth.sh
+ run: .github/assets/install_geth.sh
- uses: taiki-e/install-action@nextest
- uses: Swatinem/rust-cache@v2
with:
diff --git a/.github/workflows/label-pr.yml b/.github/workflows/label-pr.yml
index 857d354a8fb8..07727173531b 100644
--- a/.github/workflows/label-pr.yml
+++ b/.github/workflows/label-pr.yml
@@ -19,5 +19,5 @@ jobs:
uses: actions/github-script@v7
with:
script: |
- const label_pr = require('./.github/scripts/label_pr.js')
+ const label_pr = require('./.github/assets/label_pr.js')
await label_pr({github, context})
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index c07cee38830b..d0329aefc89f 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -58,7 +58,7 @@ jobs:
with:
cache-on-failure: true
- name: Run no_std checks
- run: .github/scripts/check_no_std.sh
+ run: .github/assets/check_no_std.sh
crate-checks:
runs-on: ubuntu-latest
From f3c45734bfbdb093e378e86295c08bc79cd1afe1 Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Tue, 2 Jul 2024 14:53:47 +0200
Subject: [PATCH 02/13] use xargs instead
---
.github/workflows/hive.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml
index c7ac25cc536a..379023893849 100644
--- a/.github/workflows/hive.yml
+++ b/.github/workflows/hive.yml
@@ -236,7 +236,7 @@ jobs:
- name: Parse hive output
run: |
- find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" -exec python .github/assets/parse.py {} --exclusion .github/assets/hive_exclusion.json +
+ find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/parse.py {} --exclusion .github/assets/hive_exclusion.json
- name: Print simulator output
if: ${{ failure() }}
From 4d2147cb9142cf8726bcd1b723d5265a1a830745 Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Tue, 2 Jul 2024 15:26:56 +0200
Subject: [PATCH 03/13] checkout repo for assets
---
.github/workflows/hive.yml | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml
index 379023893849..a1cdfe46026b 100644
--- a/.github/workflows/hive.yml
+++ b/.github/workflows/hive.yml
@@ -184,6 +184,10 @@ jobs:
permissions:
issues: write
steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
- name: Download artifacts
uses: actions/download-artifact@v4
with:
@@ -233,7 +237,7 @@ jobs:
# -f title='Hive Test Failure: ${{ matrix.sim }}' \
# -f body="!!!!!!! This is an automated issue created by the hive test failure !!!!!!!
The hive test for ${{ matrix.sim }} failed. Please investigate and fix the issue.
[Link to the failed run](https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }})" \
# -f "labels[]=C-hivetest"
-
+
- name: Parse hive output
run: |
find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/parse.py {} --exclusion .github/assets/hive_exclusion.json
From 2f0389f58ad3a9b2ac9884c7c5405bf3007ddb27 Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Tue, 2 Jul 2024 16:00:28 +0200
Subject: [PATCH 04/13] rm expected failure for testing
---
.github/assets/hive_exclusion.json | 3 ---
1 file changed, 3 deletions(-)
diff --git a/.github/assets/hive_exclusion.json b/.github/assets/hive_exclusion.json
index 991297ae03e7..bad814011b6d 100644
--- a/.github/assets/hive_exclusion.json
+++ b/.github/assets/hive_exclusion.json
@@ -63,8 +63,5 @@
"Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth)",
"Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth)"
- ],
- "sync": [
- "sync reth -> reth"
]
}
\ No newline at end of file
From 0f4e400a58489481d7a93a25db44fb08d6519921 Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Tue, 2 Jul 2024 16:26:50 +0200
Subject: [PATCH 05/13] Revert "rm expected failure for testing"
This reverts commit 2f0389f58ad3a9b2ac9884c7c5405bf3007ddb27.
---
.github/assets/hive_exclusion.json | 3 +++
1 file changed, 3 insertions(+)
diff --git a/.github/assets/hive_exclusion.json b/.github/assets/hive_exclusion.json
index bad814011b6d..991297ae03e7 100644
--- a/.github/assets/hive_exclusion.json
+++ b/.github/assets/hive_exclusion.json
@@ -63,5 +63,8 @@
"Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth)",
"Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth)"
+ ],
+ "sync": [
+ "sync reth -> reth"
]
}
\ No newline at end of file
From ee9322b1d4b4043d34afe3dc2b72ea3c6b8cdf75 Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Tue, 2 Jul 2024 16:29:11 +0200
Subject: [PATCH 06/13] update
---
.../assets/{hive_exclusion.json => hive/expected_failures.json} | 0
.github/assets/{ => hive}/parse.py | 2 +-
.github/workflows/hive.yml | 2 +-
3 files changed, 2 insertions(+), 2 deletions(-)
rename .github/assets/{hive_exclusion.json => hive/expected_failures.json} (100%)
rename .github/assets/{ => hive}/parse.py (94%)
diff --git a/.github/assets/hive_exclusion.json b/.github/assets/hive/expected_failures.json
similarity index 100%
rename from .github/assets/hive_exclusion.json
rename to .github/assets/hive/expected_failures.json
diff --git a/.github/assets/parse.py b/.github/assets/hive/parse.py
similarity index 94%
rename from .github/assets/parse.py
rename to .github/assets/hive/parse.py
index eece2dd5da48..43037a2fa260 100644
--- a/.github/assets/parse.py
+++ b/.github/assets/hive/parse.py
@@ -39,4 +39,4 @@
print("Unexpected Passes:", unexpected_passes)
sys.exit(1)
-print("All tests passed or expected failures/passes as per the exclusion list.")
\ No newline at end of file
+print("Success.")
\ No newline at end of file
diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml
index a1cdfe46026b..d0dc52a41ef7 100644
--- a/.github/workflows/hive.yml
+++ b/.github/workflows/hive.yml
@@ -240,7 +240,7 @@ jobs:
- name: Parse hive output
run: |
- find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/parse.py {} --exclusion .github/assets/hive_exclusion.json
+ find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.json
- name: Print simulator output
if: ${{ failure() }}
From c0d1eb0d1fda8eed0a45d50881c7a000652c8815 Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Tue, 2 Jul 2024 16:31:46 +0200
Subject: [PATCH 07/13] re-enable gh issue creation
---
.github/workflows/hive.yml | 46 +++++++++++++++++++-------------------
1 file changed, 23 insertions(+), 23 deletions(-)
diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml
index d0dc52a41ef7..463c420abe59 100644
--- a/.github/workflows/hive.yml
+++ b/.github/workflows/hive.yml
@@ -215,33 +215,33 @@ jobs:
cd hivetests
hive --sim "${{ matrix.sim }}$" --sim.limit "${{matrix.limit}}/${{join(matrix.include, '|')}}" --client reth || true
- # - name: Create github issue if sim failed
- # env:
- # GH_TOKEN: ${{ github.token }}
- # if: ${{ failure() }}
- # run: |
- # echo "Simulator failed, creating issue"
- # # Check if issue already exists
- # # get all issues with the label C-hivetest, loop over each page and check if the issue already exists
-
- # existing_issues=$(gh api /repos/paradigmxyz/reth/issues -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -F "labels=C-hivetest" --method GET | jq '.[].title')
- # if [[ $existing_issues == *"Hive Test Failure: ${{ matrix.sim }}"* ]]; then
- # echo "Issue already exists"
- # exit 0
- # fi
- # gh api \
- # --method POST \
- # -H "Accept: application/vnd.github+json" \
- # -H "X-GitHub-Api-Version: 2022-11-28" \
- # /repos/${{ github.repository }}/issues \
- # -f title='Hive Test Failure: ${{ matrix.sim }}' \
- # -f body="!!!!!!! This is an automated issue created by the hive test failure !!!!!!!
The hive test for ${{ matrix.sim }} failed. Please investigate and fix the issue.
[Link to the failed run](https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }})" \
- # -f "labels[]=C-hivetest"
-
- name: Parse hive output
run: |
find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.json
+ - name: Create github issue if sim failed
+ env:
+ GH_TOKEN: ${{ github.token }}
+ if: ${{ failure() }}
+ run: |
+ echo "Simulator failed, creating issue"
+ # Check if issue already exists
+ # get all issues with the label C-hivetest, loop over each page and check if the issue already exists
+
+ existing_issues=$(gh api /repos/paradigmxyz/reth/issues -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -F "labels=C-hivetest" --method GET | jq '.[].title')
+ if [[ $existing_issues == *"Hive Test Failure: ${{ matrix.sim }}"* ]]; then
+ echo "Issue already exists"
+ exit 0
+ fi
+ gh api \
+ --method POST \
+ -H "Accept: application/vnd.github+json" \
+ -H "X-GitHub-Api-Version: 2022-11-28" \
+ /repos/${{ github.repository }}/issues \
+ -f title='Hive Test Failure: ${{ matrix.sim }}' \
+ -f body="!!!!!!! This is an automated issue created by the hive test failure !!!!!!!
The hive test for ${{ matrix.sim }} failed. Please investigate and fix the issue.
[Link to the failed run](https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }})" \
+ -f "labels[]=C-hivetest"
+
- name: Print simulator output
if: ${{ failure() }}
run: |
From d3c02c560b639122a6606b363d93cdf571349dfc Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Tue, 2 Jul 2024 17:42:25 +0200
Subject: [PATCH 08/13] update suite names
---
.github/assets/hive/expected_failures.json | 60 +++++++++++-----------
1 file changed, 31 insertions(+), 29 deletions(-)
diff --git a/.github/assets/hive/expected_failures.json b/.github/assets/hive/expected_failures.json
index 991297ae03e7..23c9d51305ba 100644
--- a/.github/assets/hive/expected_failures.json
+++ b/.github/assets/hive/expected_failures.json
@@ -15,35 +15,7 @@
"eth_getTransactionReceipt/get-blob-tx (reth)",
"eth_getTransactionReceipt/get-dynamic-fee (reth)"
],
- "engine": [
- "Blob Transaction Ordering, Multiple Clients (Cancun) (reth)",
- "Inconsistent Head in ForkchoiceState (Cancun) (reth)",
- "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Cancun) (reth)",
- "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
- "Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)",
- "Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth)",
- "Invalid NewPayload, ParentBeaconBlockRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
- "Invalid NewPayload, BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
- "Invalid NewPayload, Blob Count on BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
- "Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
-
- "Inconsistent Head in ForkchoiceState (Paris) (reth)",
- "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Paris) (reth)",
- "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)",
- "Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Paris) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Paris) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)",
-
+ "engine-withdrawals": [
"Withdrawals Fork On Genesis (Paris) (reth)",
"Withdrawals Fork on Block 1 (Paris) (reth)",
"Withdrawals Fork on Block 2 (Paris) (reth)",
@@ -64,6 +36,36 @@
"Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth)"
],
+ "engine-api": [
+ "Inconsistent Head in ForkchoiceState (Paris) (reth)",
+ "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Paris) (reth)",
+ "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)",
+ "Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Paris) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Paris) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)"
+ ],
+ "engine-cancun": [
+ "Blob Transaction Ordering, Multiple Clients (Cancun) (reth)",
+ "Inconsistent Head in ForkchoiceState (Cancun) (reth)",
+ "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Cancun) (reth)",
+ "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
+ "Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)",
+ "Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)",
+ "Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth)",
+ "Invalid NewPayload, ParentBeaconBlockRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
+ "Invalid NewPayload, BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
+ "Invalid NewPayload, Blob Count on BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
+ "Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)"
+ ],
"sync": [
"sync reth -> reth"
]
From 00b5561a23552671054715c107c7bfbb3aa83ceb Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Wed, 3 Jul 2024 16:03:55 +0200
Subject: [PATCH 09/13] use yaml instead + move issues
---
.github/assets/hive/expected_failures.json | 72 --------------------
.github/assets/hive/expected_failures.yaml | 78 ++++++++++++++++++++++
.github/assets/hive/parse.py | 16 ++---
.github/workflows/hive.yml | 18 +----
4 files changed, 87 insertions(+), 97 deletions(-)
delete mode 100644 .github/assets/hive/expected_failures.json
create mode 100644 .github/assets/hive/expected_failures.yaml
diff --git a/.github/assets/hive/expected_failures.json b/.github/assets/hive/expected_failures.json
deleted file mode 100644
index 23c9d51305ba..000000000000
--- a/.github/assets/hive/expected_failures.json
+++ /dev/null
@@ -1,72 +0,0 @@
-{
- "rpc-compat": [
- "debug_getRawBlock/get-invalid-number (reth)",
- "debug_getRawHeader/get-invalid-number (reth)",
- "debug_getRawReceipts/get-invalid-number (reth)",
- "debug_getRawTransaction/get-invalid-hash (reth)",
-
- "eth_call/call-callenv (reth)",
- "eth_createAccessList/create-al-contract-eip1559 (reth)",
- "eth_createAccessList/create-al-contract (reth)",
- "eth_feeHistory/fee-history (reth)",
- "eth_getStorageAt/get-storage-invalid-key-too-large (reth)",
- "eth_getStorageAt/get-storage-invalid-key (reth)",
- "eth_getTransactionReceipt/get-access-list (reth)",
- "eth_getTransactionReceipt/get-blob-tx (reth)",
- "eth_getTransactionReceipt/get-dynamic-fee (reth)"
- ],
- "engine-withdrawals": [
- "Withdrawals Fork On Genesis (Paris) (reth)",
- "Withdrawals Fork on Block 1 (Paris) (reth)",
- "Withdrawals Fork on Block 2 (Paris) (reth)",
- "Withdrawals Fork on Block 3 (Paris) (reth)",
- "Withdraw to a single account (Paris) (reth)",
- "Withdraw to two accounts (Paris) (reth)",
- "Withdraw many accounts (Paris) (reth)",
- "Withdraw zero amount (Paris) (reth)",
- "Empty Withdrawals (Paris) (reth)",
- "Corrupted Block Hash Payload (INVALID) (Paris) (reth)",
- "Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload (Paris) (reth)",
- "Withdrawals Fork on Block 1 - 8 Block Re-Org, Sync (Paris) (reth)",
- "Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload (Paris) (reth)",
- "Withdrawals Fork on Block 8 - 10 Block Re-Org Sync (Paris) (reth)",
- "Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth)",
- "Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org Sync (Paris) (reth)",
- "Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth)",
- "Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth)"
-
- ],
- "engine-api": [
- "Inconsistent Head in ForkchoiceState (Paris) (reth)",
- "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Paris) (reth)",
- "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)",
- "Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Paris) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Paris) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)"
- ],
- "engine-cancun": [
- "Blob Transaction Ordering, Multiple Clients (Cancun) (reth)",
- "Inconsistent Head in ForkchoiceState (Cancun) (reth)",
- "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Cancun) (reth)",
- "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
- "Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)",
- "Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)",
- "Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth)",
- "Invalid NewPayload, ParentBeaconBlockRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
- "Invalid NewPayload, BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
- "Invalid NewPayload, Blob Count on BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)",
- "Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)"
- ],
- "sync": [
- "sync reth -> reth"
- ]
-}
\ No newline at end of file
diff --git a/.github/assets/hive/expected_failures.yaml b/.github/assets/hive/expected_failures.yaml
new file mode 100644
index 000000000000..0e67b646471e
--- /dev/null
+++ b/.github/assets/hive/expected_failures.yaml
@@ -0,0 +1,78 @@
+# https://github.com/paradigmxyz/reth/issues/7015
+# https://github.com/paradigmxyz/reth/issues/6332
+rpc-compat:
+ - debug_getRawBlock/get-invalid-number (reth)
+ - debug_getRawHeader/get-invalid-number (reth)
+ - debug_getRawReceipts/get-invalid-number (reth)
+ - debug_getRawTransaction/get-invalid-hash (reth)
+
+ - eth_call/call-callenv (reth)
+ - eth_createAccessList/create-al-contract-eip1559 (reth)
+ - eth_createAccessList/create-al-contract (reth)
+ - eth_feeHistory/fee-history (reth)
+ - eth_getStorageAt/get-storage-invalid-key-too-large (reth)
+ - eth_getStorageAt/get-storage-invalid-key (reth)
+ - eth_getTransactionReceipt/get-access-list (reth)
+ - eth_getTransactionReceipt/get-blob-tx (reth)
+ - eth_getTransactionReceipt/get-dynamic-fee (reth)
+
+# https://github.com/paradigmxyz/reth/issues/8732
+engine-withdrawals:
+ - Withdrawals Fork On Genesis (Paris) (reth)
+ - Withdrawals Fork on Block 1 (Paris) (reth)
+ - Withdrawals Fork on Block 2 (Paris) (reth)
+ - Withdrawals Fork on Block 3 (Paris) (reth)
+ - Withdraw to a single account (Paris) (reth)
+ - Withdraw to two accounts (Paris) (reth)
+ - Withdraw many accounts (Paris) (reth)
+ - Withdraw zero amount (Paris) (reth)
+ - Empty Withdrawals (Paris) (reth)
+ - Corrupted Block Hash Payload (INVALID) (Paris) (reth)
+ - Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload (Paris) (reth)
+ - Withdrawals Fork on Block 1 - 8 Block Re-Org, Sync (Paris) (reth)
+ - Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload (Paris) (reth)
+ - Withdrawals Fork on Block 8 - 10 Block Re-Org Sync (Paris) (reth)
+ - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth)
+ - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org Sync (Paris) (reth)
+ - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth)
+ - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth)
+
+# https://github.com/paradigmxyz/reth/issues/8305
+# https://github.com/paradigmxyz/reth/issues/6217
+engine-api:
+ - Inconsistent Head in ForkchoiceState (Paris) (reth)
+ - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Paris) (reth)
+ - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)
+ - Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)
+
+# https://github.com/paradigmxyz/reth/issues/8305
+# https://github.com/paradigmxyz/reth/issues/6217
+# https://github.com/paradigmxyz/reth/issues/8306
+# https://github.com/paradigmxyz/reth/issues/7144
+engine-cancun:
+ - Blob Transaction Ordering, Multiple Clients (Cancun) (reth)
+ - Inconsistent Head in ForkchoiceState (Cancun) (reth)
+ - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Cancun) (reth)
+ - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+ - Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)
+ - Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth)
+ - Invalid NewPayload, ParentBeaconBlockRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+ - Invalid NewPayload, BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+ - Invalid NewPayload, Blob Count on BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+ - Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+
+# https://github.com/paradigmxyz/reth/issues/8579
+sync:
+ - sync reth -> reth
\ No newline at end of file
diff --git a/.github/assets/hive/parse.py b/.github/assets/hive/parse.py
index 43037a2fa260..6011dbd5b6bd 100644
--- a/.github/assets/hive/parse.py
+++ b/.github/assets/hive/parse.py
@@ -1,20 +1,20 @@
-import json
+import yaml
import sys
import argparse
# Argument parser setup
parser = argparse.ArgumentParser(description="Check for unexpected test results based on an exclusion list.")
-parser.add_argument("main_json", help="Path to the main JSON file.")
-parser.add_argument("--exclusion", required=True, help="Path to the exclusion JSON file.")
+parser.add_argument("main_yaml", help="Path to the main YAML file.")
+parser.add_argument("--exclusion", required=True, help="Path to the exclusion YAML file.")
args = parser.parse_args()
-# Load main JSON
-with open(args.main_json, 'r') as file:
- main_data = json.load(file)
+# Load main YAML
+with open(args.main_yaml, 'r') as file:
+ main_data = yaml.safe_load(file)
-# Load exclusion JSON
+# Load exclusion YAML
with open(args.exclusion, 'r') as file:
- exclusion_data = json.load(file)
+ exclusion_data = yaml.safe_load(file)
exclusions = exclusion_data.get(main_data['name'], [])
# Collect unexpected failures and passes
diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml
index 463c420abe59..d96e6b786910 100644
--- a/.github/workflows/hive.yml
+++ b/.github/workflows/hive.yml
@@ -66,12 +66,9 @@ jobs:
strategy:
fail-fast: false
matrix:
- # TODO: enable etherem/sync once resolved:
- # https://github.com/paradigmxyz/reth/issues/8579
# ethereum/rpc to be deprecated:
# https://github.com/ethereum/hive/pull/1117
sim: [smoke/genesis, smoke/network, ethereum/sync]
- # sim: [smoke/genesis, smoke/network]
include:
- sim: devp2p
limit: discv4
@@ -98,24 +95,14 @@ jobs:
- TestBlobViolations
- sim: ethereum/engine
limit: engine-exchange-capabilities
- # TODO: enable engine-withdrawals once resolved:
- # https://github.com/paradigmxyz/reth/issues/8732
- sim: ethereum/engine
limit: engine-withdrawals
- sim: ethereum/engine
limit: engine-auth
- sim: ethereum/engine
limit: engine-transition
- # TODO: enable engine-api once resolved:
- # https://github.com/paradigmxyz/reth/issues/8305
- # https://github.com/paradigmxyz/reth/issues/6217
- sim: ethereum/engine
limit: engine-api
- # TODO: enable cancun once resolved:
- # https://github.com/paradigmxyz/reth/issues/8305
- # https://github.com/paradigmxyz/reth/issues/6217
- # https://github.com/paradigmxyz/reth/issues/8306
- # https://github.com/paradigmxyz/reth/issues/7144
- sim: ethereum/engine
limit: cancun
# eth_ rpc methods
@@ -138,9 +125,6 @@ jobs:
- eth_getTransactionReceipt
- eth_sendRawTransaction
- eth_syncing
- # TODO: enable debug_ rpc-compat once resolved:
- # https://github.com/paradigmxyz/reth/issues/7015
- # https://github.com/paradigmxyz/reth/issues/6332
# debug_ rpc methods
- sim: ethereum/rpc-compat
include: [debug_]
@@ -217,7 +201,7 @@ jobs:
- name: Parse hive output
run: |
- find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.json
+ find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.yaml
- name: Create github issue if sim failed
env:
From 19feadf3e2c6225d8865d74c707a8a2a1b7dd461 Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Wed, 3 Jul 2024 16:06:08 +0200
Subject: [PATCH 10/13] hive uses json, exclusion list yaml
---
.github/assets/hive/parse.py | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/.github/assets/hive/parse.py b/.github/assets/hive/parse.py
index 6011dbd5b6bd..ee75fdf55317 100644
--- a/.github/assets/hive/parse.py
+++ b/.github/assets/hive/parse.py
@@ -1,27 +1,28 @@
+import json
import yaml
import sys
import argparse
# Argument parser setup
parser = argparse.ArgumentParser(description="Check for unexpected test results based on an exclusion list.")
-parser.add_argument("main_yaml", help="Path to the main YAML file.")
+parser.add_argument("report_json", help="Path to the hive report JSON file.")
parser.add_argument("--exclusion", required=True, help="Path to the exclusion YAML file.")
args = parser.parse_args()
-# Load main YAML
-with open(args.main_yaml, 'r') as file:
- main_data = yaml.safe_load(file)
+# Load hive JSON
+with open(args.report_json, 'r') as file:
+ report = json.load(file)
# Load exclusion YAML
with open(args.exclusion, 'r') as file:
exclusion_data = yaml.safe_load(file)
- exclusions = exclusion_data.get(main_data['name'], [])
+ exclusions = exclusion_data.get(report['name'], [])
# Collect unexpected failures and passes
unexpected_failures = []
unexpected_passes = []
-for test in main_data['testCases'].values():
+for test in report['testCases'].values():
test_name = test['name']
test_pass = test['summaryResult']['pass']
if test_name in exclusions:
From cfe9eb3c6626183aa0b8a51f9b327256ffc3d35c Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Wed, 3 Jul 2024 19:27:59 +0200
Subject: [PATCH 11/13] add more expected failures (hive related)
---
.github/assets/hive/expected_failures.yaml | 42 ++++++++++++++++++++++
1 file changed, 42 insertions(+)
diff --git a/.github/assets/hive/expected_failures.yaml b/.github/assets/hive/expected_failures.yaml
index 0e67b646471e..ba29ca5e7097 100644
--- a/.github/assets/hive/expected_failures.yaml
+++ b/.github/assets/hive/expected_failures.yaml
@@ -51,6 +51,27 @@ engine-api:
- Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)
- Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)
+ # Hive issue
+ # https://github.com/ethereum/hive/issues/1135
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+
# https://github.com/paradigmxyz/reth/issues/8305
# https://github.com/paradigmxyz/reth/issues/6217
# https://github.com/paradigmxyz/reth/issues/8306
@@ -73,6 +94,27 @@ engine-cancun:
- Invalid NewPayload, Blob Count on BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
- Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+ # Hive issue
+ # https://github.com/ethereum/hive/issues/1135
+ - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+ - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+
# https://github.com/paradigmxyz/reth/issues/8579
sync:
- sync reth -> reth
\ No newline at end of file
From f93c740f6c8c6e63f3b618fbc87cede98a999924 Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Thu, 4 Jul 2024 11:24:59 +0200
Subject: [PATCH 12/13] add time out failures to hive
---
.github/assets/hive/expected_failures.yaml | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/.github/assets/hive/expected_failures.yaml b/.github/assets/hive/expected_failures.yaml
index ba29ca5e7097..831cb966fb64 100644
--- a/.github/assets/hive/expected_failures.yaml
+++ b/.github/assets/hive/expected_failures.yaml
@@ -36,6 +36,14 @@ engine-withdrawals:
- Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org Sync (Paris) (reth)
- Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth)
- Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth)
+
+ # https://github.com/paradigmxyz/reth/issues/8304#issuecomment-2208515839
+ - Sync after 2 blocks - Withdrawals on Block 1 - Single Withdrawal Account - No Transactions (Paris) (reth)
+ - Sync after 2 blocks - Withdrawals on Block 1 - Single Withdrawal Account (Paris) (reth)
+ - Sync after 2 blocks - Withdrawals on Genesis - Single Withdrawal Account (Paris) (reth)
+ - Sync after 2 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts - No Transactions (Paris) (reth)
+ - Sync after 2 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts (Paris) (reth)
+ - Sync after 128 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts (Paris) (reth)
# https://github.com/paradigmxyz/reth/issues/8305
# https://github.com/paradigmxyz/reth/issues/6217
From 882315943e1a2c735b6f5fdeed66f0e149d1092f Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Thu, 4 Jul 2024 11:26:35 +0200
Subject: [PATCH 13/13] add parallelism 2
---
.github/workflows/hive.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml
index e859ec2866b8..d18ffd65ff91 100644
--- a/.github/workflows/hive.yml
+++ b/.github/workflows/hive.yml
@@ -203,7 +203,7 @@ jobs:
- name: Run ${{ matrix.sim }} simulator
run: |
cd hivetests
- hive --sim "${{ matrix.sim }}$" --sim.limit "${{matrix.limit}}/${{join(matrix.include, '|')}}" --client reth || true
+ hive --sim "${{ matrix.sim }}$" --sim.limit "${{matrix.limit}}/${{join(matrix.include, '|')}}" --sim.parallelism 2 --client reth || true
- name: Parse hive output
run: |